whatap 0.4.98 → 0.5.1

package/lib/logsink/sender-util.js

@@ -0,0 +1,51 @@
+var conf = require('../conf/configure');
+var SecurityMaster      = require('../net/security-master');
+var LogSinkPack = require('../pack/log-sink-pack');
+var DataPackSender = require('../data/datapack-sender');
+var DateUtil = require('../util/dateutil')
+var ZipSend = require('./zip/zip-send');
+
+function SenderUtil() {
+    this.conf = conf;
+    this.secu = SecurityMaster;
+}
+
+SenderUtil.prototype.send = function (log) {
+    const p = new LogSinkPack();
+    p.time = log.time;
+    if (!p.time) {
+        p.time = DateUtil.currentTime();
+    }
+    p.category = log.category;
+
+    if (this.secu.ONAME) {
+        p.tags.putString('oname', this.secu.ONAME);
+    }
+    if (this.conf.OKIND) {
+        p.tags.putString('okindName', this.conf.OKIND_NAME);
+    }
+    if (this.conf.ONODE) {
+        p.tags.putString('onodeName', this.conf.ONODE_NAME);
+    }
+
+    if (log.tags != null) {
+        p.tags.putAllMapValue(log.tags);
+    }
+    if (log.fields != null) {
+        if (p.fields == null) {
+            p.fields = log.fields;
+        } else {
+            p.fields.putAllMapValue(log.fields);
+        }
+    }
+
+    p.content = log.content;
+
+    if(conf.getProperty('logsink_zip_enabled', false)){
+        ZipSend.getInstance().add(p);
+    }else{
+        DataPackSender.sendLogSinkPack(p)
+    }
+};
+
+module.exports = SenderUtil;

package/lib/logsink/zip/zip-send.js

@@ -0,0 +1,177 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var conf            = require('../../conf/configure'),
+    DateUtil        = require('../../util/dateutil'),
+    RequestQueue    = require('../../util/request-queue'),
+    DataPackSender = require('../../data/datapack-sender'),
+    DataOutputX = require('../../io/data-outputx'),
+    ZipPack = require('../../pack/zip-pack'),
+    Logger          = require('../../logger');
+var zlib = require('zlib');
+
+var logsink_enabled = conf.getProperty('logsink_enabled', false);
+var logsink_zip_enabled = conf.getProperty('logsink_zip_enabled', false);
+var logsink_max_buffer_size = conf.getProperty('logsink_max_buffer_size', 1024*64);
+var profile_zip_min_size = conf.getProperty('profile_zip_min_size', 100)
+var logsink_queue_size = conf.getProperty('logsink_queue_size', 500);
+var logsink_max_wait_time = conf.getProperty('logsink_max_wait_time', 2000);
+conf.on('logsink_enabled', function(newProperty) {
+    logsink_enabled = newProperty;
+    if (logsink_enabled && logsink_zip_enabled) {
+        ZipSend.getInstance().startProcessQueue();
+    } else {
+        ZipSend.getInstance().stopProcessQueue();
+        ZipSend.resetInstance();
+    }
+});
+conf.on('logsink_zip_enabled', function(newProperty) {
+    logsink_zip_enabled = newProperty;
+    if (logsink_enabled && logsink_zip_enabled) {
+        ZipSend.getInstance().startProcessQueue();
+    } else {
+        ZipSend.getInstance().stopProcessQueue();
+        ZipSend.resetInstance();
+    }
+});
+
+class ZipSend {
+    constructor() {
+        if (ZipSend.instance) {
+            return ZipSend.instance;
+        }
+        this.queue = new RequestQueue(logsink_queue_size);
+        this.buffer = Buffer.alloc(0);
+        this.packCount = 0;
+        this.first_time = null;
+        this.isProcessing = false;
+        ZipSend.instance = this;
+
+        if (logsink_enabled && logsink_zip_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!ZipSend.instance) {
+            ZipSend.instance = new ZipSend();
+        }
+        return ZipSend.instance;
+    }
+
+    static resetInstance() {
+        if(ZipSend.instance){
+            ZipSend.instance = null;
+        }
+    }
+
+    add(p){
+        var ok = this.queue.put(p);
+        if(!ok){
+            DataPackSender.sendLogSinkPack(p);
+        }
+    }
+
+    // async addWait(p, waitTimeForFull) {
+    //     var ok = this.queue.put(p);
+    //     if (ok === false){
+    //         if(waitTimeForFull > 0){
+    //             while(this.queue.put(p) === false){
+    //                 await new Promise(resolve => setTimeout(resolve, waitTimeForFull));
+    //             }
+    //         }
+    //     }
+    // }
+
+    async processQueue() {
+        this.isProcessing = true;
+        while (this.isProcessing) {
+            if(!logsink_enabled || !logsink_zip_enabled) {
+                this.isProcessing = false;
+                break;
+            }
+
+            const p = await this.queue.getByTimeout(logsink_max_wait_time);
+            if(p) {
+                await this.append(p);
+            }else{
+                await this.sendAndClear();
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    async append(p) {
+        var b = DataOutputX.toBytesPack(p);
+        this.packCount++;
+        this.buffer = Buffer.concat([this.buffer, b]);
+
+        if(!this.first_time){
+            this.first_time = p.time;
+            if (this.buffer.length >= logsink_max_buffer_size) {
+                await this.sendAndClear();
+            }
+        }else{
+            if(this.buffer.length >= logsink_max_buffer_size || (p.time - this.first_time) >= conf.getProperty('logsink_max_wait_time', 2000)){
+                await this.sendAndClear();
+            }
+        }
+    }
+
+    async sendAndClear() {
+        if(this.buffer.length === 0){ return; }
+
+        var p = new ZipPack();
+        p.time = DateUtil.currentTime();
+        p.recordCount = this.packCount;
+        p.records = this.buffer;
+
+        await this.doZip(p);
+        if(conf.getProperty('debug_logsink_zip_enabled', false)){
+            Logger.print('WHATAP-703', `zip status=${p.status} records=${p.recordCount} | ${this.buffer.length} => ${p.records.length}`);
+        }
+        DataPackSender.sendLogSinkPack(p);
+
+        this.buffer = Buffer.alloc(0);
+        this.first_time = 0;
+        this.packCount = 0;
+    }
+
+    async doZip(p){
+        if(p.status !== 0){ return; }
+        if(p.records.length < profile_zip_min_size){ return; }
+        p.status = 1;
+
+        try{
+            p.records = await this._doZip(p.records);
+        }catch (error) {
+            Logger.print('WHATAP-701', 'Error occurred during compression.' + error, false);
+        }
+    }
+
+    _doZip(data) {
+        return new Promise((resolve, reject) => {
+            zlib.gzip(data, (err, buffer) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(buffer);
+                }
+            });
+        });
+    }
+
+    startProcessQueue() {
+        if (!this.isProcessing) {
+            this.processQueue()
+        }
+    }
+    stopProcessQueue() {
+        this.isProcessing = false;
+    }
+}
+
+module.exports = ZipSend;

package/lib/net/paramdef.js

@@ -34,6 +34,7 @@ exports.AGENT_LOG_READ = 23;
 
 exports.GET_ACTIVE_TRANSACTION_LIST = 25;
 exports.GET_ACTIVE_TRANSACTION_DETAIL = 26;
+exports.GET_TOPOLOGY = 27
 	
 exports.NODE_MODULE_DEPENDENCY = 101;
 

package/lib/net/security-master.js

@@ -105,7 +105,7 @@ var SecurityMaster = {
         }
 
         //OidUtil.setCmd(Configuration.getProperty("sun.java.command"));
-        this.ONAME = OidUtil.mkOname(Configuration.getProperty("whatap.name", name_pattern));
+        this.ONAME = OidUtil.mkOname(Configuration.getProperty("whatap.name", name_pattern), port);
         this.OID = OidUtil.mkOid(this.ONAME);
 
         if (lastOid != this.OID) {

package/lib/observers/global-observer.js

@@ -7,11 +7,28 @@
 var TraceContextManager = require('../trace/trace-context-manager'),
     Logger = require('../logger');
 
+var HttpStepX = require('../step/http-stepx');
+const Hexa32 = require("../util/hexa32");
+const SecurityMaster = require("../net/security-master");
+const conf = require("../conf/configure");
+const KeyGen = require("../util/keygen");
+const HashUtil = require("../util/hashutil");
+const DataTextAgent = require("../data/datatext-agent");
+const shimmer = require('../core/shimmer');
+const StatError = require("../stat/stat-error");
+const TextTypes = require("../lang/text-types");
+const URLPatternDetector  = require('../trace/serviceurl-pattern-detector').Detector;
+
 var GlobalObserver = function (agent) {
     this.agent = agent;
     this.packages = ['global'];
 }
 
+var transaction_status_error_enable = conf.getProperty('transaction_status_error_enable', true);
+conf.on('transaction_status_error_enable', function (newProps) {
+    transaction_status_error_enable = newProps;
+})
+
 GlobalObserver.prototype.inject = function (mod, moduleName) {
     var self = this;
 
@@ -26,6 +43,189 @@ GlobalObserver.prototype.inject = function (mod, moduleName) {
             });
         });
     });
+
+    shimmer.wrap(mod, 'fetch', function(original) {
+        return async function(...args) {
+            var info = args[1] ? args[1] : {};
+            var ctx = TraceContextManager._asyncLocalStorage.getStore();
+
+            if (ctx) {
+                interTxTraceAutoOn(ctx);
+
+                if (conf.mtrace_enabled) {
+                    addTraceHeaders(info, ctx);
+                    args[1] = info;
+                }
+            }
+
+            try {
+                const response = await original.apply(this, args);
+                handleResponse(ctx, args, response);
+                return response;
+            } catch (e) {
+                handleError(ctx, args, e);
+                throw e;
+            }
+        };
+    });
 };
 
+function addTraceHeaders(info, ctx) {
+    if (info.headers) {
+        info.headers['x-wtap-po'] = transferPOID(ctx);
+    } else {
+        info.headers = {
+            'x-wtap-po': transferPOID(ctx)
+        };
+    }
+    if (conf.stat_mtrace_enabled) {
+        info.headers[conf._trace_mtrace_spec_key1] = transferSPEC_URL(ctx);
+    }
+    if (conf.mtid_mtrace_enabled && ctx.mtid.isZero() === false) {
+        info.headers[conf._trace_mtrace_caller_key] = transferMTID_CALLERTX(ctx);
+    }
+}
+
+function handleResponse(ctx, args, response) {
+    if (!ctx || !args[0]) return;
+
+    const url = new URL(args[0]);
+    setupContext(ctx, url);
+
+    var step = createStep(ctx, url);
+
+    if (!response.ok && transaction_status_error_enable) {
+        recordError(ctx, step, response);
+    }
+
+    ctx.profile.push(step);
+}
+
+function handleError(ctx, args, error) {
+    if (!ctx || !args[0]) return;
+
+    const url = new URL(args[0]);
+    setupContext(ctx, url);
+
+    var step = createStep(ctx, url);
+
+    if (transaction_status_error_enable) {
+        recordError(ctx, step, error);
+    }
+
+    ctx.profile.push(step);
+}
+
+function setupContext(ctx, url) {
+    ctx.httpc_host = url.hostname;
+    ctx.httpc_url = url.pathname;
+    ctx.httpc_port = url.port || (url.protocol === 'https:' ? 443 : 80);
+}
+
+function createStep(ctx, url) {
+    var step = new HttpStepX();
+    step.start_time = ctx.getElapsedTime();
+    step.url = HashUtil.hashFromString(ctx.httpc_url);
+    DataTextAgent.HTTPC_URL.add(step.url, ctx.httpc_url);
+    step.host = HashUtil.hashFromString(ctx.httpc_host);
+    DataTextAgent.HTTPC_HOST.add(step.host, ctx.httpc_host);
+    step.port = ctx.httpc_port;
+    return step;
+}
+
+function recordError(ctx, step, response) {
+    if (step.error.isZero()) {
+        var cleanUrl = ctx.httpc_url.split('?')[0];
+        ctx.service_name = URLPatternDetector.normalize(cleanUrl);
+        ctx.service_hash = HashUtil.hashFromString(ctx.service_name);
+
+        const errorMessage = response.statusText || (response.cause ? response.cause.message : '') || response.message || 'Unknown error';
+        step.error = StatError.addError(response.status || null, errorMessage, ctx.service_hash, TextTypes.HTTPC_URL, step.url);
+
+        if (ctx.error.isZero()) {
+            ctx.error = step.error;
+            ctx.statusCode = response.status || null;
+            ctx.statusMessage = errorMessage;
+        }
+    }
+}
+
+var transfer_poid;
+function transferPOID(ctx) {
+    if (transfer_poid)
+        return transfer_poid;
+    transfer_poid = Hexa32.toString32(SecurityMaster.PCODE) + ','
+        + Hexa32.toString32(SecurityMaster.OKIND) + ',' + Hexa32.toString32(SecurityMaster.OID);
+    return transfer_poid;
+}
+
+function transferMTID_CALLERTX(ctx) {
+    if (ctx.transfer_id)
+        return ctx.transfer_id;
+    var x = Hexa32.toString32(ctx.mtid) + ',' + (ctx.mdepth + 1) + ',' + Hexa32.toString32(ctx.txid);
+    ctx.transfer_id = x;
+    return ctx.transfer_id;
+}
+
+function transferSPEC_URL(ctx) {
+    if (ctx.transfer_info)
+        return ctx.transfer_info;
+    var x = conf.mtrace_spec + ',' + ctx.service_hash;
+    ctx.transfer_info = x;
+    return ctx.transfer_info;
+}
+
+var check_seq = 1;
+function interTxTraceAutoOn(ctx) {
+    if (conf.mtrace_enabled == false || ctx.httpc_checked || ctx.mtid.isZero() === false)
+        return;
+    ctx.httpc_checked = true;
+    if (conf.mtrace_rate >= 100) {
+        ctx.mtid = KeyGen.next();
+        return;
+    }
+    check_seq++;
+    switch (Math.floor(conf.mtrace_rate / 10)) {
+        case 10:
+            ctx.mtid = KeyGen.next();
+            break;
+        case 9:
+            if (check_seq % 10 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 8:
+            if (check_seq % 5 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 7:
+            if (check_seq % 4 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 6:
+            if (check_seq % 3 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 5:
+            if (check_seq % 2 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 4:
+            if (check_seq % 3 === 0 || check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 3:
+            if (check_seq % 4 === 0 || check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 2:
+            if (check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 1:
+            if (check_seq % 10 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+    }
+}
+
 exports.GlobalObserver = GlobalObserver;