whatap 0.4.98 → 0.5.0

package/lib/logsink/sender-util.js

@@ -0,0 +1,51 @@
+var conf = require('../conf/configure');
+var SecurityMaster      = require('../net/security-master');
+var LogSinkPack = require('../pack/log-sink-pack');
+var DataPackSender = require('../data/datapack-sender');
+var DateUtil = require('../util/dateutil')
+var ZipSend = require('./zip/zip-send');
+
+function SenderUtil() {
+    this.conf = conf;
+    this.secu = SecurityMaster;
+}
+
+SenderUtil.prototype.send = function (log) {
+    const p = new LogSinkPack();
+    p.time = log.time;
+    if (!p.time) {
+        p.time = DateUtil.currentTime();
+    }
+    p.category = log.category;
+
+    if (this.secu.ONAME) {
+        p.tags.putString('oname', this.secu.ONAME);
+    }
+    if (this.conf.OKIND) {
+        p.tags.putString('okindName', this.conf.OKIND_NAME);
+    }
+    if (this.conf.ONODE) {
+        p.tags.putString('onodeName', this.conf.ONODE_NAME);
+    }
+
+    if (log.tags != null) {
+        p.tags.putAllMapValue(log.tags);
+    }
+    if (log.fields != null) {
+        if (p.fields == null) {
+            p.fields = log.fields;
+        } else {
+            p.fields.putAllMapValue(log.fields);
+        }
+    }
+
+    p.content = log.content;
+
+    if(conf.getProperty('logsink_zip_enabled', false)){
+        ZipSend.getInstance().add(p);
+    }else{
+        DataPackSender.sendLogSinkPack(p)
+    }
+};
+
+module.exports = SenderUtil;

package/lib/logsink/zip/zip-send.js

@@ -0,0 +1,177 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var conf            = require('../../conf/configure'),
+    DateUtil        = require('../../util/dateutil'),
+    RequestQueue    = require('../../util/request-queue'),
+    DataPackSender = require('../../data/datapack-sender'),
+    DataOutputX = require('../../io/data-outputx'),
+    ZipPack = require('../../pack/zip-pack'),
+    Logger          = require('../../logger');
+var zlib = require('zlib');
+
+var logsink_enabled = conf.getProperty('logsink_enabled', false);
+var logsink_zip_enabled = conf.getProperty('logsink_zip_enabled', false);
+var logsink_max_buffer_size = conf.getProperty('logsink_max_buffer_size', 1024*64);
+var profile_zip_min_size = conf.getProperty('profile_zip_min_size', 100)
+var logsink_queue_size = conf.getProperty('logsink_queue_size', 500);
+var logsink_max_wait_time = conf.getProperty('logsink_max_wait_time', 2000);
+conf.on('logsink_enabled', function(newProperty) {
+    logsink_enabled = newProperty;
+    if (logsink_enabled && logsink_zip_enabled) {
+        ZipSend.getInstance().startProcessQueue();
+    } else {
+        ZipSend.getInstance().stopProcessQueue();
+        ZipSend.resetInstance();
+    }
+});
+conf.on('logsink_zip_enabled', function(newProperty) {
+    logsink_zip_enabled = newProperty;
+    if (logsink_enabled && logsink_zip_enabled) {
+        ZipSend.getInstance().startProcessQueue();
+    } else {
+        ZipSend.getInstance().stopProcessQueue();
+        ZipSend.resetInstance();
+    }
+});
+
+class ZipSend {
+    constructor() {
+        if (ZipSend.instance) {
+            return ZipSend.instance;
+        }
+        this.queue = new RequestQueue(logsink_queue_size);
+        this.buffer = Buffer.alloc(0);
+        this.packCount = 0;
+        this.first_time = null;
+        this.isProcessing = false;
+        ZipSend.instance = this;
+
+        if (logsink_enabled && logsink_zip_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!ZipSend.instance) {
+            ZipSend.instance = new ZipSend();
+        }
+        return ZipSend.instance;
+    }
+
+    static resetInstance() {
+        if(ZipSend.instance){
+            ZipSend.instance = null;
+        }
+    }
+
+    add(p){
+        var ok = this.queue.put(p);
+        if(!ok){
+            DataPackSender.sendLogSinkPack(p);
+        }
+    }
+
+    // async addWait(p, waitTimeForFull) {
+    //     var ok = this.queue.put(p);
+    //     if (ok === false){
+    //         if(waitTimeForFull > 0){
+    //             while(this.queue.put(p) === false){
+    //                 await new Promise(resolve => setTimeout(resolve, waitTimeForFull));
+    //             }
+    //         }
+    //     }
+    // }
+
+    async processQueue() {
+        this.isProcessing = true;
+        while (this.isProcessing) {
+            if(!logsink_enabled || !logsink_zip_enabled) {
+                this.isProcessing = false;
+                break;
+            }
+
+            const p = await this.queue.getByTimeout(logsink_max_wait_time);
+            if(p) {
+                await this.append(p);
+            }else{
+                await this.sendAndClear();
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    async append(p) {
+        var b = DataOutputX.toBytesPack(p);
+        this.packCount++;
+        this.buffer = Buffer.concat([this.buffer, b]);
+
+        if(!this.first_time){
+            this.first_time = p.time;
+            if (this.buffer.length >= logsink_max_buffer_size) {
+                await this.sendAndClear();
+            }
+        }else{
+            if(this.buffer.length >= logsink_max_buffer_size || (p.time - this.first_time) >= conf.getProperty('logsink_max_wait_time', 2000)){
+                await this.sendAndClear();
+            }
+        }
+    }
+
+    async sendAndClear() {
+        if(this.buffer.length === 0){ return; }
+
+        var p = new ZipPack();
+        p.time = DateUtil.currentTime();
+        p.recordCount = this.packCount;
+        p.records = this.buffer;
+
+        await this.doZip(p);
+        if(conf.getProperty('debug_logsink_zip_enabled', false)){
+            Logger.print('WHATAP-703', `zip status=${p.status} records=${p.recordCount} | ${this.buffer.length} => ${p.records.length}`);
+        }
+        DataPackSender.sendLogSinkPack(p);
+
+        this.buffer = Buffer.alloc(0);
+        this.first_time = 0;
+        this.packCount = 0;
+    }
+
+    async doZip(p){
+        if(p.status !== 0){ return; }
+        if(p.records.length < profile_zip_min_size){ return; }
+        p.status = 1;
+
+        try{
+            p.records = await this._doZip(p.records);
+        }catch (error) {
+            Logger.print('WHATAP-701', 'Error occurred during compression.' + error, false);
+        }
+    }
+
+    _doZip(data) {
+        return new Promise((resolve, reject) => {
+            zlib.gzip(data, (err, buffer) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(buffer);
+                }
+            });
+        });
+    }
+
+    startProcessQueue() {
+        if (!this.isProcessing) {
+            this.processQueue()
+        }
+    }
+    stopProcessQueue() {
+        this.isProcessing = false;
+    }
+}
+
+module.exports = ZipSend;

package/lib/net/paramdef.js

@@ -34,6 +34,7 @@ exports.AGENT_LOG_READ = 23;
 
 exports.GET_ACTIVE_TRANSACTION_LIST = 25;
 exports.GET_ACTIVE_TRANSACTION_DETAIL = 26;
+exports.GET_TOPOLOGY = 27
 	
 exports.NODE_MODULE_DEPENDENCY = 101;
 

package/lib/observers/global-observer.js

@@ -7,6 +7,15 @@
 var TraceContextManager = require('../trace/trace-context-manager'),
     Logger = require('../logger');
 
+var HttpStepX = require('../step/http-stepx');
+const Hexa32 = require("../util/hexa32");
+const SecurityMaster = require("../net/security-master");
+const conf = require("../conf/configure");
+const KeyGen = require("../util/keygen");
+const HashUtil = require("../util/hashutil");
+const DataTextAgent = require("../data/datatext-agent");
+const shimmer = require('../core/shimmer');
+
 var GlobalObserver = function (agent) {
     this.agent = agent;
     this.packages = ['global'];
@@ -26,6 +35,137 @@ GlobalObserver.prototype.inject = function (mod, moduleName) {
             });
         });
     });
+
+    shimmer.wrap(mod, 'fetch', function(original) {
+        return function(...args) {
+            var info = args[1] ? args[1] : {};
+            var ctx = TraceContextManager._asyncLocalStorage.getStore();
+
+            if (ctx) {
+                interTxTraceAutoOn(ctx);
+
+                if (conf.mtrace_enabled) {
+                    if (info.headers) {
+                        info.headers['x-wtap-po'] = transferPOID(ctx);
+                    } else {
+                        info.headers = {
+                            'x-wtap-po': transferPOID(ctx)
+                        };
+                    }
+                    if (conf.stat_mtrace_enabled) {
+                        info.headers[conf._trace_mtrace_spec_key1] = transferSPEC_URL(ctx);
+                    }
+                    if (conf.mtid_mtrace_enabled && ctx.mtid.isZero() === false) {
+                        info.headers[conf._trace_mtrace_caller_key] = transferMTID_CALLERTX(ctx);
+                    }
+                    args[1] = info;
+                }
+            }
+
+            const promise = original.apply(this, args);
+
+            promise.then(response => {
+                var ctx = TraceContextManager._asyncLocalStorage.getStore();
+                if (ctx && args[0]) {
+                    const url = new URL(args[0]);
+                    ctx.httpc_host = url.hostname;
+                    ctx.httpc_url = url.pathname;
+                    ctx.httpc_port = url.port || (url.protocol === 'https:' ? 443 : 80);
+
+                    var step = new HttpStepX();
+                    step.start_time = ctx.getElapsedTime();
+                    step.url = HashUtil.hashFromString(ctx.httpc_url);
+                    DataTextAgent.HTTPC_URL.add(step.url, ctx.httpc_url);
+                    step.host = HashUtil.hashFromString(ctx.httpc_host);
+                    DataTextAgent.HTTPC_HOST.add(step.host, ctx.httpc_host);
+                    step.port = ctx.httpc_port;
+
+                    ctx.profile.push(step);
+                }
+            }).catch(e => {
+                Logger.printError("WHATAP-704", "Promise error occurred during fetch: " + e, false);
+            });
+            return promise;
+        };
+    });
 };
 
+var transfer_poid;
+function transferPOID(ctx) {
+    if (transfer_poid)
+        return transfer_poid;
+    transfer_poid = Hexa32.toString32(SecurityMaster.PCODE) + ','
+        + Hexa32.toString32(SecurityMaster.OKIND) + ',' + Hexa32.toString32(SecurityMaster.OID);
+    return transfer_poid;
+}
+
+function transferMTID_CALLERTX(ctx) {
+    if (ctx.transfer_id)
+        return ctx.transfer_id;
+    var x = Hexa32.toString32(ctx.mtid) + ',' + (ctx.mdepth + 1) + ',' + Hexa32.toString32(ctx.txid);
+    ctx.transfer_id = x;
+    return ctx.transfer_id;
+}
+
+function transferSPEC_URL(ctx) {
+    if (ctx.transfer_info)
+        return ctx.transfer_info;
+    var x = conf.mtrace_spec + ',' + ctx.service_hash;
+    ctx.transfer_info = x;
+    return ctx.transfer_info;
+}
+
+var check_seq = 1;
+function interTxTraceAutoOn(ctx) {
+    if (conf.mtrace_enabled == false || ctx.httpc_checked || ctx.mtid.isZero() === false)
+        return;
+    ctx.httpc_checked = true;
+    if (conf.mtrace_rate >= 100) {
+        ctx.mtid = KeyGen.next();
+        return;
+    }
+    check_seq++;
+    switch (Math.floor(conf.mtrace_rate / 10)) {
+        case 10:
+            ctx.mtid = KeyGen.next();
+            break;
+        case 9:
+            if (check_seq % 10 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 8:
+            if (check_seq % 5 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 7:
+            if (check_seq % 4 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 6:
+            if (check_seq % 3 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 5:
+            if (check_seq % 2 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 4:
+            if (check_seq % 3 === 0 || check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 3:
+            if (check_seq % 4 === 0 || check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 2:
+            if (check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 1:
+            if (check_seq % 10 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+    }
+}
+
 exports.GlobalObserver = GlobalObserver;