whatap 1.0.2 → 1.0.3-canary.0

package/lib/data/dataprofile-agent.js

@@ -1,162 +0,0 @@
-/**
- * Copyright 2016 the WHATAP project authors. All rights reserved.
- * Use of this source code is governed by a license that
- * can be found in the LICENSE file.
- */
-
-var DataPackSender  = require('./datapack-sender'),
-    StatTranX       = require('../stat/stat-tranx'),
-    StatRemoteIp    = require('../stat/stat-remote-ip'),
-    StatUserAgent   = require('../stat/stat-useragent'),
-    StatTranxDomain   = require('../stat/stat-tx-domain'),
-    StatTranxMtCaller   = require('../stat/stat-tx-caller'),
-    StatTranxReferer   = require('../stat/stat-tx-referer'),
-    StatRemoteIpUrl   = require('../stat/stat-remote-ipurl'),
-    conf            = require('../conf/configure'),
-    HashUtil            = require('../util/hashutil'),
-    EventLevel          = require('../data/event-level'),
-    DataTextAgent       = require('../data/datatext-agent'),
-    Long = require('long'),
-    Logger          = require('../logger');
-
-var ZipProfile = require('./zipprofile');
-var profile_zip_enabled = conf.getProperty('profile_zip_enabled', false);
-var stat_ipurl_enabled = conf.getProperty('stat_ipurl_enabled', false);
-conf.on('profile_zip_enabled', function(newProperty) {
-    profile_zip_enabled = newProperty;
-});
-conf.on('stat_ipurl_enabled', function(newProperty) {
-    stat_ipurl_enabled = newProperty;
-});
-
-var DataProfileAgent = {
-    last_reject : 0,
-    sendProfile : function (ctx, profile, rejected) {
-        try {
-            StatRemoteIp.incRemoteIp(ctx.remoteIp);
-            StatUserAgent.incUserAgent(ctx.userAgent);
-
-            var transaction = profile.service;
-            if(stat_ipurl_enabled && ctx.remoteIp){
-                var tc = StatRemoteIpUrl.getService(ctx.remoteIp, ctx.service_hash);
-                if (tc) {
-                    tc.count++;
-                    if (tc.errorLevel >= EventLevel.WARNING) {
-                        tc.error++;
-                    }
-                    tc.time += transaction.elapsed;
-                }
-            }
-
-            if(conf.stat_domain_enabled && ctx.http_host_hash!=0) {
-                StatTranxDomain.add(ctx.http_host_hash, ctx.service_hash, transaction.elapsed, ctx.error != 0);
-            }
-
-            if(conf.trace_referer_enabled && ctx.referer!=0) {
-                StatTranxReferer.add(ctx.referer, ctx.service_hash, transaction.elapsed, ctx.error != 0);
-            }
-
-            if(conf.stat_mtrace_enabled && ctx.mcaller_pcode!=0) {
-                var key = StatTranxMtCaller.getService(ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_spec, ctx.mcaller_url,ctx.service_hash);
-                transaction.mthis_spec = conf.mtrace_spec_hash
-
-                if (ctx.mcaller_spec && ctx.mcaller_spec!='') {
-                    transaction.mcaller_spec = key.mcaller_spec = HashUtil.hashFromString(ctx.mcaller_spec);
-                    DataTextAgent.MTRACE_SPEC.add(transaction.mcaller_spec, ctx.mcaller_spec);
-                }
-                if (ctx.mcaller_url  && ctx.mcaller_url!='') {
-                    transaction.mcaller_url = key.mcaller_url = HashUtil.hashFromString(ctx.mcaller_url);
-                    DataTextAgent.MTRACE_CALLER_URL.add(transaction.mcaller_url, ctx.mcaller_url);
-                }
-
-                // additional multi transaction
-                //key.url = ctx.service_hash;
-                var tc = StatTranxMtCaller.getService(key);
-                if (tc != null) {
-                    tc.count++;
-                    if (transaction.errorLevel >= EventLevel.WARNING) {
-                        tc.error++;
-                    }
-                    tc.time += transaction.elapsed;
-                }
-
-                // StatTranxMtCaller.add(ctx.mcaller_pcode,
-                //      ctx.mcaller_okind,
-                //      transaction.mcaller_spec,
-                //      transaction.mcaller_url,
-                //      ctx.service_hash,
-                //      transaction.elapsed,
-                //      ctx.error != 0);
-            }
-
-            var stat = StatTranX.getService(transaction.service);
-
-            if(stat !== null) {
-                stat.count++;
-                if(transaction.error.isZero() ==false) {
-                    stat.error++;
-                }
-                if(transaction.elapsed < 0) {
-                    transaction.elapsed = 0;
-                }
-
-                stat.actived += ctx.profileActive;
-                stat.time_sum += transaction.elapsed;
-                if (transaction.elapsed > stat.time_max) {
-                    stat.time_max = transaction.elapsed;
-                }
-
-                switch (transaction.apdex) {
-                    case 2:
-                        stat.apdex_satisfied++;
-                        break;
-                    case 1:
-                        stat.apdex_tolerated++;
-                        break;
-                }
-                if (stat.time_min==0 || transaction.elapsed < stat.time_min) {
-                    stat.time_min = transaction.elapsed;
-                }
-                stat.time_std += (Long.fromInt(transaction.elapsed) * Long.fromInt(transaction.elapsed));
-
-                stat.sql_count += transaction.sqlCount;
-                stat.sql_time +=transaction.sqlTime;
-                stat.sql_fetch += ctx.rs_count;
-                stat.sql_fetch_time +=ctx.rs_time;
-
-                stat.httpc_count += transaction.httpcCount;
-                stat.httpc_time +=transaction.httpcTime;
-
-                stat.malloc_sum +=transaction.malloc;
-                stat.cpu_sum +=transaction.cpuTime;
-
-                if (rejected) {
-                    var now = Date.now();
-                    if (now < this.last_reject + 1000) {
-                        return;
-                    }
-                    this.last_reject = now;
-                } else if (stat.profiled == true    // 이전(5분구간 내)에 프로파일이 수집된점이 있음
-                    && ctx.profileActive == 0       // 액티브 스택을 추적한적이 없음
-                    && transaction.elapsed < conf.profile_basetime
-                    && transaction.error.isZero()) {
-                    return;
-                }
-                stat.profiled = true;
-            }
-
-            var steps = ctx.profile.getSteps();
-            profile.setProfile(steps);
-            if(profile_zip_enabled){
-                ZipProfile.getInstance().add(profile);
-            } else {
-                DataPackSender.sendProfilePack(profile);
-            }
-
-        } catch(e) {
-            Logger.printError('WHATAP-199', 'dataprofile-agent error', e);
-        }
-    }
-};
-
-module.exports = DataProfileAgent;

package/lib/data/datatext-agent.js

@@ -1,135 +0,0 @@
-/**
- * Copyright 2016 the WHATAP project authors. All rights reserved.
- * Use of this source code is governed by a license that
- * can be found in the LICENSE file.
- */
-var RequestQueue    = require('../util/request-queue'),
-    TextPack        = require('../pack/text-pack').TextPack,
-    TextRec         = require('../pack/text-pack').TextRec,
-    TextTypes       = require('../lang/text-types'),
-    DataPackSender  = require('../data/datapack-sender'),
-    DateUtil        = require('../util/dateutil'),
-    IntSet        = require('../util/intset');
-
-const BUFFERD_MAX   = 10 * 1024;
-const INTERVAL      = 500;   // 제어 가능해야 한다.
-
-const DBC               = new HashedData(TextTypes.DB_URL, 1000);
-const METHOD            = new HashedData(TextTypes.METHOD, 5000);
-const HTTPC_URL         = new HashedData(TextTypes.HTTPC_URL, 2000);
-const HTTPC_HOST        = new HashedData(TextTypes.HTTPC_HOST, 2000);
-const ERROR             = new HashedData(TextTypes.ERROR, 1000);
-const SERVICE           = new HashedData(TextTypes.SERVICE, 2000);
-const SQL               = new HashedData(TextTypes.SQL, 2000);
-const STACK             = new HashedData(TextTypes.STACK_ELEMENTS, 20000);
-const MESSAGE           = new HashedData(TextTypes.MESSAGE, 2000);
-const USERAGENT         = new HashedData(TextTypes.USER_AGENT, 1000);
-const USER_AGENT_OS     = new HashedData(TextTypes.USER_AGENT_OS, 1000);
-const USER_AGENT_BROWSER= new HashedData(TextTypes.USER_AGENT_BROWSER, 1000);
-const REFERER           = new HashedData(TextTypes.REFERER, 1000);
-const HTTP_DOMAIN       = new HashedData(TextTypes.HTTP_DOMAIN, 500);
-const MTRACE_SPEC       = new HashedData(TextTypes.MTRACE_SPEC, 500);
-const MTRACE_CALLER_URL       = new HashedData(TextTypes.MTRACE_CALLER_URL, 500);
-const OKIND       = new HashedData(TextTypes.OKIND_NAME, 100);
-const ONODE       = new HashedData(TextTypes.ONODE_NAME, 100);
-
-var DataTextAgent = function () {
-
-    if(typeof DataTextAgent.instance == 'object') {
-        return DataTextAgent.instance;
-    }
-
-    this.date = DateUtil.yyyymmddhh();
-    this.queue = new RequestQueue(500);
-    this.bufferPack = new TextPack();
-    this.bufferedLength = 0;
-    this.run();
-
-    DataTextAgent.instance = this;
-};
-
-DataTextAgent.prototype.run = function () {
-    var r = this.queue.get();
-    if(r !== null) {
-        this.bufferPack.records.push(r);
-        this.bufferedLength += r.text.length;
-        if(this.bufferedLength >= BUFFERD_MAX) {
-            this.send();
-        }
-    } else {
-        if(this.bufferedLength > 0) {
-            this.send();
-        }
-    }
-
-    if(this.date !== DateUtil.yyyymmddhh()) {
-        this.reset();
-        this.date = DateUtil.yyyymmddhh();
-    }
-
-    var self = this;
-    setTimeout(function () {
-        self.run();
-    }, INTERVAL, 'whatap');
-};
-DataTextAgent.prototype.add = function (type, hash, text) {
-    this.queue.put(new TextRec(type, hash, text));
-};
-DataTextAgent.prototype.send = function () {
-    DataPackSender.sendTextPack(this.bufferPack);
-    this.bufferPack = new TextPack();
-    this.bufferedLength = 0;
-};
-DataTextAgent.prototype.reset = function () {
-    DBC.sent.clear();
-    METHOD.sent.clear();
-    HTTPC_URL.sent.clear();
-    HTTPC_HOST.sent.clear();
-    ERROR.sent.clear();
-    SERVICE.sent.clear();
-    SQL.sent.clear();
-    STACK.sent.clear();
-    MESSAGE.sent.clear();
-    USERAGENT.sent.clear();
-    USER_AGENT_OS.sent.clear();
-    USER_AGENT_BROWSER.sent.clear();
-    REFERER.sent.clear();
-    HTTP_DOMAIN.sent.clear();
-    MTRACE_SPEC.sent.clear();
-    MTRACE_CALLER_URL.sent.clear();
-    OKIND.sent.clear();
-    ONODE.sent.clear();
-};
-
-function HashedData(type, n) {
-    this.type = type || 0;
-    this.sent = new IntSet().setMax(n);
-}
-HashedData.prototype.add = function (hash, text) {
-    if(require('../trace/trace-context-manager').initialized === false) { return; }
-    // 보내지 않은 Hash인 경우 추가
-    if(this.sent.hasKey(hash) == false) {
-        this.sent.add(hash);
-        new DataTextAgent().add(this.type, hash, text);
-    }
-};
-
-exports.agent = new DataTextAgent();
-exports.DBC = DBC;
-exports.METHOD = METHOD;
-exports.HTTPC_URL = HTTPC_URL;
-exports.HTTPC_HOST = HTTPC_HOST;
-exports.ERROR = ERROR;
-exports.SERVICE = SERVICE;
-exports.SQL = SQL;
-exports.STACK = STACK;
-exports.MESSAGE = MESSAGE;
-exports.USERAGENT = USERAGENT;
-exports.USER_AGENT_OS = USER_AGENT_OS;
-exports.USER_AGENT_BROWSER = USER_AGENT_BROWSER;
-exports.REFERER = REFERER;
-exports.HTTP_DOMAIN = HTTP_DOMAIN;
-exports.MTRACE_SPEC = MTRACE_SPEC;
-exports.MTRACE_CALLER_URL = MTRACE_CALLER_URL;
-exports.OKIND = OKIND;
-exports.ONODE = ONODE;

package/lib/data/event-level.js

@@ -1,15 +0,0 @@
-/**
- * Copyright 2016 the WHATAP project authors. All rights reserved.
- * Use of this source code is governed by a license that
- * can be found in the LICENSE file.
- */
-
-var EventLevel = {
-    FATAL: Buffer.alloc(30),
-    WARNING: Buffer.alloc(20),
-    INFO: Buffer.alloc(10),
-    NONE: Buffer.alloc(0)
-}
-
-module.exports = EventLevel;
-

package/lib/data/test.js

@@ -1,49 +0,0 @@
-var IntSet  = require('../util/intset') ,
-TagCountPack = require('../pack/tagcount-pack');
-const DataPackSender = require('./datapack-sender');
-
-var Test = function() {
-    this.sent = new IntSet().setMax(10);
-    this.init()
-}
-
-Test.prototype.init = function() {
-    this.sent.add(1);
-    this.sent.add(2);
-    this.sent.add(3);
-    this.sent.add(4);
-    this.sent.add(5);
-    this.sent.add(6);
-    this.sent.add(7);
-    this.sent.add(8);
-    this.sent.add(9);
-    this.sent.add(10);
-    this.sent.add(11);
-    this.sent.add(12);
-    this.sent.add(13);
-    this.sent.add(14);
-    if(this.sent.hasKey(14) == false) {
-        console.log(this.sent.getLastValue())
-        this.sent.add(15);
-    }
-    var p = new TagCountPack()
-    p.category = 'aaaa' 
-    p.pcode = 101
-    p.oid = 1
-    p.putTagString('tag1','tag1')
-    p.putTagString('tag2','tag2')
-    p.putTagString('tag3','tag3')
-    p.putTagString('tag4','tag4')
-    p.putTagString('tag5','tag5')
-    p.putTagString('tag6','tag6')
-    p.putInt('filed1' , 100)
-    p.putInt('filed2' , 200)
-    p.putInt('filed3' , 300)
-    p.putInt('filed4' , 400)
-    p.putInt('filed5' , 500)
-    p.putInt('filed6' , 600)
-    
-    DataPackSender.sendTagCounterPack(p);
-}
-
-exports.Test = new Test();

package/lib/data/zipprofile.js

@@ -1,197 +0,0 @@
-/**
- * Copyright 2016 the WHATAP project authors. All rights reserved.
- * Use of this source code is governed by a license that
- * can be found in the LICENSE file.
- */
-
-var conf            = require('../conf/configure'),
-    secuMaster      = require('../net/security-master'),
-    TcpRequestMgr   = require('../net/tcprequest-mgr'),
-    DateUtil        = require('./../util/dateutil'),
-    RequestQueue    = require('../util/request-queue'),
-    DataPackSender = require('../data/datapack-sender'),
-    DataOutputX = require('../io/data-outputx'),
-    ZipPack = require('../pack/zip-pack'),
-    DataPackSender = require('../data/datapack-sender'),
-    Logger          = require('../logger');
-
-const zlib = require('zlib');
-
-var profile_zip_enabled = conf.getProperty('profile_zip_enabled', false);
-var profile_zip_max_buffer_size = conf.getProperty('profile_zip_max_buffer_size', 1024*1024);
-var profile_zip_min_size = conf.getProperty('profile_zip_min_size', 100)
-var profile_zip_queue_size = conf.getProperty('profile_zip_queue_size', 500);
-var profile_zip_max_wait_time = conf.getProperty('profile_zip_max_wait_time', 1000);
-var net_send_max_bytes = conf.getProperty('net_send_max_bytes', 5 * 1024 * 1024);
-var debug_profile_zip_enabled = conf.getProperty('debug_profile_zip_enabled', false);
-var debug_profile_zip_interval = conf.getProperty('debug_profile_zip_interval', 5000);
-conf.on('profile_zip_enabled', function(newProperty) {
-    profile_zip_enabled = newProperty;
-    if (profile_zip_enabled) {
-        ZipProfile.getInstance().startProcessQueue();
-    } else {
-        ZipProfile.getInstance().stopProcessQueue();
-        ZipProfile.resetInstance();
-    }
-});
-
-class ZipProfile {
-    constructor() {
-        if (ZipProfile.instance) {
-            return ZipProfile.instance;
-        }
-        this.queue = new RequestQueue(profile_zip_queue_size);
-        this.no_zip_sent = 0;
-        this.zip_sent = 0;
-        this.last_log = null;
-        this.buffer = Buffer.alloc(0);
-        this.packCount = 0;
-        this.first_time = null;
-        this.isProcessing = false;
-        ZipProfile.instance = this;
-
-        if (profile_zip_enabled) {
-            this.startProcessQueue();
-        }
-    }
-
-    static getInstance() {
-        if (!ZipProfile.instance) {
-            ZipProfile.instance = new ZipProfile();
-        }
-        return ZipProfile.instance;
-    }
-
-    static resetInstance() {
-        if(ZipProfile.instance){
-            ZipProfile.instance = null;
-        }
-    }
-
-    add(p){
-        var ok = this.queue.put(p);
-        if(!ok){
-            DataPackSender.sendPack(p);
-            this.no_zip_sent++;
-        }
-    }
-
-    async processQueue() {
-        this.isProcessing = true;
-        while (this.isProcessing) {
-            if(!profile_zip_enabled) { break; }
-
-            const p = await this.queue.getByTimeout(profile_zip_max_wait_time);
-            if(p) {
-                await this.append(p);
-            }else{
-                await this.sendAndClear();
-            }
-            await new Promise(resolve => setTimeout(resolve, 100));
-        }
-    }
-
-    async append(p) {
-        var b = DataOutputX.toBytesPack(p);
-        if(b.length >= net_send_max_bytes){ return; }
-
-        this.buffer = Buffer.concat([this.buffer, b]);
-        this.packCount++;
-
-        if(!this.first_time){
-            this.first_time = p.time;
-            if(this.buffer.length >= profile_zip_max_buffer_size){
-                await this.sendAndClear();
-            }
-        }else{
-            if(this.buffer.length >= profile_zip_max_buffer_size || (p.time - this.first_time) >= conf.getProperty('profile_zip_max_wait_time', 5000)){
-                await this.sendAndClear();
-            }
-        }
-    }
-
-    async sendAndClear() {
-        if(this.buffer.length === 0){ return; }
-
-        var p = new ZipPack();
-        p.time = DateUtil.currentTime();
-        p.recordCount = this.packCount;
-        p.records = this.buffer;
-
-        await this.doZip(p);
-        if(debug_profile_zip_enabled){
-            if(debug_profile_zip_interval){
-                Logger.print('WHATAP-ZIP-DEBUG', `PROFILE status=${p.status} records=${p.recordCount} | ${this.buffer.length} => ${p.records.length} queue=${this.queue.size()}`, false)
-            } else {
-                this.zip_sent++;
-                var now = Date.now();
-                if(now > (this.last_log + debug_profile_zip_interval)){
-                    this.last_log = now;
-                    this.log(p);
-                    this.zip_sent = 0;
-                }
-            }
-        }
-
-        p.pcode = secuMaster.PCODE;
-        p.oid = secuMaster.OID;
-        p.okind = conf.OKIND;
-        p.onode = conf.ONODE;
-
-        TcpRequestMgr.addProfile(0, p);
-
-        this.buffer = Buffer.alloc(0);
-        this.first_time = 0;
-        this.packCount = 0;
-    }
-
-    log(p){
-        try{
-            var sb = `PROFILE `;
-            sb += `zip_sent=${this.zip_sent}`;
-            sb += ` records=${p.records}`;
-            sb += ` | =${this.buffer.length} => ${p.records.length}`;
-            sb += ` queue=${this.queue.size()}`;
-            if (this.no_zip_sent > 0) {
-                sb += ` no_zip_sent=${this.no_zip_sent}`;
-            }
-            Logger.print('WHATAP-ZIP-DEBUG', sb, false);
-        }catch (e) {
-        }
-    }
-
-    async doZip(p){
-        if(p.status !== 0){ return; }
-        if(p.records.length < profile_zip_min_size){ return; }
-        p.status = 1;
-
-        try{
-            p.records = await this._doZip(p.records);
-        }catch (e) {
-            Logger.print('WHATAP-ZIP-ERROR', 'Error occurred during compression.', false);
-        }
-    }
-
-    _doZip(data) {
-        return new Promise((resolve, reject) => {
-            zlib.gzip(data, (err, buffer) => {
-                if (err) {
-                    reject(err);
-                } else {
-                    resolve(buffer);
-                }
-            });
-        });
-    }
-
-    startProcessQueue() {
-        if (!this.isProcessing) {
-            this.processQueue()
-        }
-    }
-    stopProcessQueue() {
-        this.isProcessing = false;
-    }
-}
-
-module.exports = ZipProfile;

package/lib/env/constants.js

@@ -1,21 +0,0 @@
-/**
- * Copyright 2016 the WHATAP project authors. All rights reserved.
- * Use of this source code is governed by a license that
- * can be found in the LICENSE file.
- */
-
-var numType = {
-	INT8_MIN : -128,
-	INT8_MAX : 127,
-	INT16_MIN : -32768,
-	INT16_MAX : 32767,
-	INT24_MIN : -8388608,
-	INT24_MAX : 8388607,
-	INT32_MIN : -2147483648,
-	INT32_MAX : 2147483647,
-	INT40_MIN : -549755813888,
-	INT40_MAX : 549755813887,
-	INT64_MIN : -4503599627370496,
-	INT64_MAX : 4503599627370495
-};
-module.exports.numType = numType;