whatap 1.0.1 → 1.0.2

package/lib/stat/stat-sql.js

@@ -0,0 +1,113 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+const TABLE_MAX_SIZE = 5000;
+
+var LinkedMap    = require('../util/linkedmap'),
+    SqlRec              = require('../pack/statsql-rec'),
+    StatSqlPack         = require('../pack/statsql-pack'),
+    DataPackSender      = require('../data/datapack-sender');
+
+function StatSql() {
+    if(typeof StatSql.instance === 'object') {
+        return StatSql.instance;
+    }
+
+    this.table = new LinkedMap(TABLE_MAX_SIZE + 1, 1).setMax(TABLE_MAX_SIZE);
+    this.table.hash = function (k) {
+        if(k._hash_){
+            return k._hash_;
+        }
+        var result = 1;
+        result = 31 * result + k.dbc;
+        result = 31 * result + k.sql;
+        
+        k._hash_=result;
+        return result;
+    }
+    this.table.equals = function (k1,k2) {
+        return k1.dbc === k2.dbc && k1.sql === k2.sql;
+    }
+    this.table.create = function (key) {
+        if(this.isFull()) {
+            return null;
+        }
+        var rec = new SqlRec();
+        rec.dbc = key.dbc;
+        rec.sql = key.sql;
+        rec.time_min = Number.MAX_SAFE_INTEGER;
+        return rec;
+    };
+
+    StatSql.instance = this;
+}
+StatSql.prototype.getSql = function (dbc, sql) {
+    var key = {dbc: dbc , 
+               sql: sql
+              };
+    return this.table.intern(key);
+};
+StatSql.prototype.addFetch = function (dbc, sql, fetch, fetchTime) {
+    if(sql === 0) {
+        return;
+    }
+    var r = this.getSql(dbc, sql);
+    if(r != null) {
+        r.fetch_count += fetch;
+        r.fetch_time += fetchTime;
+    }
+};
+// StatSql.prototype.addUpdate = function (dbc, sql, updated) {
+//     if(sql === 0) {
+//         return;
+//     }
+//     var r = this.getSql(dbc, sql);
+//     if(r != null) {
+//         r.update_count = r.update_count.add(updated);
+//     }
+// };
+
+StatSql.prototype.addSqlTime = function (urlHash, dbc,  sql, time, isErr, updated) {
+    
+    urlHash = urlHash || 0;
+    sql = sql || 0;
+    time = time || 0;
+    updated = updated || 0;
+    
+    if(sql === 0) {
+        return;
+    }
+    var sqlRec = this.getSql(dbc, sql);
+    if(sqlRec != null) {
+        sqlRec.count_total++;
+        sqlRec.time_sum +=time;
+        sqlRec.time_std += time*time;
+        sqlRec.time_min = Math.min(sqlRec.time_min, time);
+        sqlRec.time_max = Math.max(sqlRec.time_max, time);
+        if(isErr) {
+            sqlRec.count_error++;
+        }
+        sqlRec.update_count +=updated;
+        sqlRec.service = urlHash;
+    }
+};
+StatSql.prototype.send = function (now) {
+    if(this.table.size() === 0) {
+        return;
+    }
+
+    var p = new StatSqlPack().setRecords(this.table.size(), this.table.values());
+    p.time = now;
+    this.table.clear();
+    DataPackSender.sendStatSqlPack(p);
+};
+StatSql.prototype.clear = function () {
+    this.table.clear();
+};
+
+
+//싱글톤으로 동작해야한다. 
+module.exports = new StatSql();

package/lib/stat/stat-tranx.js

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var IntKeyLinkedMap     = require('../util/intkey-linkedmap'),
+    TransactionRec      = require('../pack/statservice-rec'),
+    StatServicePack     = require('../pack/statservice-pack'),
+    StatServicePack1     = require('../pack/statservice-pack1'),
+    conf        = require('./../conf/configure'),
+    Logger          = require('../logger'),
+    DataPackSender      = require('../data/datapack-sender')
+    
+const TABLE_MAX_SIZE = 5000;
+
+function StatTranx() {
+    if(typeof StatTranx.instance === 'object') {
+        return StatTranx.instance;
+    }
+
+    this.table = new IntKeyLinkedMap(TABLE_MAX_SIZE + 1, 1).setMax(TABLE_MAX_SIZE);
+    this.table.create = function (hash) {
+        if(this.isFull()) {
+            return null;
+        }
+        var rec = new TransactionRec();
+        rec.setUrlHash(hash);
+        rec.time_min = Number.MAX_VALUE;
+        return rec;
+    };
+
+    StatTranx.instance = this;
+}
+
+StatTranx.prototype.getService = function (hash) {
+    return this.table.intern(hash);
+};
+StatTranx.prototype.send = function (now) {
+    if(this.table.size() === 0) {
+        return;
+    }
+    var p = undefined;
+    if(conf.stat_service1_enabled){
+        p = new StatServicePack1().setRecords(this.table.size(), this.table.values());
+        p.spec = conf.mtrace_spec_hash;
+    }else{
+        p = new StatServicePack().setRecords(this.table.size(), this.table.values());
+    }
+    p.time = now;
+    this.table.clear();
+    DataPackSender.sendStatServicePack(p);
+};
+StatTranx.prototype.clear = function () {
+    this.table.clear();
+};
+
+module.exports = new StatTranx();

package/lib/stat/stat-tx-caller.js

@@ -0,0 +1,160 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+ var LinkedMap       = require('../util/linkedmap'),
+ AnyList       = require('../util/anylist'),
+ StatGeneralPack   = require('../pack/stat-general-pack'),
+ DataPackSender  = require('../data/datapack-sender'),
+ Logger          = require('../logger'),
+ HashUtil        = require('../util/hashutil'),
+ StringUtil      = require('../util/string-util'),
+ DataTextAgent       = require('../data/datatext-agent'),
+ conf        = require('../conf/configure'),
+ TimeCount      = require('../pack/time-count');
+
+const TABLE_MAX_SIZE = 5000;
+
+function PK(caller_pcode, caller_okind, caller_spec, caller_url, url) {
+ this.caller_pcode = caller_pcode;
+ this.caller_okind = caller_okind;
+ this.caller_spec = caller_spec ;
+ this.caller_url = caller_url;
+ this.url = url ;
+}
+
+PK.prototype.hashCode = function () {
+ if(this._hash_){
+     return this._hash_;
+ }
+ var prime = 31;
+ var result = 1;
+ result = prime * result + this.caller_pcode;
+ result = prime * result + this.caller_okind;
+ result = prime * result + this.caller_spec;
+ result = prime * result + this.caller_url;
+ result = prime * result + this.url;
+ this._hash_=result;
+ return result;
+};
+PK.prototype.equals = function (other) {
+ if(this === other) {
+     return  true;
+ }
+ if(other == null) {
+     return false;
+ }
+ if(other instanceof PK === false) {
+     return false;
+ }
+ return this.caller_pcode === other.caller_pcode 
+ && this.caller_okind === other.caller_okind
+ && this.caller_spec === other.caller_spec
+ && this.caller_url === other.caller_url
+ && this.url === other.url;
+};
+
+
+function StatTranxMtCaller() {
+ if(typeof StatTranxMtCaller.instance === 'object') {
+     return StatTranxMtCaller.instance;
+ }
+
+ this.table = new LinkedMap(TABLE_MAX_SIZE + 1 , 1).setMax(TABLE_MAX_SIZE);
+ this.table.create = function (key) {
+     if(this.size() >= TABLE_MAX_SIZE) {
+         return null;
+     }
+     return new TimeCount();
+ };
+
+ conf.on('stat_mtrace_max_count', function(props){
+     StatTranxMtCaller.instance.table.setMax(props);
+ })
+
+ StatTranxMtCaller.instance = this;
+}
+
+StatTranxMtCaller.prototype.getService = function (caller_pcode, caller_okind, caller_spec, caller_url,url) {
+ return this.table.intern(new PK(caller_pcode,caller_okind,caller_spec, caller_url, url));
+};
+StatTranxMtCaller.prototype.add = function (caller_pcode, caller_spec, caller_url, url, 
+ time, isErr) {
+ try {
+     var rec = this.getService(caller_pcode, caller_spec, caller_url,url);
+     if(rec) {
+         rec.count++;
+         if(isErr){
+             rec.error++;
+         }
+         rec.toime +=time;
+     }
+ } catch(e) {
+     Logger.printError("WHATAP-610", "StatTranxMtCaller add error", e, false);
+ }
+};
+StatTranxMtCaller.prototype.send = function (now) {
+ if(this.table.size() === 0) {
+     return;
+ }
+ var mtrace_spec_hash = 0;
+ // if(conf.mtrace_spec) {
+ //     mtrace_spec_hash = HashUtil.hash(conf.mtrace_spec);
+ //     DataTextAgent.MTRACE_SPEC.add(mtrace_spec_hash,conf.mtrace_spec);
+ // }
+ if( StringUtil.isNotEmpty(conf.mtrace_spec) ){
+     DataTextAgent.MTRACE_SPEC.add(conf.mtrace_spec_hash, conf.mtrace_spec);
+ }        
+         
+ var rsize=this.table.size();
+ var caller_pcode =new AnyList(AnyList.LONG, rsize); /*pcode=LONG*/
+ var caller_okind =new AnyList(AnyList.INT, rsize);
+ var caller_spec =new  AnyList(AnyList.INT, rsize);
+ var caller_url =new  AnyList(AnyList.INT, rsize);
+ var spec =new  AnyList(AnyList.INT, rsize);
+ var url =new  AnyList(AnyList.INT, rsize);
+ var count = new  AnyList(AnyList.INT, rsize);
+ var error = new  AnyList(AnyList.INT, rsize);
+ var time = new  AnyList(AnyList.LONG, rsize);
+
+var en = this.table.entries();
+ for (var i = 0; i<rsize && en.hasMoreElements(); i++) {
+     var ent = en.nextElement();
+     var k = ent.getKey();
+     var v = ent.getValue();
+
+     caller_pcode.put(i, k.caller_pcode);
+     caller_okind.put(i, k.caller_okind);
+     caller_spec.put(i, k.caller_spec);
+     caller_url.put(i, k.caller_url);
+     spec.put(i, mtrace_spec_hash);
+     url.put(i, k.url);
+     count.put(i, v.count);
+     error.put(i, v.error);
+     time.put(i, v.time);
+ }
+ this.table.clear();        
+
+ var out = new StatGeneralPack();
+ out.put("caller_pcode", caller_pcode);
+ out.put("caller_okind", caller_okind);
+ out.put("caller_spec", caller_spec);
+ out.put("caller_url", caller_url);
+ out.put("spec", spec);
+ out.put("url", url);
+ out.put("count", count);
+ out.put("error", error);
+ out.put("time", time);
+
+ out.id = "mt";
+ out.time = now;
+         
+ DataPackSender.sendPack(out);
+};
+StatTranxMtCaller.prototype.clear = function () {
+ this.table.clear();
+};
+
+module.exports = new StatTranxMtCaller();

package/lib/stat/stat-tx-domain.js

@@ -0,0 +1,111 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var LinkedMap       = require('../util/linkedmap'),
+    AnyList       = require('../util/anylist'),
+    StatGeneralPack   = require('../pack/stat-general-pack'),
+    DataPackSender  = require('../data/datapack-sender'),
+    Logger          = require('../logger'),
+    TimeCount      = require('../pack/time-count');
+
+const TABLE_MAX_SIZE = 5000;
+
+function PK(domain, url) {
+    this.domain = domain || 0;
+    this.url = url || 0;
+}
+
+PK.prototype.hashCode = function () {
+    var prime = 31;
+    var result = 1;
+    result = prime * result + this.domain;
+    result = prime * result + this.url;
+    return result;
+};
+PK.prototype.equals = function (other) {
+    if(this === other) {
+        return  true;
+    }
+    if(other == null) {
+        return false;
+    }
+    return this.domain === other.domain && this.url === other.url;
+};
+
+
+function StatTranxDomain() {
+    if(typeof StatTranxDomain.instance === 'object') {
+        return StatTranxDomain.instance;
+    }
+
+    this.table = new LinkedMap(TABLE_MAX_SIZE + 1 , 1).setMax(TABLE_MAX_SIZE);
+    this.table.create = function (key) {
+        if(this.size() >= TABLE_MAX_SIZE) {
+            return null;
+        }
+        return new TimeCount();
+    };
+    StatTranxDomain.instance = this;
+}
+
+StatTranxDomain.prototype.getService = function (domain, url) {
+    return this.table.intern(new PK(domain,url));
+};
+StatTranxDomain.prototype.add = function (domain, url, time, isErr) {
+    try {
+        var rec = this.getService(domain, url);
+        if(rec) {
+            rec.count++;
+            if(isErr){
+                rec.error++;
+            }
+            rec.toime +=time;
+        }
+    } catch(e) {
+        Logger.printError("WHATAP-610", "StatTranxDomain add error", e, false);
+    }
+};
+StatTranxDomain.prototype.send = function (now) {
+    if(this.table.size() === 0) {
+        return;
+    }
+    var rsize=this.table.size();
+    var domain =new AnyList(AnyList.INT, rsize);
+    var url =new  AnyList(AnyList.INT, rsize);
+    var count = new  AnyList(AnyList.INT, rsize);
+    var error = new  AnyList(AnyList.INT, rsize);
+    var time = new  AnyList(AnyList.LONG, rsize);
+
+   var en = this.table.entries();
+    for (var i = 0; i<rsize &&  en.hasMoreElements(); i++) {
+        var ent = en.nextElement();
+        var k = ent.getKey();
+        var v = ent.getValue();
+
+        domain.put(i, k.domain);
+        url.put(i, k.url);
+        count.put(i, v.count);
+        error.put(i, v.error);
+        time.put(i, v.time);
+    }
+    this.table.clear();        
+
+    var out = new StatGeneralPack();
+    out.put("domain", domain);
+    out.put("url", url);
+    out.put("count", count);
+    out.put("error", error);
+    out.put("time", time);
+
+    out.id = "dom";
+    out.time = now;
+    DataPackSender.sendPack(out);
+};
+StatTranxDomain.prototype.clear = function () {
+    this.table.clear();
+};
+
+module.exports = new StatTranxDomain();

package/lib/stat/stat-tx-referer.js

@@ -0,0 +1,112 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var LinkedMap       = require('../util/linkedmap'),
+    AnyList       = require('../util/anylist'),
+    StatGeneralPack   = require('../pack/stat-general-pack'),
+    DataPackSender  = require('../data/datapack-sender'),
+    Logger          = require('../logger'),
+    TimeCount      = require('../pack/time-count');
+
+const TABLE_MAX_SIZE = 5000;
+
+function PK(referer, url) {
+    this.referer = referer || 0;
+    this.url = url || 0;
+}
+
+PK.prototype.hashCode = function () {
+    var prime = 31;
+    var result = 1;
+    result = prime * result + this.referer;
+    result = prime * result + this.url;
+    return result;
+};
+PK.prototype.equals = function (other) {
+    if(this === other) {
+        return  true;
+    }
+    if(other == null) {
+        return false;
+    }
+    return this.referer === other.referer && this.url === other.url;
+};
+
+
+function StatTranxReferer() {
+    if(typeof StatTranxReferer.instance === 'object') {
+        return StatTranxReferer.instance;
+    }
+
+    this.table = new LinkedMap(TABLE_MAX_SIZE + 1 , 1).setMax(TABLE_MAX_SIZE);
+    this.table.create = function (key) {
+        if(this.size() >= TABLE_MAX_SIZE) {
+            return null;
+        }
+        return new TimeCount();
+    };
+    StatTranxReferer.instance = this;
+}
+
+StatTranxReferer.prototype.getService = function (referer, url) {
+    return this.table.intern(new PK(referer,url));
+};
+StatTranxReferer.prototype.add = function (referer, url, time, isErr) {
+    try {
+        var rec = this.getService(referer, url);
+        if(rec) {
+            rec.count++;
+            if(isErr){
+                rec.error++;
+            }
+            rec.time +=time;
+        }
+    } catch(e) {
+        Logger.printError("WHATAP-610", "StatTranxReferer add error", e, false);
+    }
+};
+StatTranxReferer.prototype.send = function (now) {
+    if(this.table.size() === 0) {
+        return;
+    }
+    var rsize=this.table.size();
+    var referer =new AnyList(AnyList.INT, rsize);
+    var url =new  AnyList(AnyList.INT, rsize);
+    var count = new  AnyList(AnyList.INT, rsize);
+    var error = new  AnyList(AnyList.INT, rsize);
+    var time = new  AnyList(AnyList.LONG, rsize);
+
+   var en = this.table.entries();
+    for (var i = 0; i<rsize &&  en.hasMoreElements(); i++) {
+        var ent = en.nextElement();
+        var k = ent.getKey();
+        var v = ent.getValue();
+
+        referer.put(i, k.referer);
+        url.put(i, k.url);
+        count.put(i, v.count);
+        error.put(i, v.error);
+        time.put(i, v.time);
+    }
+    this.table.clear();        
+
+    var out = new StatGeneralPack();
+
+    out.put("referer", referer);
+    out.put("url", url);
+    out.put("count", count);
+    out.put("error", error);
+    out.put("time", time);
+
+    out.id = "referer";
+    out.time = now;
+    DataPackSender.sendPack(out);
+};
+StatTranxReferer.prototype.clear = function () {
+    this.table.clear();
+};
+
+module.exports = new StatTranxReferer();

package/lib/stat/stat-useragent.js

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var IntIntMap           = require('../util/intint-map'),
+    StatUserAgentPack   = require('../pack/statuseragent-pack'),
+    DataPackSender      = require('../data/datapack-sender'),
+    Long                = require('long');
+
+const TABLE_MAX_SIZE = 1000;
+
+function StatUserAgent() {
+    if(typeof StatUserAgent.instance === 'object') {
+        return StatUserAgent.instance;
+    }
+
+    this.table = new StatUserAgentPack();
+    this.table.userAgents.setMax(TABLE_MAX_SIZE);
+
+    StatUserAgent.instance = this;
+}
+
+StatUserAgent.prototype.incUserAgent = function (userAgent) {
+    if(userAgent === 0) {
+        return;
+    }
+    this.table.userAgents.addNoOver(userAgent, 1);
+};
+StatUserAgent.prototype.send = function (now) {
+    if(this.table.userAgents.size() === 0) {
+        return;
+    }
+
+    var p = this.table;
+    p.time = now;
+    DataPackSender.sendStatUserAgentPack(p);
+
+    this.table = new StatUserAgentPack();
+    this.table.userAgents.setMax(TABLE_MAX_SIZE);
+
+};
+StatUserAgent.prototype.clear = function () {
+    this.table.userAgents.clear();
+};
+
+module.exports = new StatUserAgent();

package/lib/stat/timingsender.js

@@ -0,0 +1,76 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+
+var StatError       = require('./stat-error'),
+    StatTranx       = require('./stat-tranx'),
+    StatSql         = require('./stat-sql'),
+    StatHttpc       = require('./stat-httpc'),
+    StatRemoteIp    = require('./stat-remote-ip'),
+    StatRemoteIpUrl = require('./stat-remote-ipurl'),
+    StatUserAgent   = require('./stat-useragent'),
+    StatTranxMtCaller   = require('./stat-tx-caller'),
+    StatTranxDomain   = require('./stat-tx-domain'),
+    StatTranxReferer   = require('./stat-tx-referer'),
+    Logger          = require('../logger'),
+    DateUtil        = require('./../util/dateutil');
+
+const MILLIS_FIVE_MINUTE = 5 * 60 * 1000;
+
+function TimingSender(){
+    this.time = Date.now();
+    this.run();
+    this.intervalIndex = undefined;
+}
+TimingSender.prototype.run = function(){
+    var lastUnit = getCurrentFiveMinUnit();
+    if(this.intervalIndex){
+        clearInterval(this.intervalIndex); this.intervalIndex = undefined;
+    }
+    
+    this.intervalIndex = setInterval(function () {
+        var nowUnit = getCurrentFiveMinUnit();
+        if(nowUnit === lastUnit) { return; }
+        try{
+            lastUnit = nowUnit;
+            var now = DateUtil.currentTime();
+            var conf = require('../conf/configure');
+
+            if(conf.stat_enabled) {
+                StatTranx.send(now);
+                StatSql.send(now);
+                StatHttpc.send(now);
+                StatRemoteIp.send(now);
+                StatRemoteIpUrl.send(now);
+                StatUserAgent.send(now);
+                StatError.send(now);
+                StatTranxDomain.send(now);
+                StatTranxMtCaller.send(now);
+                StatTranxReferer.send(now);
+            } else {
+                StatTranx.clear();
+                StatSql.clear();
+                StatHttpc.clear();
+                StatRemoteIp.clear();
+                StatRemoteIpUrl.clear();
+                StatUserAgent.clear();
+                StatError.clear();
+                StatTranxDomain.clear();
+                StatTranxMtCaller.clear();
+                StatTranxReferer.clear();
+            }
+        }catch(e){
+            Logger.printError('WHATAP-997', 'TimingSender (run)', e, true);
+        }
+
+    }, 1000);
+};
+
+function getCurrentFiveMinUnit() {
+    return parseInt(DateUtil.currentTime() / MILLIS_FIVE_MINUTE) * MILLIS_FIVE_MINUTE
+}
+
+module.exports = new TimingSender();