whatap 1.0.1 → 1.0.2

package/lib/counter/meter/meter-socket.io.js

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var Long = require('long');
+
+function MeterSocketio() {
+
+    if(typeof MeterSocketio.instance === 'object') {
+        return MeterSocketio.instance;
+    }
+
+    this.total_connected = 0;
+    this.connected_count = 0;
+    this.received_count =0;
+    this.received_data =0;
+    this.sent_count =0;
+    this.sent_data =0;
+
+    MeterSocketio.instance = this;
+}
+
+MeterSocketio.prototype.connected = function (count) {
+    this.total_connected += count;
+    this.connected_count += count;
+};
+MeterSocketio.prototype.disconnected = function () {
+    this.connected_count--;
+};
+MeterSocketio.prototype.receive = function (connectid, data) {
+    if(typeof data === 'string') {
+        this.received_data += data.length;
+        this.received_count ++;
+    } else if(typeof data === 'object') {
+        this.received_data += JSON.stringify(data).length;
+        this.received_count ++;
+    }
+};
+MeterSocketio.prototype.send = function (connectid, data) {
+    if(typeof data === 'string') {
+        this.sent_data += data.length;
+        this.sent_count ++;
+    } else if(typeof data === 'object') {
+        this.sent_data += JSON.stringify(data).length;
+        this.sent_count ++;
+    }
+};
+
+module.exports = new MeterSocketio();

package/lib/counter/meter/meter-sql.js

@@ -0,0 +1,71 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var conf      = require('../../conf/configure'),
+    IntIntMap = require('../../util/intint-map'),
+    Long      = require('long');
+var IntKeyLinkedMap = require('../../util/intkey-linkedmap');
+
+function Bucket() {
+    this.count = 0;
+    this.time = 0;
+    this.error = 0;
+
+    this.fetch_count = 0;
+    this.fetch_time = 0;
+
+    
+    this.db_conn_active = new IntIntMap(7, 1);
+    this.db_conn_idle = new IntIntMap(7, 1);
+}
+
+function MeterSql() {
+    this.bucket = new Bucket();
+    this.stat = new IntKeyLinkedMap(500, 1).setMax(500);
+    this.stat.create = function (key) {
+			return new Bucket();
+	};
+}
+
+MeterSql.prototype.add = function (dbc, elapsed, err) {
+    
+    var b = this.bucket;
+    b.count++;
+    b.time +=elapsed;
+    if(err) {
+        b.error++;
+    }
+    if(conf.sql_dbc_meter_enabled && dbc!=0) {
+        b = this.stat.intern(dbc);
+        b.count++;
+
+        b.time += elapsed;
+        if (err) {
+            b.error++;
+        }
+    }
+};
+MeterSql.prototype.getBucketReset = function () {
+    var b = this.bucket;
+    this.bucket = new Bucket();
+    return b;
+};
+MeterSql.prototype.resetStat = function () {
+    if(this.stat.size() > 0){
+        this.stat.clear();
+    }
+}
+MeterSql.prototype.addFetch = function (dbc, count, time) {
+    this.bucket.fetch_count += count;
+    this.bucket.fetch_time += time;
+};
+
+MeterSql.prototype.setConnection = function (active, idle, hash) {
+    this.bucket.db_conn_active.put(hash, active);
+    this.bucket.db_conn_idle.put(hash, idle);
+};
+
+module.exports = new MeterSql();

package/lib/counter/meter/meter-users.js

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var conf                = require('../../conf/configure'),
+    LongKeyLinkedMap   = require('../../util/longkey-linkedmap'),
+    HyperLogLog         = require('../../util/cardinality/hyperloglog'),
+    Long                = require('long');
+
+var add = function (userid) {
+    if(conf.trace_user_enabled == false) { return; }
+
+    if(userid instanceof Long  &&  userid.isZero() == false) {
+        meterUser.users.putLast(userid, Date.now());
+    }
+};
+
+var addActive = function (userid) {
+    if(conf.trace_user_enabled == false) { return; }
+
+    if(userid instanceof Long  &&  userid.isZero() == false) {
+        meterUser.users.put(userid, Date.now());
+    }
+};
+
+var getRealtimeUsers = function () {
+    var max_think_time = conf.realtime_user_thinktime_max;
+    var now = Date.now();
+
+    var loglog = new HyperLogLog();
+    if(meterUser.users.size() == 0) {
+        return loglog;
+    }
+
+    var en = meterUser.users.entries();
+    while(en.hasMoreElements()) {
+        var e = en.nextElement();
+        if(now - e.getValue() > max_think_time) {
+            meterUser.users.remove(e.getKey());
+        } else {
+            loglog.offer(e.getKey());
+        }
+    }
+
+    return loglog;
+};
+
+var meterUser = {
+    MAX_USERS : 70000,
+    users : new LongKeyLinkedMap().setMax(70000),
+    add : add,
+    addActive : addActive,
+    getRealtimeUsers : getRealtimeUsers
+};
+
+module.exports = meterUser;

package/lib/counter/meter.js

@@ -0,0 +1,183 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+function Meter() {
+	this.BUCKET_SIZE = 301;
+	this.initTime = Math.round(Date.now() / 1000);
+	this.lastTime = this.initTime;
+	this.lastOffset = this.initTime % this.BUCKET_SIZE;
+	// this.clear = function() {};
+};
+Meter.prototype.getPosition = function() {
+	var currentTime = Math.round(Date.now() / 1000);
+	if (currentTime != this.lastTime) {
+		for (i = 0; i < (currentTime - this.lastTime) && i < this.BUCKET_SIZE; i++) {
+			this.lastOffset = (this.lastOffset + 1 >= this.BUCKET_SIZE) ? 0
+					: this.lastOffset + 1;
+			this.clear(this.lastOffset);
+		}
+		this.lastTime = currentTime;
+		this.lastOffset = this.lastTime % this.BUCKET_SIZE;
+	}
+
+	return this.lastOffset;
+
+};
+
+Meter.prototype.check = function(period) {
+	if (period >= this.BUCKET_SIZE)
+		period = this.BUCKET_SIZE - 1;
+	return period;
+};
+
+Meter.prototype.getTime = function() {
+	return new Date().getTme() / 1000;
+};
+
+Meter.prototype.desc = function(pos) {
+	if (pos === 0)
+		pos = this.BUCKET_SIZE - 1;
+	else
+		pos--;
+	return pos;
+};
+
+function MeterService() {
+	if (typeof MeterService.instance === "object") {
+		return MeterService.instance;
+	}
+	this.txCount = new Array(this.BUCKET_SIZE);
+	this.errCount = new Array(this.BUCKET_SIZE);
+	this.txDuration = new Array(this.BUCKET_SIZE);
+
+	for ( var i = 0; i < this.BUCKET_SIZE; i++) {
+		this.txCount[i] = 0;
+		this.errCount[i] = 0;
+		this.txDuration[i] = 0;
+	}
+	MeterService.instance = this;
+};
+
+MeterService.prototype = new Meter();
+
+MeterService.prototype.clear = function(offset) {
+	this.txCount[offset] = 0;
+	this.errCount[offset] = 0;
+	this.txDuration[offset] = 0;
+};
+
+MeterService.prototype.add = function(elapsed, err) {
+
+	var pos = this.getPosition();
+	this.txCount[pos]++;
+	this.txDuration[pos] += elapsed;
+	if (err)
+		this.errCount[pos]++;
+
+};
+
+MeterService.prototype.getTPS = function(period) {
+	period = this.check(period);
+	var pos = this.getPosition();
+
+	var sum = 0;
+	for ( var i = 0; i < period; i++, pos = this.desc(pos)) {
+		sum += this.txCount[pos];
+	}
+	return (sum / period);
+};
+
+MeterService.prototype.getElapsedTime = function(period) {
+	period = this.check(period);
+	var pos = this.getPosition();
+
+	var sum = 0;
+	var cnt = 0;
+	for ( var i = 0; i < period; i++, pos = this.desc(pos)) {
+		sum += this.txDuration[pos];
+		cnt += this.txCount[pos];
+	}
+	return ((cnt === 0) ? 0 : sum / cnt);
+};
+
+MeterService.prototype.getError = function(period) {
+	period = this.check(period);
+	var pos = this.getPosition();
+
+	var cnt = 0;
+	var err = 0;
+	for ( var i = 0; i < period; i++, pos = this.desc(pos)) {
+		cnt += this.txCount[pos];
+		err += this.errCount[pos];
+	}
+	return ((cnt === 0) ? 0 : ((err / cnt) * 100.0));
+};
+
+MeterService.prototype.getServiceCount = function(period) {
+	period = this.check(period);
+	var pos = this.getPosition();
+
+	var sum = 0;
+	for ( var i = 0; i < period; i++, pos = this.desc(pos)) {
+		sum += this.txCount[pos];
+	}
+	return sum;
+};
+
+MeterService.prototype.getServiceError = function(period) {
+	period = this.check(period);
+	var pos = this.getPosition();
+
+	var sum = 0;
+	for ( var i = 0; i < period; i++, pos = this.desc(pos)) {
+		sum += this.errCount[pos];
+	}
+	return sum;
+};
+
+function MeterResource() {
+	this.value = new Array(this.BUCKET_SIZE);
+	this.count = new Array(this.BUCKET_SIZE);
+
+	for ( var i = 0; i < this.BUCKET_SIZE; i++) {
+		this.value[i] = 0;
+		this.count[i] = 0;
+
+	}
+
+};
+MeterResource.prototype = new Meter();
+
+MeterResource.prototype.clear = function(offset) {
+	this.value[offset] = 0;
+	this.count[offset] = 0;
+};
+
+MeterResource.prototype.add = function(value) {
+	var pos = this.getPosition();
+	this.value[pos] += value;
+	this.count[pos]++;
+
+};
+
+MeterResource.prototype.getAvg = function(period) {
+	var period = this.check(period);
+	var pos = this.getPosition();
+	var count = 0;
+	var sum = 0;
+
+	for ( var i = 0; i < period; i++, pos = this.desc(pos)) {
+		sum += this.value[pos];
+		count += this.count[pos];
+	}
+	if (count === 0) {
+		return 0;
+	}
+	return sum / count;
+};
+
+module.exports.MeterService = MeterService;
+module.exports.MeterResource = MeterResource;

package/lib/counter/task/activetransaction.js

@@ -4,40 +4,91 @@
  * can be found in the LICENSE file.
  */
 
-var conf = require('../../conf/configure'),
+var CounterTask = require('./counter-task'),
+    StatSql = require('../../stat/stat-sql'),
+    conf = require('../../conf/configure'),
     TraceContextManager = require('../../trace/trace-context-manager'),
+    MeterActiveX = require('../meter/meter-activex'),
+    ActiveStackPack = require('../../pack/activestack-pack'),
+    ActiveStackStep = require('../../step/activestack-step'),
+    DataPackSender = require('../../data/datapack-sender'),
+    DataTextAgent = require('../../data/datatext-agent'),
+    MeterUsers = require('../meter/meter-users'),
     KeyGen = require('../../util/keygen'),
     HashUtil = require('../../util/hashutil'),
+    ResourceProfile = require('../../util/resourceprofile'),
     Logger          = require('../../logger'),
     Long = require('long');
 
-function ActiveTransaction() {}
-ActiveTransaction.prototype.getActiveStats = function () {
-    var stats = new Array(5).fill(0);
+function ActiveTransaction() {
+    CounterTask.call(this);
+    this.last_stack_sec = 0;
+}
+
+ActiveTransaction.prototype = new CounterTask();
+ActiveTransaction.prototype.constructor = ActiveTransaction;
+ActiveTransaction.prototype.process = function (p) {
+
+    p.active_tx_slice = [0, 0, 0];
+    p.active_stat = new Array(5).fill(0);
+
+    var activeX = null;
+    if (conf.actx_meter_enabled) {
+        activeX = MeterActiveX;
+        activeX.reinit();
+    }
+
     var en = TraceContextManager.getContextEnumeration();
     try {
         while (en.hasMoreElements()) {
             var ctx = en.nextElement();
-            if (!ctx) {
-                break;
+            if (ctx == null) {
+                return false;
+            }
+            var elapsed = ctx.getElapsedTime();
+            if (elapsed < conf.trace_active_transaction_yellow_time) {
+                p.active_tx_slice[0]++;
+            } else if (elapsed < conf.trace_active_transaction_red_time) {
+                p.active_tx_slice[1]++;
+            } else {
+                p.active_tx_slice[2]++;
+                if (elapsed >= conf.trace_active_transaction_hang_time) {
+                    p.act_svc_hang++;
+                }
+            }
+            p.active_tx_count++;
+            if (activeX) {
+                activeX.addTx(ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_oid);
+                var httpc = ctx.active_httpc_hash;
+                if (httpc != 0) {
+                    activeX.addHttpc(httpc);
+                } else {
+                    var dbc = ctx.active_dbc;
+                    if (dbc != 0) {
+                        activeX.addSql(dbc);
+                    }
+                }
             }
-            if (ctx.active_sqlhash) {
-                stats[1]++;
-            } else if (ctx.active_httpc_hash) {
-                stats[2]++; // httpc
+
+
+            if (ctx.active_sqlhash != 0) {
+                p.active_stat[1]++; // sql
+            } else if (ctx.active_httpc_hash != 0) {
+                p.active_stat[2]++; // httpc
             } else if (ctx.db_opening) {
-                stats[3]++; // dbc
+                p.active_stat[3]++; // dbc
             } else if (ctx.socket_connecting) {
-                stats[4]++; // socket
+                p.active_stat[4]++; // socket
             } else {
-                stats[0]++; // method
+                p.active_stat[0]++; // method
             }
+
+            MeterUsers.addActive(ctx.userid);
         }
-        return stats;
     } catch (e) {
-        Logger.printError("WHATAP-602", 'Active Transaction(process)', e)
-        return stats;
+        Logger.printError("WHATAP-071", 'Active Transaction(process)', e)
     }
+
 };
 
-module.exports = new ActiveTransaction();
+module.exports = ActiveTransaction;

package/lib/counter/task/agentinfo.js

@@ -0,0 +1,107 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var CounterTask = require('./counter-task'),
+    StatSql = require('../../stat/stat-sql'),
+    conf = require('../../conf/configure'),
+    SecurityMaster = require('../../net/security-master'),
+    TraceContextManager = require('../../trace/trace-context-manager'),
+    MeterActiveX = require('../meter/meter-activex'),
+    ActiveStackPack = require('../../pack/activestack-pack'),
+    ParamPack = require('../../pack/param-pack'),
+    EventPack = require('../../pack/event-pack'),
+    ActiveStackStep = require('../../step/activestack-step'),
+    DataPackSender = require('../../data/datapack-sender'),
+    DataTextAgent = require('../../data/datatext-agent').agent,
+    EventLevel  = require('../../data/event-level'),
+    MeterUsers = require('../meter/meter-users'),
+    KeyGen = require('../../util/keygen'),
+    TextTypes = require('../../lang/text-types'),
+    ResourceProfile = require('../../util/resourceprofile'),
+    Logger          = require('../../logger'),
+    HashUtil = require('../../util/hashutil'),
+    KubeUtil        = require('../../util/kube-util'),
+    DateUtil        = require('../../util/dateutil'),
+    fs = require('fs'),
+    PiiMask = require('../../pii/pii-mask');
+
+function AgentInfo() {
+    CounterTask.call(this);
+
+    this.startTime = Date.now();
+    this.last_name_sent = 0;
+}
+
+AgentInfo.prototype = new CounterTask();
+AgentInfo.prototype.constructor = AgentInfo;
+AgentInfo.prototype.process = function (p) {
+    // PiiMask.update();
+
+    p.starttime = Date.now();
+    /*
+    if (p.starttime > this.startTime + DateUtil.MILLIS_PER_FIVE_MINUTE) {
+        this.startTime = now + DateUtil.MILLIS_PER_HOUR * 12;
+
+        DataPackSender.sendBoot(AgentInfo.info);
+        // ComponentsVersions.send();
+    }*/
+
+    this.sendName(p.starttime);
+
+    p.containerKey = KubeUtil.container_key;
+
+    this.checkAutoScaleIn();
+}
+
+AgentInfo.prototype.sendName = function(now /* long */){
+    if (now - this.last_name_sent < DateUtil.MILLIS_PER_FIVE_MINUTE) {
+        return;
+    }
+    this.last_name_sent = now;
+    
+    var sm = SecurityMaster;
+    if (sm.ONAME != null) {
+        DataTextAgent.add(TextTypes.ONAME, sm.OID, sm.ONAME);
+    }
+    
+    if (sm.OKIND != 0) {
+        DataTextAgent.add(TextTypes.OKIND_NAME, sm.OKIND, sm.OKIND_NAME);
+    }
+    
+    if (sm.ONODE != 0) {
+        DataTextAgent.add(TextTypes.ONODE_NAME, sm.ONODE, sm.ONODE_NAME);
+    }
+    if (KubeUtil.container_key != 0) {
+        DataTextAgent.add(TextTypes.CONTAINER_ID, KubeUtil.container_key, KubeUtil.container_id);
+    }
+
+    if (conf.simula_agent_count > 0) {
+        for (var i = 0; i < conf.simula_agent_count; i++) {
+            var oname = sm.ONAME + "#" + i;
+            var ohash = HashUtil.hashFromString(oname);
+            DataTextAgent.add(TextTypes.ONAME, ohash, oname);
+        }
+    }
+
+    if(conf.mtrace_spec != undefined && conf.mtrace_spec.length >0) {
+        DataTextAgent.add(TextTypes.MTRACE_SPEC, conf.mtrace_spec_hash, conf.mtrace_spec);
+    }
+}
+
+AgentInfo.prototype.checkAutoScaleIn = function() {
+    var home = conf["whatap.home"];
+    var file = fs.existsSync(home, process.pid + ".shutdown");
+    if (file) {
+        var e = new EventPack();
+        e.title = "SILENT_SHUTDOWN";
+        e.level = EventLevel.INFO;
+        DataPackSender.sendPack(e);
+    }
+}
+
+module.exports = AgentInfo;
+
+