whatap 0.4.97 → 0.5.0

package/lib/pack/log-sink-pack.js

@@ -0,0 +1,139 @@
+const Pack = require('./pack');
+const PackEnum = require('./packenum');
+const MapValue = require('../value/map-value');
+const DataOutputX = require('../io/data-outputx');
+const DataInputX = require('../io/data-inputx');
+const HashUtil = require('../util/hashutil');
+const Value = require('../value/value');
+const BooleanValue = require('../value/boolean-value');
+
+function LogSinkPack() {
+    Pack.call(this);
+    this.category = '';
+    this.tagHash = 0;
+    this.tags = new MapValue();
+    this.line = 0;
+    this.content = '';
+    this.fields = new MapValue();
+    this.dropped = false;
+}
+
+LogSinkPack.prototype = Object.create(Pack.prototype);
+LogSinkPack.prototype.constructor = LogSinkPack;
+
+LogSinkPack.prototype.getPackType = function () {
+    return PackEnum.LOGSINK;
+};
+
+LogSinkPack.prototype.toString = function () {
+    return `${Pack.prototype.toString.call(this)} [category=${this.category}, tagHash=${this.tagHash}, tags=${this.tags}, content=${this.content}, fields=${this.fields}]`;
+};
+
+LogSinkPack.prototype.getTagAsBytes = function () {
+    const out = new DataOutputX();
+    out.writeValue(this.tags);
+    return out.toByteArray();
+};
+
+LogSinkPack.prototype.getContentBytes = function () {
+    const o = new DataOutputX();
+    o.writeByte(1);
+    o.writeText(this.content);
+    o.writeDecimal(this.line);
+    return o.toByteArray();
+};
+
+LogSinkPack.prototype.putCtr = function (key, value) {
+    if (key.charAt(0) === '!') {
+        this.fields.putValue(key, value);
+    }
+};
+
+LogSinkPack.prototype.getCtr = function (key) {
+    return this.fields.get(key);
+};
+
+LogSinkPack.prototype.getCtrBoolean = function (key) {
+    const v = this.fields.get(key);
+    if (v instanceof BooleanValue) {
+        return v.value;
+    }
+    return false;
+};
+
+LogSinkPack.prototype.content = function () {
+    return this.content || '';
+};
+
+LogSinkPack.prototype.setContent = function (str) {
+    this.content = str;
+};
+
+LogSinkPack.prototype.setContentBytes = function (d) {
+    try {
+        if (!d || d.length < 1) return;
+        const inStream = new DataInputX(d);
+        const ver = inStream.readByte();
+        if (ver === 1) {
+            this.content = inStream.readText();
+            this.line = inStream.readDecimal();
+        }
+    } catch (e) {
+        // Handle error
+    }
+};
+
+LogSinkPack.prototype.write = function (dout) {
+    Pack.prototype.write.call(this, dout);
+    dout.writeByte(0);
+    dout.writeText(this.category);
+    dout.writeDecimal(this.tagHash);
+    dout.writeValue(this.tags);
+    dout.writeDecimal(this.line);
+    dout.writeText(this.content);
+    if (this.fields.size() > 0) {
+        dout.writeBoolean(true);
+        dout.writeValue(this.fields);
+    } else {
+        dout.writeBoolean(false);
+    }
+};
+
+LogSinkPack.prototype.read = function (din) {
+    Pack.prototype.read.call(this, din);
+    const ver = din.readByte();
+    this.category = din.readText();
+    this.tagHash = din.readDecimal();
+    this.tags = din.readValue();
+    this.line = din.readDecimal();
+    this.content = din.readText();
+    if (din.readBoolean()) {
+        this.fields = din.readValue();
+    }
+    return this;
+};
+
+LogSinkPack.prototype.resetTagHash = function () {
+    const out = new DataOutputX();
+    out.writeValue(this.tags);
+    const tagBytes = out.toByteArray();
+    this.tagHash = HashUtil.hash64(tagBytes);
+    return tagBytes;
+};
+
+LogSinkPack.prototype.transferOidToTag = function () {
+    if (this.oid !== 0 && !this.tags.containsKey('oid')) {
+        this.tags.putValue('oid', this.oid);
+        this.tagHash = 0;
+    }
+    if (this.okind !== 0 && !this.tags.containsKey('okind')) {
+        this.tags.putValue('okind', this.okind);
+        this.tagHash = 0;
+    }
+    if (this.onode !== 0 && !this.tags.containsKey('onode')) {
+        this.tags.putValue('onode', this.onode);
+        this.tagHash = 0;
+    }
+};
+
+module.exports = LogSinkPack;

package/lib/pack/packenum.js

@@ -32,6 +32,7 @@ exports.HITMAP5M    = 0x1700;
 
 exports.TAG_COUNT   = 0x1601;
 exports.ZIP         = 0x170b;
+exports.LOGSINK     = 0x170a;
 
 exports.PACK_NAME = {
     0x0100 : 'PARAMETER',
@@ -58,6 +59,7 @@ exports.PACK_NAME = {
     0x1600 : 'COUNTER5M',
     0x1700 : 'HITMAP5M',
     0x1601 : 'TAG_COUNT',
-    0x170b : 'ZIP'
+    0x170b : 'ZIP',
+    0x170a : 'LOGSINK',
 };
  

package/lib/pack/zip-pack.js

@@ -10,7 +10,6 @@ var PackEnum = require('./packenum'),
     Step = require('../step/step'),
     DataOutputX = require('../io/data-outputx'),
     DataInputX = require('../io/data-inputx');
-var zlib = require('zlib');
 
 function ZipPack() {
     Pack.call(this);

package/lib/stat/{stat-remoteip.js → stat-remote-ip.js}

@@ -9,23 +9,23 @@ var StatRemoteIpPack    = require('../pack/statremote-pack'),
 
 var TABLE_MAX_SIZE = 70000;
 
-function StatRemoteIP() {
-    if(typeof StatRemoteIP.instance === 'object') {
-        return StatRemoteIP.instance;
+function StatRemoteIp() {
+    if(typeof StatRemoteIp.instance === 'object') {
+        return StatRemoteIp.instance;
     }
 
     this.table = new StatRemoteIpPack();
     this.table.iptable.setMax(TABLE_MAX_SIZE);
 
-    StatRemoteIP.instance = this;
+    StatRemoteIp.instance = this;
 }
 
-StatRemoteIP.prototype.incRemoteIp = function (ip) {
+StatRemoteIp.prototype.incRemoteIp = function (ip) {
     if(ip) {
         this.table.iptable.addNoOver(ip,1);
     }
 };
-StatRemoteIP.prototype.send = function (now) {
+StatRemoteIp.prototype.send = function (now) {
     if(this.table.iptable.size() === 0) {
         return;
     }
@@ -37,10 +37,10 @@ StatRemoteIP.prototype.send = function (now) {
     this.table = new StatRemoteIpPack();
     this.table.iptable.setMax(TABLE_MAX_SIZE);
 };
-StatRemoteIP.prototype.clear = function () {
+StatRemoteIp.prototype.clear = function () {
     this.table.iptable.clear();
 };
 
-module.exports = new StatRemoteIP();
+module.exports = new StatRemoteIp();
 
 

package/lib/stat/stat-remote-ipurl.js

@@ -0,0 +1,88 @@
+const StatGeneralPack = require('../pack/stat-general-pack');
+const LongKeyLinkedMap = require('../util/longkey-linkedmap');
+const TimeCount = require('../pack/time-count');
+const conf = require('../conf/configure');
+const BitUtil = require('../util/bitutil');
+const ZipProfile = require('../data/zipprofile');
+const DataPackSender = require('../data/datapack-sender');
+const AnyList       = require('../util/anylist');
+
+var TABLE_MAX_SIZE = 10000;
+
+function StatRemoteIpurl() {
+    if (typeof StatRemoteIpurl.instance === 'object') {
+        return StatRemoteIpurl.instance;
+    }
+
+    this.TABLE_MAX_SIZE = TABLE_MAX_SIZE;
+    this.table = new LongKeyLinkedMap(TABLE_MAX_SIZE + 1, 1).setMax(TABLE_MAX_SIZE);
+    this.table.create = function (key) {
+        return new TimeCount();
+    };
+
+    StatRemoteIpurl.instance = this;
+}
+
+StatRemoteIpurl.prototype.getService = function (ip, url_hash) {
+    var key = BitUtil.composite(ip, url_hash);
+    if (this.table.size() < conf.getProperty('stat_ipurl_max_count', 10000)) {
+        return this.table.intern(key);
+    } else {
+        return this.table.get(key);
+    }
+};
+
+StatRemoteIpurl.prototype.send = function (now) {
+    if (this.table.size() === 0) return;
+
+    var currTable = this.table;
+    var sz = Math.max(this.TABLE_MAX_SIZE, conf.getProperty('stat_ipurl_max_count', 10000));
+    this.table = new LongKeyLinkedMap(sz + 1, 1).setMax(conf.getProperty('stat_ipurl_max_count', 10000));
+    this.table.create = function (key) {
+        return new TimeCount();
+    };
+
+    var rsize = this.table.size();
+    var ip = new AnyList(AnyList.INT, rsize);
+    var url = new AnyList(AnyList.INT, rsize);
+    var count = new AnyList(AnyList.INT, rsize);
+    var error = new AnyList(AnyList.INT, rsize);
+    var time = new AnyList(AnyList.LONG, rsize);
+
+    var entries = currTable.entries();
+    var i = 0;
+    while(entries.hasMoreElements()){
+        var ent = entries.nextElement();
+        var k = ent.getKey();
+        var v = ent.getValue();
+
+        ip.put(i, BitUtil.getHigh(k));
+        url.put(i, BitUtil.getLow(k));
+        count.put(i, v.count);
+        error.put(i, v.error);
+        time.put(i, v.time);
+        i++;
+    }
+
+    const out = new StatGeneralPack();
+    out.put("ip", ip);
+    out.put("url", url);
+    out.put("count", count);
+    out.put("error", error);
+    out.put("time", time);
+
+    out.id = "ip-url";
+    out.time = now;
+
+    DataPackSender.sendStatRemoteIpUrlPack(out);
+};
+
+StatRemoteIpurl.prototype.clear = function () {
+    this.table.clear();
+};
+
+StatRemoteIpurl.prototype.size = function () {
+    return this.table.size();
+};
+
+module.exports = new StatRemoteIpurl();

package/lib/stat/timingsender.js

@@ -9,7 +9,8 @@ var StatError       = require('./stat-error'),
     StatTranx       = require('./stat-tranx'),
     StatSql         = require('./stat-sql'),
     StatHttpc       = require('./stat-httpc'),
-    StatRemoteIp    = require('./stat-remoteip'),
+    StatRemoteIp    = require('./stat-remote-ip'),
+    StatRemoteIpUrl = require('./stat-remote-ipurl'),
     StatUserAgent   = require('./stat-useragent'),
     StatTranxMtCaller   = require('./stat-tx-caller'),
     StatTranxDomain   = require('./stat-tx-domain'),
@@ -43,6 +44,7 @@ TimingSender.prototype.run = function(){
                 StatSql.send(now);
                 StatHttpc.send(now);
                 StatRemoteIp.send(now);
+                StatRemoteIpUrl.send(now);
                 StatUserAgent.send(now);
                 StatError.send(now);
                 StatTranxDomain.send(now);
@@ -53,6 +55,7 @@ TimingSender.prototype.run = function(){
                 StatSql.clear();
                 StatHttpc.clear();
                 StatRemoteIp.clear();
+                StatRemoteIpUrl.clear();
                 StatUserAgent.clear();
                 StatError.clear();
                 StatTranxDomain.clear();

package/lib/topology/link.js

@@ -0,0 +1,63 @@
+var { InetAddress } = require('net');
+var CompareUtil = require('../util/compare-util');
+var HashUtil = require('../util/hashutil');
+
+function LINK() {
+    this.ip = null;
+    this.port = 0;
+
+    this.toString = function () {
+        if (this.ip === null) return `0.0.0.0:${this.port}`;
+        try {
+            return `${InetAddress.getByAddress(this.ip).getHostAddress()}:${this.port}`;
+        } catch (e) {
+            return `0.0.0.0:${this.port}`;
+        }
+    };
+
+    this.include = function (k) {
+        if (!CompareUtil.equals(this.ip, k.ip)) return false;
+        if (this.port === 0) return true;
+        return this.port === k.port;
+    };
+
+    this.equals = function (o) {
+        if (o.constructor !== this.constructor) return false;
+        var k = o;
+        if (!CompareUtil.equals(this.ip, k.ip)) return false;
+        return this.port === k.port;
+    };
+
+    this.hashCode = function () {
+        return HashUtil.hash(this.ip) | this.port;
+    };
+
+    this.toBytes = function (out) {
+        out.writeBlob(this.ip);
+        out.writeInt(this.port);
+        return this;
+    };
+
+    this.toObject = function (din) {
+        this.ip = din.readBlob();
+        this.port = din.readInt();
+        return this;
+    };
+}
+
+LINK.create = function (ipStr, port) {
+    var k = new LINK();
+    try {
+        k.ip = InetAddress.getByName(ipStr).getAddress();
+    } catch (e) {
+        k.ip = new Buffer(4);
+    }
+    k.port = port;
+    return k;
+};
+
+LINK.prototype = new LINK();
+LINK.prototype.constructor = LINK;
+
+module.exports = LINK;
+

package/lib/topology/nodeinfo.js

@@ -0,0 +1,123 @@
+const DataOutputX = require('../io/data-outputx');
+const DataInputX = require('../io/data-inputx');
+const MapValue = require('../value/map-value');
+const LINK = require('./link');
+const net = require('net');
+
+function NodeInfo() {
+    this.attr = new MapValue();
+    this.listen = new Set();
+    this.outter = new Set();
+
+    this.addListen = function (localIPs, address) {
+        const ipo = this.getIPPORT(address);
+        if (!ipo || this.isLocal127(ipo)) return;
+
+        if (ipo.ip === '*' || ipo.ip === '0.0.0.0' || ipo.ip === '::') {
+            localIPs.forEach(local_ip => {
+                const k = LINK.create(local_ip, parseInt(ipo.port, 10));
+                if (k) this.listen.add(k);
+            });
+        } else {
+            const k = LINK.create(ipo.ip, parseInt(ipo.port, 10));
+            if (k) this.listen.add(k);
+        }
+    };
+
+    this.addOutter = function (local, remote) {
+        const localIPO = this.getIPPORT(local);
+        if (!localIPO || this.isIPv6(localIPO)) return;
+        if (this.hasListen(localIPO.ip, localIPO.port)) return;
+
+        const remoteIPO = this.getIPPORT(remote);
+        if (!remoteIPO || this.isIPv6(remoteIPO) || this.isLocal127(remoteIPO)) return;
+
+        const k = LINK.create(remoteIPO.ip, parseInt(remoteIPO.port, 10));
+        if (k) this.outter.add(k);
+    };
+
+    this.hasListen = function (ip, port) {
+        const k = LINK.create(ip, parseInt(port, 10));
+        return this.listen.has(k);
+    };
+
+    this.getIPPORT = function (address) {
+        try {
+            const ipo = {};
+            const x = address.lastIndexOf(':');
+            ipo.ip = address.substring(0, x);
+            ipo.port = address.substring(x + 1);
+            return ipo;
+        } catch (e) {
+            return null;
+        }
+    };
+
+    this.toBytes = function () {
+        const out = new DataOutputX();
+        out.writeByte(0);
+        out.writeValue(this.attr);
+        this.toLinkBytes(this.listen, out);
+        this.toLinkBytes(this.outter, out);
+        return out.toByteArray();
+    };
+
+    this.toObject = function (b) {
+        const din = new DataInputX(b);
+        const ver = din.readByte();
+        this.attr = din.readValue();
+        this.listen = this.toLinkObject(din);
+        this.outter = this.toLinkObject(din);
+        return this;
+    };
+
+    this.toLinkObject = function (din) {
+        const data = new Set();
+        const sz = din.readDecimal();
+        for (let i = 0; i < sz; i++) {
+            data.add(new LINK().toObject(din));
+        }
+        return data;
+    };
+
+    this.toLinkBytes = function (data, out) {
+        out.writeDecimal(data.size);
+        if (data.size === 0) return;
+        data.forEach(k => k.toBytes(out));
+    };
+
+    this.toJSON = function () {
+        const o = new Map();
+        o.put('attr', JSON.stringify(this.attr));
+        o.put('listen', JSON.parse(this.listen));
+        o.put('outter', JSON.parse(this.outter));
+        return o;
+    };
+
+    this.toJSON = function (data) {
+        const out = new JSONArray();
+        data.forEach(k => out.put(k.toString()));
+        return out;
+    };
+
+    this.isIPv6 = function(data) {
+        var ip = data.ip;
+        if(ip){
+            return net.isIPv6(ip);
+        }
+        return false;
+    };
+
+    this.isLocal127 = function (data) {
+        var ip = data.ip;
+        if(ip){
+            return ip === '127.0.0.1';
+        }
+        return false;
+    }
+}
+
+NodeInfo.prototype = new NodeInfo();
+NodeInfo.prototype.constructor = NodeInfo;
+
+module.exports = NodeInfo;

package/lib/topology/status-detector.js

@@ -0,0 +1,111 @@
+var { exec } = require('child_process');
+var os = require('os');
+var NodeInfo = require('./nodeinfo');
+var Logger = require('../logger');
+
+function StatusDetector() {
+    this.process = async function () {
+        let node = new NodeInfo();
+        let stat = await this.netstat();
+
+        if (!stat) {
+            node = new NodeInfo();
+        } else {
+            try {
+                node = this.parse(stat);
+            } catch (e) {
+                node = new NodeInfo();
+                Logger.printError("WHATAP-203", "NodeInfo parse error", e, false);
+            }
+        }
+
+        node.attr.type = 'nodejs';
+        node.attr.time = Date.now();
+        node.attr.ip = this.getLocalIpAddress();
+        node.attr.pid = process.pid;
+        node.attr.pname = process.title;
+
+        return node;
+    };
+
+    this.netstat = function () {
+        return new Promise((resolve, reject) => {
+            let cmd = 'netstat -an -t';
+            if (os.platform() === 'darwin') {
+                cmd = 'netstat -an -p tcp';
+            } else if (os.platform() === 'win32') {
+                resolve('');
+                return;
+            }
+
+            exec(cmd, (error, stdout, stderr) => {
+                if (error) {
+                    reject(error);
+                    return;
+                }
+                resolve(stdout);
+            });
+        });
+    };
+
+    this.parse = function (netstat) {
+        var node = new NodeInfo();
+        var localIPs = this.getLocalIpSet();
+
+        var lines = netstat.split('\n');
+        lines.forEach(line => {
+            if (line.startsWith('tcp')) {
+                this.parseLine(node, line, localIPs);
+            }
+        });
+
+        return node;
+    };
+
+    this.parseLine = function (node, line, localIPs) {
+        var tokens = line.trim().split(/\s+/);
+        if (!tokens[0].startsWith('tcp')) return;
+
+        var localAddress = tokens[3].replace(/(\d+\.\d+\.\d+\.\d+)\.(\d+)/, '$1:$2');
+        var remoteAddress = tokens[4].replace(/(\d+\.\d+\.\d+\.\d+)\.(\d+)/, '$1:$2');
+
+        if (tokens[5] === 'LISTEN') {
+            node.addListen(localIPs, localAddress);
+        } else {
+            node.addOutter(localAddress, remoteAddress);
+        }
+    };
+
+    this.getLocalIpSet = function () {
+        var ipSet = new Set();
+        var interfaces = os.networkInterfaces();
+        for (var devName in interfaces) {
+            var iface = interfaces[devName];
+            iface.forEach(alias => {
+                if (alias.family === 'IPv4' && !alias.internal) {
+                    ipSet.add(alias.address);
+                }
+            });
+        }
+        return ipSet;
+    };
+
+    this.getLocalIpAddress = function () {
+        var interfaces = os.networkInterfaces();
+        for (var devName in interfaces) {
+            var iface = interfaces[devName];
+            for (let i = 0; i < iface.length; i++) {
+                var alias = iface[i];
+                if (alias.family === 'IPv4' && !alias.internal) {
+                    return alias.address;
+                }
+            }
+        }
+        return '0.0.0.0';
+    };
+}
+
+StatusDetector.prototype = new StatusDetector();
+StatusDetector.prototype.constructor = StatusDetector;
+
+module.exports = StatusDetector;