whatap 0.4.98 → 0.5.1

package/lib/observers/socket.io-observer.js

@@ -4,27 +4,24 @@
  * can be found in the LICENSE file.
  */
 
-var MeterSocketio = require('../counter/meter/meter-socket.io'),
-    TraceContextManager = require('../trace/trace-context-manager'),
+var TraceContextManager = require('../trace/trace-context-manager'),
     SocketStep = require('../step/socket-step'),
     conf = require('../conf/configure'),
     IPUtil = require('../util/iputil'),
     Logger = require('../logger');
-const {Detector: URLPatternDetector} = require("whatap/lib/trace/serviceurl-pattern-detector");
-const HashUtil = require("whatap/lib/util/hashutil");
-const DataTextAgent = require("whatap/lib/data/datatext-agent");
-const ResourceProfile = require("whatap/lib/util/resourceprofile");
+const {Detector: URLPatternDetector} = require("../trace/serviceurl-pattern-detector");
+const DataTextAgent = require("../data/datatext-agent");
+const ResourceProfile = require("../util/resourceprofile");
 const ProfilePack         = require('../pack/profile-pack');
 const TxRecord            = require('../service/tx-record');
 const DateUtil            = require('../util/dateutil');
 const SecurityMaster      = require('../net/security-master');
 const DataProfileAgent    = require('../data/dataprofile-agent');
-const MessageStep = require("whatap/lib/step/message-step");
-const MeterUsers = require("whatap/lib/counter/meter/meter-users");
-const DataPackSender = require("whatap/lib/data/datapack-sender");
+const MeterUsers = require("../counter/meter/meter-users");
 const MeterService        = require('../counter/meter/meter-service').MeterService;
+const shimmer = require('../core/shimmer');
 
-var trace_background_socket_enabled = conf.getProperty('trace_background_socket_enabled', false);
+var trace_background_socket_enabled = conf.getProperty('trace_background_socket_enabled', true);
 conf.on('trace_background_socket_enabled', function (newProps) {
     trace_background_socket_enabled = newProps;
 })
@@ -55,66 +52,78 @@ SocketIOObserver.prototype.inject = function (mod, moduleName) {
     Logger.initPrint("SocketIOObserver");
 
     var self = this;
-    var aop = self.agent.aop;
 
-    aop.after(mod.Server.prototype, 'on', function (obj, args, ret, lctx) {
-        aop.after(mod.Socket.prototype, 'emit', function (_obj, _args, _ret, _lctx) {
-            if(!trace_background_socket_enabled) {
-                return;
+    shimmer.wrap(mod.prototype, 'on', function (original) {
+        return function (event, listener) {
+            if (event === 'connection') {
+                return original.call(this, event, function (socket) {
+                    shimmer.wrap(socket, 'emit', function (origEmit) {
+                        return function (emitEvent, ...args) {
+                            if (trace_background_socket_enabled) {
+                                self.__handleSocketEmitEvent(socket, emitEvent, args);
+                            }
+
+                            return origEmit.apply(this, [emitEvent, ...args]);
+                        };
+                    });
+
+                    return listener.apply(this, [socket]);
+                });
             }
-            if(trace_sampling_enabled){
-                var now = Date.now();
-                if(!socket_count.start_time) {
-                    socket_count.start_time = now;
-                }
-
-                if ((now - socket_count.start_time) > 5000) {
-                    socket_count.start_time = now;
-                    socket_count.count = 0;
-                }
-
-                socket_count.count++;
-                if(socket_count.count > trace_sampling_tps) {
-                    MeterService.add(0, 1, 0, 0, 0, 0);
-                    return;
-                }
+
+            return original.call(this, event, listener);
+        };
+    });
+};
+
+SocketIOObserver.prototype.__handleSocketEmitEvent = function(socket, event, args) {
+    if (trace_sampling_enabled) {
+        var now = Date.now();
+        if (!socket_count.start_time) {
+            socket_count.start_time = now;
+        }
+
+        if ((now - socket_count.start_time) > 5000) {
+            socket_count.start_time = now;
+            socket_count.count = 0;
+        }
+
+        socket_count.count++;
+        if (socket_count.count > trace_sampling_tps) {
+            MeterService.add(0, 1, 0, 0, 0, 0);
+            return;
+        }
+    }
+
+    TraceContextManager._asyncLocalStorage.run(initCtx(socket, args), () => {
+        try {
+            var ctx = TraceContextManager._asyncLocalStorage.getStore();
+            if (!ctx) return;
+
+            ctx.footprint('Socket Emit Event: ' + event);
+
+            var host;
+            if (socket.conn && socket.conn.remoteAddress && socket.conn.remoteAddress.includes(':')) {
+                host = socket.conn.remoteAddress.substring(socket.conn.remoteAddress.lastIndexOf(':') + 1);
             }
 
-            TraceContextManager._asyncLocalStorage.run(initCtx(_obj, _args), () => {
-                try{
-                    var ctx = TraceContextManager._asyncLocalStorage.getStore();
-                    if (!ctx) {
-                        return;
-                    }
-
-                    var socket = _obj;
-                    var host;
-                    if (socket.handshake && socket.handshake.address && socket.handshake.address.includes(':')) {
-                        host = socket.handshake.address.substring(socket.handshake.address.lastIndexOf(':') + 1);
-                    }
-
-                    ctx.socket_connecting = true;
-                    ctx.footprint('Socket Connecting: ' + host);
-
-                    var step = new SocketStep();
-                    step.start_time = ctx.getElapsedTime();
-                    step.ipaddr = Buffer.from(IPUtil.stringToBytes(host));
-                    // step.port = port;
-
-                    ctx.socket_connecting = false;
-                    step.elapsed = ctx.getElapsedTime() - step.start_time;
-                    ctx.profile.push(step);
-
-                    ctx.footprint('Socket Connecting Done');
-
-                    self.__endTransaction(null, ctx);
-                    return;
-                }catch (e) {
-                    Logger.printError('WHATAP-616', 'socket.io transaction error..', e, false);
-                    return;
-                }
-            });
-        });
+            ctx.socket_connecting = true;
+            ctx.footprint('Socket Connecting: ' + host);
+
+            var step = new SocketStep();
+            step.start_time = ctx.getElapsedTime();
+            step.ipaddr = Buffer.from(IPUtil.stringToBytes(host));
+            // step.port = port;
+
+            ctx.socket_connecting = false;
+            step.elapsed = ctx.getElapsedTime() - step.start_time;
+            ctx.profile.push(step)
+
+            this.__endTransaction(null, ctx);
+
+        } catch (e) {
+            Logger.printError('WHATAP-616', 'socket.io emit transaction error..', e, false);
+        }
     });
 };
 
@@ -163,7 +172,7 @@ function initCtx(socket, args) {
     if (!ctx) {return;}
 
     var remote_addr;
-    const address = socket.handshake.address;
+    const address = socket.conn.remoteAddress;
     if(address && address.includes(':')){
         remote_addr = address.substring(address.lastIndexOf(':')+1);
     }

package/lib/pack/event-pack.js

@@ -27,7 +27,7 @@ EventPack.prototype.write = function(dout) {
     dout.writeText(this.message);
     dout.writeByte(Object.keys(this.attr).length);
 
-    for(key in this.attr) {
+    for(var key in this.attr) {
         dout.writeText(key);
         dout.writeText(this.attr[key]);
     }

package/lib/pack/log-sink-pack.js

@@ -0,0 +1,139 @@
+const Pack = require('./pack');
+const PackEnum = require('./packenum');
+const MapValue = require('../value/map-value');
+const DataOutputX = require('../io/data-outputx');
+const DataInputX = require('../io/data-inputx');
+const HashUtil = require('../util/hashutil');
+const Value = require('../value/value');
+const BooleanValue = require('../value/boolean-value');
+
+function LogSinkPack() {
+    Pack.call(this);
+    this.category = '';
+    this.tagHash = 0;
+    this.tags = new MapValue();
+    this.line = 0;
+    this.content = '';
+    this.fields = new MapValue();
+    this.dropped = false;
+}
+
+LogSinkPack.prototype = Object.create(Pack.prototype);
+LogSinkPack.prototype.constructor = LogSinkPack;
+
+LogSinkPack.prototype.getPackType = function () {
+    return PackEnum.LOGSINK;
+};
+
+LogSinkPack.prototype.toString = function () {
+    return `${Pack.prototype.toString.call(this)} [category=${this.category}, tagHash=${this.tagHash}, tags=${this.tags}, content=${this.content}, fields=${this.fields}]`;
+};
+
+LogSinkPack.prototype.getTagAsBytes = function () {
+    const out = new DataOutputX();
+    out.writeValue(this.tags);
+    return out.toByteArray();
+};
+
+LogSinkPack.prototype.getContentBytes = function () {
+    const o = new DataOutputX();
+    o.writeByte(1);
+    o.writeText(this.content);
+    o.writeDecimal(this.line);
+    return o.toByteArray();
+};
+
+LogSinkPack.prototype.putCtr = function (key, value) {
+    if (key.charAt(0) === '!') {
+        this.fields.putValue(key, value);
+    }
+};
+
+LogSinkPack.prototype.getCtr = function (key) {
+    return this.fields.get(key);
+};
+
+LogSinkPack.prototype.getCtrBoolean = function (key) {
+    const v = this.fields.get(key);
+    if (v instanceof BooleanValue) {
+        return v.value;
+    }
+    return false;
+};
+
+LogSinkPack.prototype.content = function () {
+    return this.content || '';
+};
+
+LogSinkPack.prototype.setContent = function (str) {
+    this.content = str;
+};
+
+LogSinkPack.prototype.setContentBytes = function (d) {
+    try {
+        if (!d || d.length < 1) return;
+        const inStream = new DataInputX(d);
+        const ver = inStream.readByte();
+        if (ver === 1) {
+            this.content = inStream.readText();
+            this.line = inStream.readDecimal();
+        }
+    } catch (e) {
+        // Handle error
+    }
+};
+
+LogSinkPack.prototype.write = function (dout) {
+    Pack.prototype.write.call(this, dout);
+    dout.writeByte(0);
+    dout.writeText(this.category);
+    dout.writeDecimal(this.tagHash);
+    dout.writeValue(this.tags);
+    dout.writeDecimal(this.line);
+    dout.writeText(this.content);
+    if (this.fields.size() > 0) {
+        dout.writeBoolean(true);
+        dout.writeValue(this.fields);
+    } else {
+        dout.writeBoolean(false);
+    }
+};
+
+LogSinkPack.prototype.read = function (din) {
+    Pack.prototype.read.call(this, din);
+    const ver = din.readByte();
+    this.category = din.readText();
+    this.tagHash = din.readDecimal();
+    this.tags = din.readValue();
+    this.line = din.readDecimal();
+    this.content = din.readText();
+    if (din.readBoolean()) {
+        this.fields = din.readValue();
+    }
+    return this;
+};
+
+LogSinkPack.prototype.resetTagHash = function () {
+    const out = new DataOutputX();
+    out.writeValue(this.tags);
+    const tagBytes = out.toByteArray();
+    this.tagHash = HashUtil.hash64(tagBytes);
+    return tagBytes;
+};
+
+LogSinkPack.prototype.transferOidToTag = function () {
+    if (this.oid !== 0 && !this.tags.containsKey('oid')) {
+        this.tags.putValue('oid', this.oid);
+        this.tagHash = 0;
+    }
+    if (this.okind !== 0 && !this.tags.containsKey('okind')) {
+        this.tags.putValue('okind', this.okind);
+        this.tagHash = 0;
+    }
+    if (this.onode !== 0 && !this.tags.containsKey('onode')) {
+        this.tags.putValue('onode', this.onode);
+        this.tagHash = 0;
+    }
+};
+
+module.exports = LogSinkPack;

package/lib/pack/packenum.js

@@ -32,6 +32,7 @@ exports.HITMAP5M    = 0x1700;
 
 exports.TAG_COUNT   = 0x1601;
 exports.ZIP         = 0x170b;
+exports.LOGSINK     = 0x170a;
 
 exports.PACK_NAME = {
     0x0100 : 'PARAMETER',
@@ -58,6 +59,7 @@ exports.PACK_NAME = {
     0x1600 : 'COUNTER5M',
     0x1700 : 'HITMAP5M',
     0x1601 : 'TAG_COUNT',
-    0x170b : 'ZIP'
+    0x170b : 'ZIP',
+    0x170a : 'LOGSINK',
 };
  

package/lib/pack/zip-pack.js

@@ -10,7 +10,6 @@ var PackEnum = require('./packenum'),
     Step = require('../step/step'),
     DataOutputX = require('../io/data-outputx'),
     DataInputX = require('../io/data-inputx');
-var zlib = require('zlib');
 
 function ZipPack() {
     Pack.call(this);

package/lib/topology/link.js

@@ -0,0 +1,63 @@
+var { InetAddress } = require('net');
+var CompareUtil = require('../util/compare-util');
+var HashUtil = require('../util/hashutil');
+
+function LINK() {
+    this.ip = null;
+    this.port = 0;
+
+    this.toString = function () {
+        if (this.ip === null) return `0.0.0.0:${this.port}`;
+        try {
+            return `${InetAddress.getByAddress(this.ip).getHostAddress()}:${this.port}`;
+        } catch (e) {
+            return `0.0.0.0:${this.port}`;
+        }
+    };
+
+    this.include = function (k) {
+        if (!CompareUtil.equals(this.ip, k.ip)) return false;
+        if (this.port === 0) return true;
+        return this.port === k.port;
+    };
+
+    this.equals = function (o) {
+        if (o.constructor !== this.constructor) return false;
+        var k = o;
+        if (!CompareUtil.equals(this.ip, k.ip)) return false;
+        return this.port === k.port;
+    };
+
+    this.hashCode = function () {
+        return HashUtil.hash(this.ip) | this.port;
+    };
+
+    this.toBytes = function (out) {
+        out.writeBlob(this.ip);
+        out.writeInt(this.port);
+        return this;
+    };
+
+    this.toObject = function (din) {
+        this.ip = din.readBlob();
+        this.port = din.readInt();
+        return this;
+    };
+}
+
+LINK.create = function (ipStr, port) {
+    var k = new LINK();
+    try {
+        k.ip = InetAddress.getByName(ipStr).getAddress();
+    } catch (e) {
+        k.ip = new Buffer(4);
+    }
+    k.port = port;
+    return k;
+};
+
+LINK.prototype = new LINK();
+LINK.prototype.constructor = LINK;
+
+module.exports = LINK;
+

package/lib/topology/nodeinfo.js

@@ -0,0 +1,123 @@
+const DataOutputX = require('../io/data-outputx');
+const DataInputX = require('../io/data-inputx');
+const MapValue = require('../value/map-value');
+const LINK = require('./link');
+const net = require('net');
+
+function NodeInfo() {
+    this.attr = new MapValue();
+    this.listen = new Set();
+    this.outter = new Set();
+
+    this.addListen = function (localIPs, address) {
+        const ipo = this.getIPPORT(address);
+        if (!ipo || this.isLocal127(ipo)) return;
+
+        if (ipo.ip === '*' || ipo.ip === '0.0.0.0' || ipo.ip === '::') {
+            localIPs.forEach(local_ip => {
+                const k = LINK.create(local_ip, parseInt(ipo.port, 10));
+                if (k) this.listen.add(k);
+            });
+        } else {
+            const k = LINK.create(ipo.ip, parseInt(ipo.port, 10));
+            if (k) this.listen.add(k);
+        }
+    };
+
+    this.addOutter = function (local, remote) {
+        const localIPO = this.getIPPORT(local);
+        if (!localIPO || this.isIPv6(localIPO)) return;
+        if (this.hasListen(localIPO.ip, localIPO.port)) return;
+
+        const remoteIPO = this.getIPPORT(remote);
+        if (!remoteIPO || this.isIPv6(remoteIPO) || this.isLocal127(remoteIPO)) return;
+
+        const k = LINK.create(remoteIPO.ip, parseInt(remoteIPO.port, 10));
+        if (k) this.outter.add(k);
+    };
+
+    this.hasListen = function (ip, port) {
+        const k = LINK.create(ip, parseInt(port, 10));
+        return this.listen.has(k);
+    };
+
+    this.getIPPORT = function (address) {
+        try {
+            const ipo = {};
+            const x = address.lastIndexOf(':');
+            ipo.ip = address.substring(0, x);
+            ipo.port = address.substring(x + 1);
+            return ipo;
+        } catch (e) {
+            return null;
+        }
+    };
+
+    this.toBytes = function () {
+        const out = new DataOutputX();
+        out.writeByte(0);
+        out.writeValue(this.attr);
+        this.toLinkBytes(this.listen, out);
+        this.toLinkBytes(this.outter, out);
+        return out.toByteArray();
+    };
+
+    this.toObject = function (b) {
+        const din = new DataInputX(b);
+        const ver = din.readByte();
+        this.attr = din.readValue();
+        this.listen = this.toLinkObject(din);
+        this.outter = this.toLinkObject(din);
+        return this;
+    };
+
+    this.toLinkObject = function (din) {
+        const data = new Set();
+        const sz = din.readDecimal();
+        for (let i = 0; i < sz; i++) {
+            data.add(new LINK().toObject(din));
+        }
+        return data;
+    };
+
+    this.toLinkBytes = function (data, out) {
+        out.writeDecimal(data.size);
+        if (data.size === 0) return;
+        data.forEach(k => k.toBytes(out));
+    };
+
+    this.toJSON = function () {
+        const o = new Map();
+        o.put('attr', JSON.stringify(this.attr));
+        o.put('listen', JSON.parse(this.listen));
+        o.put('outter', JSON.parse(this.outter));
+        return o;
+    };
+
+    this.toJSON = function (data) {
+        const out = new JSONArray();
+        data.forEach(k => out.put(k.toString()));
+        return out;
+    };
+
+    this.isIPv6 = function(data) {
+        var ip = data.ip;
+        if(ip){
+            return net.isIPv6(ip);
+        }
+        return false;
+    };
+
+    this.isLocal127 = function (data) {
+        var ip = data.ip;
+        if(ip){
+            return ip === '127.0.0.1';
+        }
+        return false;
+    }
+}
+
+NodeInfo.prototype = new NodeInfo();
+NodeInfo.prototype.constructor = NodeInfo;
+
+module.exports = NodeInfo;

package/lib/topology/status-detector.js

@@ -0,0 +1,111 @@
+var { exec } = require('child_process');
+var os = require('os');
+var NodeInfo = require('./nodeinfo');
+var Logger = require('../logger');
+
+function StatusDetector() {
+    this.process = async function () {
+        let node = new NodeInfo();
+        let stat = await this.netstat();
+
+        if (!stat) {
+            node = new NodeInfo();
+        } else {
+            try {
+                node = this.parse(stat);
+            } catch (e) {
+                node = new NodeInfo();
+                Logger.printError("WHATAP-203", "NodeInfo parse error", e, false);
+            }
+        }
+
+        node.attr.type = 'nodejs';
+        node.attr.time = Date.now();
+        node.attr.ip = this.getLocalIpAddress();
+        node.attr.pid = process.pid;
+        node.attr.pname = process.title;
+
+        return node;
+    };
+
+    this.netstat = function () {
+        return new Promise((resolve, reject) => {
+            let cmd = 'netstat -an -t';
+            if (os.platform() === 'darwin') {
+                cmd = 'netstat -an -p tcp';
+            } else if (os.platform() === 'win32') {
+                resolve('');
+                return;
+            }
+
+            exec(cmd, (error, stdout, stderr) => {
+                if (error) {
+                    reject(error);
+                    return;
+                }
+                resolve(stdout);
+            });
+        });
+    };
+
+    this.parse = function (netstat) {
+        var node = new NodeInfo();
+        var localIPs = this.getLocalIpSet();
+
+        var lines = netstat.split('\n');
+        lines.forEach(line => {
+            if (line.startsWith('tcp')) {
+                this.parseLine(node, line, localIPs);
+            }
+        });
+
+        return node;
+    };
+
+    this.parseLine = function (node, line, localIPs) {
+        var tokens = line.trim().split(/\s+/);
+        if (!tokens[0].startsWith('tcp')) return;
+
+        var localAddress = tokens[3].replace(/(\d+\.\d+\.\d+\.\d+)\.(\d+)/, '$1:$2');
+        var remoteAddress = tokens[4].replace(/(\d+\.\d+\.\d+\.\d+)\.(\d+)/, '$1:$2');
+
+        if (tokens[5] === 'LISTEN') {
+            node.addListen(localIPs, localAddress);
+        } else {
+            node.addOutter(localAddress, remoteAddress);
+        }
+    };
+
+    this.getLocalIpSet = function () {
+        var ipSet = new Set();
+        var interfaces = os.networkInterfaces();
+        for (var devName in interfaces) {
+            var iface = interfaces[devName];
+            iface.forEach(alias => {
+                if (alias.family === 'IPv4' && !alias.internal) {
+                    ipSet.add(alias.address);
+                }
+            });
+        }
+        return ipSet;
+    };
+
+    this.getLocalIpAddress = function () {
+        var interfaces = os.networkInterfaces();
+        for (var devName in interfaces) {
+            var iface = interfaces[devName];
+            for (let i = 0; i < iface.length; i++) {
+                var alias = iface[i];
+                if (alias.family === 'IPv4' && !alias.internal) {
+                    return alias.address;
+                }
+            }
+        }
+        return '0.0.0.0';
+    };
+}
+
+StatusDetector.prototype = new StatusDetector();
+StatusDetector.prototype.constructor = StatusDetector;
+
+module.exports = StatusDetector;