whatap 1.0.1 → 1.0.2

package/lib/kube/kube-client.js

@@ -0,0 +1,144 @@
+var net             = require('net'),
+    conf            = require('../conf/configure'),
+    MapValue        = require('../value/map-value'),
+    KubeUtil        = require('../util/kube-util'),
+    StopWatch       = require('../util/stop-watch'),
+    DataOutputX     = require('../io/data-outputx'),
+    DataInputX      = require('../io/data-inputx'),
+    IPUtil          = require('../util/iputil'),
+    TcpRequestClientProxy = require('./../net/tcpreq-client-proxy'),
+    Logger          = require('../logger'),
+    SecurityMaster  = require('./../net/security-master');
+    
+const SLEEP_TIME = 15000;
+
+var KubeClient = {
+    intervalId: undefined,
+    run: function(){
+        var self = this;
+        if(this.intervalId){
+            clearInterval(this.intervalId); this.intervalId = undefined;
+        }
+        this.intervalId = setInterval(function(){
+            self.process(conf.pod_name);
+        },SLEEP_TIME);
+    },
+
+    process: function(podName){
+        var self = this;
+        var p = new MapValue();
+
+        var localAddr = (IPUtil.getIp());
+
+        SecurityMaster.IP = IPUtil.stringToInt(localAddr);
+
+		p.putString("cmd", "regist");
+		p.putLong("pcode", SecurityMaster.PCODE);
+		p.putLong("oid", SecurityMaster.OID);
+        p.putString("oname", SecurityMaster.ONAME);
+        
+		p.putString("ip", IPUtil.intToString(SecurityMaster.IP));
+		p.putString("hostname", require('os').hostname());
+        p.putBoolean("kube.micro", true);
+
+        if (podName != null) {
+			p.putString("pod_name", podName);
+		}
+		if (conf.OKIND != 0) {
+			p.putLong("okind", conf.OKIND);
+			p.putString("okind_name", conf.OKIND_NAME);
+		}
+		if (conf.ONODE != 0) {
+			p.putLong("onode", conf.ONODE);
+			p.putString("onode_name", conf.ONODE_NAME);
+		}
+		if (KubeUtil.container_id != null) {
+			p.putString("container_id", KubeUtil.container_id);
+        }
+    
+        this.sendToMaster(p, function(){
+            self.sendToNode(p);
+        });
+    },
+
+    sendToNode: function(p){
+        var self = this;
+
+        try{
+            var client = new TcpRequestClientProxy("Node " + self.node_agent_ip+ ":"+self.node_agent_port);
+            client.open( self.node_agent_ip , self.node_agent_port, function(err, data){
+                client.send(new DataOutputX().writeValue(p));
+            }, function(err, data){
+                if(err || !data){
+                    return console.log("#### errordata" , err, data);
+                }
+                var m = new DataInputX(data).readValue();
+
+                self.cpu = m.getFloat("cpu");
+                self.cpu_sys = m.getFloat("cpu_sys");
+                self.cpu_user = m.getFloat("cpu_user");
+                self.throttled_periods = m.getFloat("throttled_periods");
+                self.throttled_time = m.getFloat("throttled_time");
+
+                self.memory = m.getLong("memory");
+                self.failcnt = m.getLong("failcnt");
+                self.limit = m.getLong("limit");
+                self.maxUsage = m.getLong("maxUsage");
+
+                self.node_recv_time = Date.now();
+                self.metering = m.getFloat("metering");
+            });
+        }catch(e){
+            return Logger.printError("WHATAP-128", 'Failed to connect to kubernetes Node!')
+        }
+    },
+
+    sendToMaster: function(p, cb){
+        var self = this;
+        if (conf.master_agent_host == null) {
+            return Logger.printError("WHATAP-129", 'master_agent_host=null', e)
+		}
+
+        var connectOptions = {
+            host: 
+            conf.master_agent_host
+            // 'localhost'
+            , port: conf.master_agent_port, agent: false
+        };
+
+        try{
+            self.auto_oname_reset = conf.auto_oname_reset;
+
+            var client = new TcpRequestClientProxy("Master " + connectOptions.host+ ":"+connectOptions.port);
+            client.open( connectOptions.host , connectOptions.port, function(err, data){
+                client.send(new DataOutputX().writeValue(p));
+            }, function(err, data){
+                if(data === undefined){
+                    return Logger.printError('WHATAP-802', 'Are You Sure Kubernetes Container ??? (current whatap_micro_enabled=true)', new Error('No Master Whatap'), true);
+                }
+                
+                var m = new DataInputX(data).readValue();
+                self.node_agent_ip = m.getText("node.agent.ip");
+                self.node_agent_port = m.getInt("node.agent.port");
+                if(cb) cb();
+            } );
+        }catch(e){
+            return Logger.printError("WHATAP-128", 'Failed to connect to kubernetes Master!', e)
+        }
+    },
+
+    recvNodeAddr: function(m){
+        var host = m['node.agent.ip'];
+        var port = m['node.agent.port'];
+
+        if(!host){
+            this.node_agent_host = host;
+        }
+
+        if(!port && port !== 0){
+            this.node_agent_port = port;
+        }
+    }
+}
+
+module.exports = KubeClient;

package/lib/lang/text-types.js

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+exports.SERVICE = 1;
+exports.SQL = 2;
+exports.DB_URL = 3;
+exports.HTTPC_URL = 4;
+exports.ERROR = 5;
+exports.METHOD = 6;
+exports.STACK_ELEMENTS = 7;
+exports.REFERER = 8;
+exports.USER_AGENT = 9;
+exports.HTTPC_HOST = 10;
+exports.MESSAGE = 11;
+exports.CRUD = 12;
+exports.ONAME = 13;
+exports.COMMAND = 14;
+exports.USER_AGENT_OS = 15;
+exports.USER_AGENT_BROWSER = 16;
+exports.CITY = 17;
+exports.LOGIN = 18;
+exports.SQLPARAM = 19;
+exports.HTTP_DOMAIN = 20;
+
+exports.SYS_DEVICE_ID = 21;
+exports.SYS_MOUNT_POINT = 22;
+exports.SYS_FILE_SYSTEM = 23;
+exports.SYS_NET_DESC = 24;
+exports.SYS_PROC_CMD1 = 26;
+exports.SYS_PROC_CMD2 = 27;
+exports.SYS_PROC_USER = 28;
+exports.SYS_PROC_STATE = 29;
+exports.SYS_PROC_FILENAME = 30;
+exports.SM_LOG_FILE = 31;
+exports.EXT_META = 32;
+
+exports.DB_COUNTER_NAME = 41;
+exports.DB_COUNTER_UNIT = 42;
+
+exports.CW_AGENT_IP = 51;
+exports.CW_MXID = 52;
+
+exports.MTRACE_SPEC = 53;
+
+exports.MTRACE_CALLER_URL = 54;
+exports.OKIND_NAME = 57;
+exports.KUBE_COUNT_NAME = 59;
+
+exports.CONTAINER_ID= 60;
+exports.PODNAME= 61;
+exports.CONTAINER_IMAGE =62;
+exports.ONODE_NAME = 63;
+exports.CONTAINER_COMMAND=64;
+exports.REPLICASETNAME = 65;
+exports.NAMESPACE = 66;

package/lib/logger.js

@@ -213,23 +213,23 @@
              for (var i = 0; i < files.length; i++) {
                  var stat = fs.statSync(path.join(dir, files[i]));
                  if (stat.isDirectory()) {
-                     return true;
+                     continue;
                  }
                  var name = files[i];
                  if (name.indexOf(log_prefix) < 0) {
-                     return true;
+                     continue;
                  }
                  var x = name.lastIndexOf('.');
                  if (x < 0) {
-                     return true;
+                     continue;
                  }
                  var date = name.substr(log_prefix.length, (x - log_prefix.length));
                  if (date.length != 8) {
-                     return true;
+                     continue;
                  }
-                 var d = DateUtil.yyyymmdd(date);
-                 var fileUnit = DateUtil.getDateUnit(d);
                  try {
+                     var d = DateUtil.yyyymmdd(date);
+                     var fileUnit = DateUtil.getDateUnit(d);
                      if (nowUnit - fileUnit > conf.log_keep_days) {
                          let filePath = `${dir}/${files[i]}`;
                          fs.rmSync(filePath);

package/lib/logsink/line-log-util.js

@@ -0,0 +1,87 @@
+const conf = require('../conf/configure');
+const EventPack = require('../pack/event-pack');
+const DataPackSender = require('../data/datapack-sender');
+const Logger = require('../logger');
+const EventLevel = require('../data/event-level');
+
+class LineLogUtil {
+    constructor() {
+        this.last_log = Date.now();
+        this.last_alert = Date.now();
+    }
+
+    static getInstance() {
+        if (!LineLogUtil.instance) {
+            LineLogUtil.instance = new LineLogUtil();
+        }
+        return LineLogUtil.instance;
+    }
+
+    checkLogContent(lineLog, orgContent) {
+        if (!orgContent) {
+            return;
+        }
+
+        const len = orgContent.length;
+        let maxLength = conf.getProperty('logsink_limit_content_length', 10000);
+        if(maxLength > 10000){
+            maxLength = 10000;
+        }
+
+        if ((conf.getProperty('logsink_limit_content_enabled', true) && len > maxLength) || len > 10000) {
+            lineLog.content = orgContent.substring(0, maxLength) + " ...(truncated)";
+            lineLog.truncated = true;
+
+            this.addTruncateTag(lineLog);
+
+            const alertMessage = lineLog.category + " truncated (len=" + len + ")";
+            this.logAndAlert(alertMessage);
+        } else {
+            lineLog.content = orgContent;
+        }
+    }
+
+    log(message) {
+        if (!conf.getProperty('logsink_max_content_log_enabled', true)) {
+            return;
+        }
+
+        const now = Date.now();
+        if (now - this.last_log < conf.getProperty('logsink_max_content_log_silent_time', 60000)) {
+            return;
+        }
+        this.last_log = now;
+        Logger.print('WHATAP-101', message, false);
+    }
+
+    alertOverflow(message) {
+        if (!conf.getProperty('logsink_max_content_alert_enabled', false)) {
+            return;
+        }
+
+        const now = Date.now();
+        if (now - this.last_alert < conf.getProperty('logsink_max_content_alert_silent_time', 60000)) {
+            return;
+        }
+        this.last_alert = now;
+
+        const eventPack = new EventPack();
+        eventPack.level = EventLevel.FATAL;
+        eventPack.title = "LOG_TRUNCATED";
+        eventPack.message = "Too big Log: " + message;
+        DataPackSender.sendEventPack(eventPack);
+    }
+
+    logAndAlert(alertMessage) {
+        this.log(alertMessage);
+        this.alertOverflow(alertMessage);
+    }
+
+    addTruncateTag(lineLog) {
+        lineLog.tags.putString("truncated", "true");
+    }
+}
+
+LineLogUtil.instance = null;
+
+module.exports = LineLogUtil;

package/lib/logsink/line-log.js

@@ -0,0 +1,12 @@
+const MapValue = require('../value/map-value');
+
+var LineLog = function () {
+    this.truncated;
+    this.time;
+    this.category;
+    this.tags = new MapValue();
+    this.fields = new MapValue();
+    this.content;
+}
+
+module.exports = LineLog;

package/lib/logsink/log-sender.js

@@ -0,0 +1,78 @@
+var Configure = require('../conf/configure');
+var RequestQueue = require('../util/request-queue');
+var Logger = require('../logger');
+var SenderUtil = require('./sender-util');
+var conf = require("../conf/configure");
+
+var logsink_enabled = Configure.getProperty('logsink_enabled', false);
+var logsink_queue_size = Configure.getProperty('logsink_queue_size', 1000);
+conf.on('logsink_enabled', function(newProperty) {
+    logsink_enabled = newProperty;
+    if (logsink_enabled) {
+        LogSender.getInstance().startProcessQueue();
+    } else {
+        LogSender.getInstance().stopProcessQueue();
+        LogSender.resetInstance();
+    }
+});
+
+class LogSender {
+    constructor() {
+        if (LogSender.instance) {
+            return LogSender.instance;
+        }
+        this.queue = new RequestQueue(logsink_queue_size);
+        this.isProcessing = false;
+        LogSender.instance = this;
+
+        if (logsink_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!LogSender.instance) {
+            LogSender.instance = new LogSender();
+        }
+        return LogSender.instance;
+    }
+
+    static resetInstance() {
+        if(LogSender.instance){
+            LogSender.instance = null;
+        }
+    }
+
+    add(log) {
+        this.queue.put(log);
+    }
+
+    async processQueue() {
+        this.isProcessing = true;
+
+        while (this.isProcessing) {
+            var log = await this.queue.get();
+            if(log){
+                try {
+                    var sendUtil = new SenderUtil();
+                    sendUtil.send(log);
+                } catch (error) {
+                    Logger.printError('WHATAP-702', 'Error occurred during send LogSinkPack.' + error, false);
+                }
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    startProcessQueue() {
+        if (!this.isProcessing) {
+            this.processQueue();
+        }
+    }
+
+    stopProcessQueue() {
+        this.isProcessing = false;
+    }
+}
+
+module.exports = LogSender;

package/lib/logsink/log-tracer.js

@@ -0,0 +1,40 @@
+var conf = require('../conf/configure');
+var LineLog = require('./line-log');
+var LineLogUtil = require('./line-log-util');
+var LogSender = require('./log-sender');
+var TraceContextManager = require('../trace/trace-context-manager');
+
+var LogTracer = function () {
+    this.logSender = LogSender.getInstance();
+}
+
+LogTracer.prototype.addStdWrite = function (content, category) {
+    var lineLog = new LineLog();
+    lineLog.category = category
+
+    var lineLogUtil = LineLogUtil.getInstance();
+    lineLogUtil.checkLogContent(lineLog, content);
+
+    if(lineLog.content){
+        this.addTxTag(lineLog);
+        this.logSender.add(lineLog)
+    }
+}
+
+LogTracer.prototype.addTxTag = function (lineLog) {
+    if(!lineLog){
+        return;
+    }
+
+    var ctx = TraceContextManager.getCurrentContext();
+    if(ctx){
+        if(conf.getProperty('logsink_trace_txid_enabled', true) && !ctx.txid.isZero()){
+            lineLog.fields.putLong('@txid', ctx.txid);
+        }
+        if(conf.getProperty('logsink_trace_mtid_enabled', true) && !ctx.mtid.isZero()){
+            lineLog.fields.putLong('@mtid', ctx.mtid);
+        }
+    }
+}
+
+module.exports = LogTracer;

package/lib/logsink/sender-util.js

@@ -0,0 +1,56 @@
+var conf = require('../conf/configure');
+var SecurityMaster      = require('../net/security-master');
+var LogSinkPack = require('../pack/log-sink-pack');
+var DataPackSender = require('../data/datapack-sender');
+var DateUtil = require('../util/dateutil')
+var ZipSend = require('./zip/zip-send');
+var PiiMask = require('../pii/pii-mask')
+
+function SenderUtil() {
+    this.conf = conf;
+    this.secu = SecurityMaster;
+}
+
+SenderUtil.prototype.send = function (log) {
+    const p = new LogSinkPack();
+    p.time = log.time;
+    if (!p.time) {
+        p.time = DateUtil.currentTime();
+    }
+    p.category = log.category;
+
+    if (this.secu.ONAME) {
+        p.tags.putString('oname', this.secu.ONAME);
+    }
+    if (this.conf.OKIND) {
+        p.tags.putString('okindName', this.conf.OKIND_NAME);
+    }
+    if (this.conf.ONODE) {
+        p.tags.putString('onodeName', this.conf.ONODE_NAME);
+    }
+
+    if (log.tags != null) {
+        p.tags.putAllMapValue(log.tags);
+    }
+    if (log.fields != null) {
+        if (p.fields == null) {
+            p.fields = log.fields;
+        } else {
+            p.fields.putAllMapValue(log.fields);
+        }
+    }
+
+    p.content = log.content;
+
+    if(conf.getProperty("pii_mask_logsink_enabled", false)){
+        // PiiMask.parseLogSink(p)
+    }
+
+    if(conf.getProperty('logsink_zip_enabled', false)){
+        ZipSend.getInstance().add(p);
+    }else{
+        DataPackSender.sendLogSinkPack(p)
+    }
+};
+
+module.exports = SenderUtil;

package/lib/logsink/zip/zip-send.js

@@ -0,0 +1,177 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var conf            = require('../../conf/configure'),
+    DateUtil        = require('../../util/dateutil'),
+    RequestQueue    = require('../../util/request-queue'),
+    DataPackSender = require('../../data/datapack-sender'),
+    DataOutputX = require('../../io/data-outputx'),
+    ZipPack = require('../../pack/zip-pack'),
+    Logger          = require('../../logger');
+var zlib = require('zlib');
+
+var logsink_enabled = conf.getProperty('logsink_enabled', false);
+var logsink_zip_enabled = conf.getProperty('logsink_zip_enabled', false);
+var logsink_max_buffer_size = conf.getProperty('logsink_max_buffer_size', 1024*64);
+var profile_zip_min_size = conf.getProperty('profile_zip_min_size', 100)
+var logsink_queue_size = conf.getProperty('logsink_queue_size', 500);
+var logsink_max_wait_time = conf.getProperty('logsink_max_wait_time', 2000);
+conf.on('logsink_enabled', function(newProperty) {
+    logsink_enabled = newProperty;
+    if (logsink_enabled && logsink_zip_enabled) {
+        ZipSend.getInstance().startProcessQueue();
+    } else {
+        ZipSend.getInstance().stopProcessQueue();
+        ZipSend.resetInstance();
+    }
+});
+conf.on('logsink_zip_enabled', function(newProperty) {
+    logsink_zip_enabled = newProperty;
+    if (logsink_enabled && logsink_zip_enabled) {
+        ZipSend.getInstance().startProcessQueue();
+    } else {
+        ZipSend.getInstance().stopProcessQueue();
+        ZipSend.resetInstance();
+    }
+});
+
+class ZipSend {
+    constructor() {
+        if (ZipSend.instance) {
+            return ZipSend.instance;
+        }
+        this.queue = new RequestQueue(logsink_queue_size);
+        this.buffer = Buffer.alloc(0);
+        this.packCount = 0;
+        this.first_time = null;
+        this.isProcessing = false;
+        ZipSend.instance = this;
+
+        if (logsink_enabled && logsink_zip_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!ZipSend.instance) {
+            ZipSend.instance = new ZipSend();
+        }
+        return ZipSend.instance;
+    }
+
+    static resetInstance() {
+        if(ZipSend.instance){
+            ZipSend.instance = null;
+        }
+    }
+
+    add(p){
+        var ok = this.queue.put(p);
+        if(!ok){
+            DataPackSender.sendLogSinkPack(p);
+        }
+    }
+
+    // async addWait(p, waitTimeForFull) {
+    //     var ok = this.queue.put(p);
+    //     if (ok === false){
+    //         if(waitTimeForFull > 0){
+    //             while(this.queue.put(p) === false){
+    //                 await new Promise(resolve => setTimeout(resolve, waitTimeForFull));
+    //             }
+    //         }
+    //     }
+    // }
+
+    async processQueue() {
+        this.isProcessing = true;
+        while (this.isProcessing) {
+            if(!logsink_enabled || !logsink_zip_enabled) {
+                this.isProcessing = false;
+                break;
+            }
+
+            const p = await this.queue.getByTimeout(logsink_max_wait_time);
+            if(p) {
+                await this.append(p);
+            }else{
+                await this.sendAndClear();
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    async append(p) {
+        var b = DataOutputX.toBytesPack(p);
+        this.packCount++;
+        this.buffer = Buffer.concat([this.buffer, b]);
+
+        if(!this.first_time){
+            this.first_time = p.time;
+            if (this.buffer.length >= logsink_max_buffer_size) {
+                await this.sendAndClear();
+            }
+        }else{
+            if(this.buffer.length >= logsink_max_buffer_size || (p.time - this.first_time) >= conf.getProperty('logsink_max_wait_time', 2000)){
+                await this.sendAndClear();
+            }
+        }
+    }
+
+    async sendAndClear() {
+        if(this.buffer.length === 0){ return; }
+
+        var p = new ZipPack();
+        p.time = DateUtil.currentTime();
+        p.recordCount = this.packCount;
+        p.records = this.buffer;
+
+        await this.doZip(p);
+        if(conf.getProperty('debug_logsink_zip_enabled', false)){
+            Logger.print('WHATAP-703', `zip status=${p.status} records=${p.recordCount} | ${this.buffer.length} => ${p.records.length}`);
+        }
+        DataPackSender.sendLogSinkPack(p);
+
+        this.buffer = Buffer.alloc(0);
+        this.first_time = 0;
+        this.packCount = 0;
+    }
+
+    async doZip(p){
+        if(p.status !== 0){ return; }
+        if(p.records.length < profile_zip_min_size){ return; }
+        p.status = 1;
+
+        try{
+            p.records = await this._doZip(p.records);
+        }catch (error) {
+            Logger.print('WHATAP-701', 'Error occurred during compression.' + error, false);
+        }
+    }
+
+    _doZip(data) {
+        return new Promise((resolve, reject) => {
+            zlib.gzip(data, (err, buffer) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(buffer);
+                }
+            });
+        });
+    }
+
+    startProcessQueue() {
+        if (!this.isProcessing) {
+            this.processQueue()
+        }
+    }
+    stopProcessQueue() {
+        this.isProcessing = false;
+    }
+}
+
+module.exports = ZipSend;