whatap 0.5.26 → 1.0.0-canary.0

package/lib/logsink/line-log-util.js

@@ -1,87 +0,0 @@
-const conf = require('../conf/configure');
-const EventPack = require('../pack/event-pack');
-const DataPackSender = require('../data/datapack-sender');
-const Logger = require('../logger');
-const EventLevel = require('../data/event-level');
-
-class LineLogUtil {
-    constructor() {
-        this.last_log = Date.now();
-        this.last_alert = Date.now();
-    }
-
-    static getInstance() {
-        if (!LineLogUtil.instance) {
-            LineLogUtil.instance = new LineLogUtil();
-        }
-        return LineLogUtil.instance;
-    }
-
-    checkLogContent(lineLog, orgContent) {
-        if (!orgContent) {
-            return;
-        }
-
-        const len = orgContent.length;
-        let maxLength = conf.getProperty('logsink_limit_content_length', 10000);
-        if(maxLength > 10000){
-            maxLength = 10000;
-        }
-
-        if ((conf.getProperty('logsink_limit_content_enabled', true) && len > maxLength) || len > 10000) {
-            lineLog.content = orgContent.substring(0, maxLength) + " ...(truncated)";
-            lineLog.truncated = true;
-
-            this.addTruncateTag(lineLog);
-
-            const alertMessage = lineLog.category + " truncated (len=" + len + ")";
-            this.logAndAlert(alertMessage);
-        } else {
-            lineLog.content = orgContent;
-        }
-    }
-
-    log(message) {
-        if (!conf.getProperty('logsink_max_content_log_enabled', true)) {
-            return;
-        }
-
-        const now = Date.now();
-        if (now - this.last_log < conf.getProperty('logsink_max_content_log_silent_time', 60000)) {
-            return;
-        }
-        this.last_log = now;
-        Logger.print('WHATAP-101', message, false);
-    }
-
-    alertOverflow(message) {
-        if (!conf.getProperty('logsink_max_content_alert_enabled', false)) {
-            return;
-        }
-
-        const now = Date.now();
-        if (now - this.last_alert < conf.getProperty('logsink_max_content_alert_silent_time', 60000)) {
-            return;
-        }
-        this.last_alert = now;
-
-        const eventPack = new EventPack();
-        eventPack.level = EventLevel.FATAL;
-        eventPack.title = "LOG_TRUNCATED";
-        eventPack.message = "Too big Log: " + message;
-        DataPackSender.sendEventPack(eventPack);
-    }
-
-    logAndAlert(alertMessage) {
-        this.log(alertMessage);
-        this.alertOverflow(alertMessage);
-    }
-
-    addTruncateTag(lineLog) {
-        lineLog.tags.putString("truncated", "true");
-    }
-}
-
-LineLogUtil.instance = null;
-
-module.exports = LineLogUtil;

package/lib/logsink/line-log.js

@@ -1,12 +0,0 @@
-const MapValue = require('../value/map-value');
-
-var LineLog = function () {
-    this.truncated;
-    this.time;
-    this.category;
-    this.tags = new MapValue();
-    this.fields = new MapValue();
-    this.content;
-}
-
-module.exports = LineLog;

package/lib/logsink/log-sender.js

@@ -1,78 +0,0 @@
-var Configure = require('../conf/configure');
-var RequestQueue = require('../util/request-queue');
-var Logger = require('../logger');
-var SenderUtil = require('./sender-util');
-var conf = require("../conf/configure");
-
-var logsink_enabled = Configure.getProperty('logsink_enabled', false);
-var logsink_queue_size = Configure.getProperty('logsink_queue_size', 1000);
-conf.on('logsink_enabled', function(newProperty) {
-    logsink_enabled = newProperty;
-    if (logsink_enabled) {
-        LogSender.getInstance().startProcessQueue();
-    } else {
-        LogSender.getInstance().stopProcessQueue();
-        LogSender.resetInstance();
-    }
-});
-
-class LogSender {
-    constructor() {
-        if (LogSender.instance) {
-            return LogSender.instance;
-        }
-        this.queue = new RequestQueue(logsink_queue_size);
-        this.isProcessing = false;
-        LogSender.instance = this;
-
-        if (logsink_enabled) {
-            this.startProcessQueue();
-        }
-    }
-
-    static getInstance() {
-        if (!LogSender.instance) {
-            LogSender.instance = new LogSender();
-        }
-        return LogSender.instance;
-    }
-
-    static resetInstance() {
-        if(LogSender.instance){
-            LogSender.instance = null;
-        }
-    }
-
-    add(log) {
-        this.queue.put(log);
-    }
-
-    async processQueue() {
-        this.isProcessing = true;
-
-        while (this.isProcessing) {
-            var log = await this.queue.get();
-            if(log){
-                try {
-                    var sendUtil = new SenderUtil();
-                    sendUtil.send(log);
-                } catch (error) {
-                    Logger.printError('WHATAP-702', 'Error occurred during send LogSinkPack.' + error, false);
-                }
-            }
-            await new Promise(resolve => setTimeout(resolve, 100));
-        }
-    }
-
-    startProcessQueue() {
-        if (!this.isProcessing) {
-            this.processQueue();
-        }
-    }
-
-    stopProcessQueue() {
-        this.isProcessing = false;
-    }
-}
-
-module.exports = LogSender;

package/lib/logsink/log-tracer.js

@@ -1,40 +0,0 @@
-var conf = require('../conf/configure');
-var LineLog = require('./line-log');
-var LineLogUtil = require('./line-log-util');
-var LogSender = require('./log-sender');
-var TraceContextManager = require('../trace/trace-context-manager');
-
-var LogTracer = function () {
-    this.logSender = LogSender.getInstance();
-}
-
-LogTracer.prototype.addStdWrite = function (content, category) {
-    var lineLog = new LineLog();
-    lineLog.category = category
-
-    var lineLogUtil = LineLogUtil.getInstance();
-    lineLogUtil.checkLogContent(lineLog, content);
-
-    if(lineLog.content){
-        this.addTxTag(lineLog);
-        this.logSender.add(lineLog)
-    }
-}
-
-LogTracer.prototype.addTxTag = function (lineLog) {
-    if(!lineLog){
-        return;
-    }
-
-    var ctx = TraceContextManager.getCurrentContext();
-    if(ctx){
-        if(conf.getProperty('logsink_trace_txid_enabled', true) && !ctx.txid.isZero()){
-            lineLog.fields.putLong('@txid', ctx.txid);
-        }
-        if(conf.getProperty('logsink_trace_mtid_enabled', true) && !ctx.mtid.isZero()){
-            lineLog.fields.putLong('@mtid', ctx.mtid);
-        }
-    }
-}
-
-module.exports = LogTracer;

package/lib/logsink/sender-util.js

@@ -1,56 +0,0 @@
-var conf = require('../conf/configure');
-var SecurityMaster      = require('../net/security-master');
-var LogSinkPack = require('../pack/log-sink-pack');
-var DataPackSender = require('../data/datapack-sender');
-var DateUtil = require('../util/dateutil')
-var ZipSend = require('./zip/zip-send');
-var PiiMask = require('../pii/pii-mask')
-
-function SenderUtil() {
-    this.conf = conf;
-    this.secu = SecurityMaster;
-}
-
-SenderUtil.prototype.send = function (log) {
-    const p = new LogSinkPack();
-    p.time = log.time;
-    if (!p.time) {
-        p.time = DateUtil.currentTime();
-    }
-    p.category = log.category;
-
-    if (this.secu.ONAME) {
-        p.tags.putString('oname', this.secu.ONAME);
-    }
-    if (this.conf.OKIND) {
-        p.tags.putString('okindName', this.conf.OKIND_NAME);
-    }
-    if (this.conf.ONODE) {
-        p.tags.putString('onodeName', this.conf.ONODE_NAME);
-    }
-
-    if (log.tags != null) {
-        p.tags.putAllMapValue(log.tags);
-    }
-    if (log.fields != null) {
-        if (p.fields == null) {
-            p.fields = log.fields;
-        } else {
-            p.fields.putAllMapValue(log.fields);
-        }
-    }
-
-    p.content = log.content;
-
-    if(conf.getProperty("pii_mask_logsink_enabled", false)){
-        // PiiMask.parseLogSink(p)
-    }
-
-    if(conf.getProperty('logsink_zip_enabled', false)){
-        ZipSend.getInstance().add(p);
-    }else{
-        DataPackSender.sendLogSinkPack(p)
-    }
-};
-
-module.exports = SenderUtil;

package/lib/logsink/zip/zip-send.js

@@ -1,177 +0,0 @@
-/**
- * Copyright 2016 the WHATAP project authors. All rights reserved.
- * Use of this source code is governed by a license that
- * can be found in the LICENSE file.
- */
-
-var conf            = require('../../conf/configure'),
-    DateUtil        = require('../../util/dateutil'),
-    RequestQueue    = require('../../util/request-queue'),
-    DataPackSender = require('../../data/datapack-sender'),
-    DataOutputX = require('../../io/data-outputx'),
-    ZipPack = require('../../pack/zip-pack'),
-    Logger          = require('../../logger');
-var zlib = require('zlib');
-
-var logsink_enabled = conf.getProperty('logsink_enabled', false);
-var logsink_zip_enabled = conf.getProperty('logsink_zip_enabled', false);
-var logsink_max_buffer_size = conf.getProperty('logsink_max_buffer_size', 1024*64);
-var profile_zip_min_size = conf.getProperty('profile_zip_min_size', 100)
-var logsink_queue_size = conf.getProperty('logsink_queue_size', 500);
-var logsink_max_wait_time = conf.getProperty('logsink_max_wait_time', 2000);
-conf.on('logsink_enabled', function(newProperty) {
-    logsink_enabled = newProperty;
-    if (logsink_enabled && logsink_zip_enabled) {
-        ZipSend.getInstance().startProcessQueue();
-    } else {
-        ZipSend.getInstance().stopProcessQueue();
-        ZipSend.resetInstance();
-    }
-});
-conf.on('logsink_zip_enabled', function(newProperty) {
-    logsink_zip_enabled = newProperty;
-    if (logsink_enabled && logsink_zip_enabled) {
-        ZipSend.getInstance().startProcessQueue();
-    } else {
-        ZipSend.getInstance().stopProcessQueue();
-        ZipSend.resetInstance();
-    }
-});
-
-class ZipSend {
-    constructor() {
-        if (ZipSend.instance) {
-            return ZipSend.instance;
-        }
-        this.queue = new RequestQueue(logsink_queue_size);
-        this.buffer = Buffer.alloc(0);
-        this.packCount = 0;
-        this.first_time = null;
-        this.isProcessing = false;
-        ZipSend.instance = this;
-
-        if (logsink_enabled && logsink_zip_enabled) {
-            this.startProcessQueue();
-        }
-    }
-
-    static getInstance() {
-        if (!ZipSend.instance) {
-            ZipSend.instance = new ZipSend();
-        }
-        return ZipSend.instance;
-    }
-
-    static resetInstance() {
-        if(ZipSend.instance){
-            ZipSend.instance = null;
-        }
-    }
-
-    add(p){
-        var ok = this.queue.put(p);
-        if(!ok){
-            DataPackSender.sendLogSinkPack(p);
-        }
-    }
-
-    // async addWait(p, waitTimeForFull) {
-    //     var ok = this.queue.put(p);
-    //     if (ok === false){
-    //         if(waitTimeForFull > 0){
-    //             while(this.queue.put(p) === false){
-    //                 await new Promise(resolve => setTimeout(resolve, waitTimeForFull));
-    //             }
-    //         }
-    //     }
-    // }
-
-    async processQueue() {
-        this.isProcessing = true;
-        while (this.isProcessing) {
-            if(!logsink_enabled || !logsink_zip_enabled) {
-                this.isProcessing = false;
-                break;
-            }
-
-            const p = await this.queue.getByTimeout(logsink_max_wait_time);
-            if(p) {
-                await this.append(p);
-            }else{
-                await this.sendAndClear();
-            }
-            await new Promise(resolve => setTimeout(resolve, 100));
-        }
-    }
-
-    async append(p) {
-        var b = DataOutputX.toBytesPack(p);
-        this.packCount++;
-        this.buffer = Buffer.concat([this.buffer, b]);
-
-        if(!this.first_time){
-            this.first_time = p.time;
-            if (this.buffer.length >= logsink_max_buffer_size) {
-                await this.sendAndClear();
-            }
-        }else{
-            if(this.buffer.length >= logsink_max_buffer_size || (p.time - this.first_time) >= conf.getProperty('logsink_max_wait_time', 2000)){
-                await this.sendAndClear();
-            }
-        }
-    }
-
-    async sendAndClear() {
-        if(this.buffer.length === 0){ return; }
-
-        var p = new ZipPack();
-        p.time = DateUtil.currentTime();
-        p.recordCount = this.packCount;
-        p.records = this.buffer;
-
-        await this.doZip(p);
-        if(conf.getProperty('debug_logsink_zip_enabled', false)){
-            Logger.print('WHATAP-703', `zip status=${p.status} records=${p.recordCount} | ${this.buffer.length} => ${p.records.length}`);
-        }
-        DataPackSender.sendLogSinkPack(p);
-
-        this.buffer = Buffer.alloc(0);
-        this.first_time = 0;
-        this.packCount = 0;
-    }
-
-    async doZip(p){
-        if(p.status !== 0){ return; }
-        if(p.records.length < profile_zip_min_size){ return; }
-        p.status = 1;
-
-        try{
-            p.records = await this._doZip(p.records);
-        }catch (error) {
-            Logger.print('WHATAP-701', 'Error occurred during compression.' + error, false);
-        }
-    }
-
-    _doZip(data) {
-        return new Promise((resolve, reject) => {
-            zlib.gzip(data, (err, buffer) => {
-                if (err) {
-                    reject(err);
-                } else {
-                    resolve(buffer);
-                }
-            });
-        });
-    }
-
-    startProcessQueue() {
-        if (!this.isProcessing) {
-            this.processQueue()
-        }
-    }
-    stopProcessQueue() {
-        this.isProcessing = false;
-    }
-}
-
-module.exports = ZipSend;

package/lib/net/netflag.js

@@ -1,55 +0,0 @@
-/**
- * Copyright 2016 the WHATAP project authors. All rights reserved.
- * Use of this source code is governed by a license that
- * can be found in the LICENSE file.
- */
-
-var NetFlag = function(flag){
-    this.flag = flag;
-}
-
-NetFlag.SECURE_HIDE = 0x0001;
-NetFlag.SECURE_CYPHER = 0x0002;
-NetFlag.RESERVED1 = 0x0004;
-NetFlag.RESERVED2 = 0x0008;
-NetFlag.RESERVED3 = 0x0010;
-NetFlag.RESERVED4 = 0x0020;
-NetFlag.RESERVED5 = 0x0040;
-NetFlag.RESERVED6 = 0x0080;
-NetFlag.TIME_SYNC = 0xfe;
-NetFlag.KEY_RESET = 0xff;
-NetFlag.PREPARE_AGENT = 0xfc;
-
-NetFlag.getSecureMask = function(code){
-    if(code === NetFlag.KEY_RESET) return 0;
-    return (code & NetFlag.SECURE_HIDE ) | ( code & NetFlag.SECURE_CYPHER);
-};
-
-NetFlag.isSecure = function(code) {
-    if(code === NetFlag.KEY_RESET) return false;
-    return (code & NetFlag.SECURE_HIDE) | (code & NetFlag.SECURE_CYPHER);
-};
-
-NetFlag.prototype.set = function(mask){
-    this.flag = (this.flag | mask);
-    return this;
-};
-
-NetFlag.prototype.isA = function(m){
-    return (this.falg & m) != 0;
-};
-
-NetFlag.prototype.getFlag = function() {
-    return this.flag;
-};
-
-module.exports.NetFlag = NetFlag;
-
-var NetSrc = {
-    AGENT_JAVA_EMBED    : 1,
-    AGENT_JAVA_WATCHER  : 2,
-    SERVER_YARD         : 3,
-    SERVER_PROXY        : 4
-};
-
-module.exports.NetSrc = NetSrc;

package/lib/net/receiver.js

@@ -1,66 +0,0 @@
-/**
- * Copyright 2016 the WHATAP project authors. All rights reserved.
- * Use of this source code is governed by a license that
- * can be found in the LICENSE file.
- */
-
-
-const conf = require('../conf/configure');
-var NetFlag         = require('./netflag').NetFlag,
-    ControlHandler  = require('./../control/control-handler'),
-    DateUtil        = require('./../util/dateutil'),
-    DataInputX      = require('../io/data-inputx'),
-    Logger          = require('../logger');
-
-function Receiver(){};
-
-Receiver.prototype.read = function(out /*TcpReturn*/){
-    try {
-        if(conf.network_debug_enabled){
-            Logger.print("[Receiver] input Data " ,out.code + " // " + out.transfer_key , false);
-        }
-        if(out.code === NetFlag.TIME_SYNC) {
-            var din = new DataInputX(out.data);
-            var prevAgentTime = din.readLong().toNumber();
-            var serverTime = din.readLong().toNumber();
-            var now = Date.now();
-            var turnaroundTime = (now - prevAgentTime);
-
-            if(turnaroundTime < 500) {
-                DateUtil.setServerTime(serverTime, 1);
-            }
-            return;
-        }
-
-        if(conf.cypher_level > 0) {
-            if(out.transfer_key != conf.TRANSFER_KEY) { return; }
-            
-            switch (NetFlag.getSecureMask(out.code)) {
-                case NetFlag.SECURE_HIDE:
-                    if(conf.cypher != null) {
-                        out.data = conf.cypher.hide(out.data);
-                    } else {
-                        return;
-                    }
-                    break;
-                case NetFlag.SECURE_CYPHER:
-                    if(conf.cypher != null) {
-                        out.data = conf.cypher.decrypt(out.data);
-                    } else {
-                        return;
-                    }
-                    break;
-                default:
-                    out.data = null;
-            }
-        }
-
-        if(out.data != null && out.data.length > 0) {
-            ControlHandler.add(new DataInputX(out.data).readPack());
-        }
-    } catch (e) {
-        Logger.printError('WHATAP-195', 'Receiver error', e);
-    }
-};
-
-module.exports = Receiver;