whatap 1.0.1-canary.2 → 1.0.2

package/lib/data/zipprofile.js

@@ -0,0 +1,197 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var conf            = require('../conf/configure'),
+    secuMaster      = require('../net/security-master'),
+    TcpRequestMgr   = require('../net/tcprequest-mgr'),
+    DateUtil        = require('./../util/dateutil'),
+    RequestQueue    = require('../util/request-queue'),
+    DataPackSender = require('../data/datapack-sender'),
+    DataOutputX = require('../io/data-outputx'),
+    ZipPack = require('../pack/zip-pack'),
+    DataPackSender = require('../data/datapack-sender'),
+    Logger          = require('../logger');
+
+const zlib = require('zlib');
+
+var profile_zip_enabled = conf.getProperty('profile_zip_enabled', false);
+var profile_zip_max_buffer_size = conf.getProperty('profile_zip_max_buffer_size', 1024*1024);
+var profile_zip_min_size = conf.getProperty('profile_zip_min_size', 100)
+var profile_zip_queue_size = conf.getProperty('profile_zip_queue_size', 500);
+var profile_zip_max_wait_time = conf.getProperty('profile_zip_max_wait_time', 1000);
+var net_send_max_bytes = conf.getProperty('net_send_max_bytes', 5 * 1024 * 1024);
+var debug_profile_zip_enabled = conf.getProperty('debug_profile_zip_enabled', false);
+var debug_profile_zip_interval = conf.getProperty('debug_profile_zip_interval', 5000);
+conf.on('profile_zip_enabled', function(newProperty) {
+    profile_zip_enabled = newProperty;
+    if (profile_zip_enabled) {
+        ZipProfile.getInstance().startProcessQueue();
+    } else {
+        ZipProfile.getInstance().stopProcessQueue();
+        ZipProfile.resetInstance();
+    }
+});
+
+class ZipProfile {
+    constructor() {
+        if (ZipProfile.instance) {
+            return ZipProfile.instance;
+        }
+        this.queue = new RequestQueue(profile_zip_queue_size);
+        this.no_zip_sent = 0;
+        this.zip_sent = 0;
+        this.last_log = null;
+        this.buffer = Buffer.alloc(0);
+        this.packCount = 0;
+        this.first_time = null;
+        this.isProcessing = false;
+        ZipProfile.instance = this;
+
+        if (profile_zip_enabled) {
+            this.startProcessQueue();
+        }
+    }
+
+    static getInstance() {
+        if (!ZipProfile.instance) {
+            ZipProfile.instance = new ZipProfile();
+        }
+        return ZipProfile.instance;
+    }
+
+    static resetInstance() {
+        if(ZipProfile.instance){
+            ZipProfile.instance = null;
+        }
+    }
+
+    add(p){
+        var ok = this.queue.put(p);
+        if(!ok){
+            DataPackSender.sendPack(p);
+            this.no_zip_sent++;
+        }
+    }
+
+    async processQueue() {
+        this.isProcessing = true;
+        while (this.isProcessing) {
+            if(!profile_zip_enabled) { break; }
+
+            const p = await this.queue.getByTimeout(profile_zip_max_wait_time);
+            if(p) {
+                await this.append(p);
+            }else{
+                await this.sendAndClear();
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    async append(p) {
+        var b = DataOutputX.toBytesPack(p);
+        if(b.length >= net_send_max_bytes){ return; }
+
+        this.buffer = Buffer.concat([this.buffer, b]);
+        this.packCount++;
+
+        if(!this.first_time){
+            this.first_time = p.time;
+            if(this.buffer.length >= profile_zip_max_buffer_size){
+                await this.sendAndClear();
+            }
+        }else{
+            if(this.buffer.length >= profile_zip_max_buffer_size || (p.time - this.first_time) >= conf.getProperty('profile_zip_max_wait_time', 5000)){
+                await this.sendAndClear();
+            }
+        }
+    }
+
+    async sendAndClear() {
+        if(this.buffer.length === 0){ return; }
+
+        var p = new ZipPack();
+        p.time = DateUtil.currentTime();
+        p.recordCount = this.packCount;
+        p.records = this.buffer;
+
+        await this.doZip(p);
+        if(debug_profile_zip_enabled){
+            if(debug_profile_zip_interval){
+                Logger.print('WHATAP-ZIP-DEBUG', `PROFILE status=${p.status} records=${p.recordCount} | ${this.buffer.length} => ${p.records.length} queue=${this.queue.size()}`, false)
+            } else {
+                this.zip_sent++;
+                var now = Date.now();
+                if(now > (this.last_log + debug_profile_zip_interval)){
+                    this.last_log = now;
+                    this.log(p);
+                    this.zip_sent = 0;
+                }
+            }
+        }
+
+        p.pcode = secuMaster.PCODE;
+        p.oid = secuMaster.OID;
+        p.okind = conf.OKIND;
+        p.onode = conf.ONODE;
+
+        TcpRequestMgr.addProfile(0, p);
+
+        this.buffer = Buffer.alloc(0);
+        this.first_time = 0;
+        this.packCount = 0;
+    }
+
+    log(p){
+        try{
+            var sb = `PROFILE `;
+            sb += `zip_sent=${this.zip_sent}`;
+            sb += ` records=${p.records}`;
+            sb += ` | =${this.buffer.length} => ${p.records.length}`;
+            sb += ` queue=${this.queue.size()}`;
+            if (this.no_zip_sent > 0) {
+                sb += ` no_zip_sent=${this.no_zip_sent}`;
+            }
+            Logger.print('WHATAP-ZIP-DEBUG', sb, false);
+        }catch (e) {
+        }
+    }
+
+    async doZip(p){
+        if(p.status !== 0){ return; }
+        if(p.records.length < profile_zip_min_size){ return; }
+        p.status = 1;
+
+        try{
+            p.records = await this._doZip(p.records);
+        }catch (e) {
+            Logger.print('WHATAP-ZIP-ERROR', 'Error occurred during compression.', false);
+        }
+    }
+
+    _doZip(data) {
+        return new Promise((resolve, reject) => {
+            zlib.gzip(data, (err, buffer) => {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve(buffer);
+                }
+            });
+        });
+    }
+
+    startProcessQueue() {
+        if (!this.isProcessing) {
+            this.processQueue()
+        }
+    }
+    stopProcessQueue() {
+        this.isProcessing = false;
+    }
+}
+
+module.exports = ZipProfile;

package/lib/env/constants.js

@@ -0,0 +1,21 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var numType = {
+	INT8_MIN : -128,
+	INT8_MAX : 127,
+	INT16_MIN : -32768,
+	INT16_MAX : 32767,
+	INT24_MIN : -8388608,
+	INT24_MAX : 8388607,
+	INT32_MIN : -2147483648,
+	INT32_MAX : 2147483647,
+	INT40_MIN : -549755813888,
+	INT40_MAX : 549755813887,
+	INT64_MIN : -4503599627370496,
+	INT64_MAX : 4503599627370495
+};
+module.exports.numType = numType;

package/lib/error/error-handler.js

@@ -0,0 +1,437 @@
+/**
+ * 메시지/알림 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - Message Step
+ * @param {Error} error - 오류 객체
+ */
+function handleMessageError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'msg-',
+        textType: TextTypes.MESSAGE,
+        defaultMessage: 'Message processing error',
+        includeStack: false,
+        logPrefix: 'MESSAGE-ERROR'
+    });
+}
+
+/**
+ * 오류 정보를 Context에만 설정 (StatError 생성 없이)
+ * @param {Object} ctx - Trace Context
+ * @param {Error} error - 오류 객체
+ * @param {string} prefix - 오류 접두사
+ */
+function setContextErrorOnly(ctx, error, prefix = '') {
+    if (!ctx || !error) return;
+
+    try {
+        // 오류 클래스 정보 설정
+        ctx.error_class = error.name || (error.constructor && error.constructor.name) || 'UnknownError';
+
+        // 오류 메시지 설정
+        ctx.error_message = error.message || error.stack || 'Unknown error';
+
+        // 상태 코드 설정 (있는 경우)
+        if (error.status || error.statusCode || error.code) {
+            ctx.statusCode = error.status || error.statusCode || error.code;
+        }
+
+    } catch (e) {
+        Logger.printError(
+            'WHATAP-CONTEXT-ERROR',
+            'Error setting context error info',
+            e,
+            false
+        );
+    }
+}/**
+ * Step 타입에 따라 TextType 자동 추론
+ * @param {Object} step - Step 객체
+ * @returns {number} - TextTypes 상수
+ */
+function inferTextTypeFromStep(step) {
+    if (!step) return TextTypes.MESSAGE;
+
+    // Step 클래스명 또는 타입으로 판단
+    const stepName = step.constructor?.name || '';
+
+    if (stepName.includes('Http') || step.url !== undefined) {
+        return TextTypes.HTTPC_URL;
+    }
+    if (stepName.includes('Socket') || stepName.includes('Websocket')) {
+        return TextTypes.MESSAGE;
+    }
+    if (stepName.includes('Message') || step.desc !== undefined) {
+        return TextTypes.MESSAGE;
+    }
+    if (stepName.includes('Sql') || step.hash !== undefined) {
+        return TextTypes.SQL;
+    }
+
+    // 기본값
+    return TextTypes.MESSAGE;
+}/**
+ * 공통 오류 처리 유틸리티
+ * 모든 Observer에서 사용할 수 있는 통합 오류 처리 함수들
+ */
+
+const StatError = require('../stat/stat-error');
+const TextTypes = require('../lang/text-types');
+const Logger = require('../logger');
+const conf = require('../conf/configure');
+
+/**
+ * 공통 오류 정보 설정 함수 (범용)
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - Step 객체 (SQL, HTTP, Message 등)
+ * @param {Error} error - 오류 객체
+ * @param {Object} options - 추가 옵션
+ * @param {string} options.prefix - 오류 코드 접두사 (예: 'mysql-', 'httpc-', 'redis-')
+ * @param {number} options.textType - TextTypes 상수 (기본값: null, step 타입에 따라 자동 결정)
+ * @param {boolean} options.includeStack - 스택 트레이스 포함 여부 (기본값: true)
+ * @param {string} options.defaultMessage - 기본 오류 메시지
+ * @param {boolean} options.setStatError - StatError 생성 여부 (기본값: true)
+ * @param {string} options.logPrefix - 로그 접두사 (기본값: 'COMMON-ERROR')
+ */
+function setCommonErrorInfo(ctx, step, error, options = {}) {
+    if (!ctx || !error) {
+        return;
+    }
+
+    const {
+        prefix = '',
+        textType = null,
+        includeStack = true,
+        defaultMessage = 'Error',
+        setStatError = true,
+        logPrefix = 'COMMON-ERROR'
+    } = options;
+
+    try {
+        // 오류 클래스 정보 설정
+        const errorClass = error.name || (error.constructor && error.constructor.name) || 'UnknownError';
+        ctx.error_class = errorClass;
+
+        // 오류 코드 생성
+        const errorCode = error.code || error.errno || error.errorNum || 'UNKNOWN';
+        const fullErrorCode = prefix ? `${prefix}${errorCode}` : errorCode;
+
+        // 오류 메시지 설정
+        const errorMessage = error.message || error.sqlMessage || defaultMessage;
+
+        // 스택 트레이스 처리
+        if (includeStack && conf.trace_sql_error_stack && conf.trace_sql_error_depth && error.stack) {
+            const traceDepth = conf.trace_sql_error_depth;
+            const errorStack = error.stack.split("\n");
+
+            if (errorStack.length > traceDepth) {
+                ctx.error_message = errorStack.slice(0, traceDepth + 1).join("\n");
+            } else {
+                ctx.error_message = error.stack;
+            }
+        } else {
+            ctx.error_message = errorMessage;
+        }
+
+        // StatError 생성 (step이 있고 설정이 활성화된 경우)
+        if (step && setStatError) {
+            const shouldAddError = shouldIgnoreError(errorCode, errorMessage);
+
+            if (shouldAddError) {
+                // textType이 null이면 step 타입에 따라 자동 결정
+                const finalTextType = textType || inferTextTypeFromStep(step);
+
+                step.error = StatError.addError(
+                    fullErrorCode,
+                    errorMessage,
+                    ctx.service_hash,
+                    finalTextType,
+                    step.hash || step.url || null
+                );
+
+                // Context 오류가 아직 설정되지 않았다면 설정
+                if (ctx.error && ctx.error.isZero && ctx.error.isZero()) {
+                    ctx.error = step.error;
+                }
+            }
+        }
+
+        Logger.printError(
+            `WHATAP-${logPrefix.toUpperCase()}`,
+            `${prefix || 'SYSTEM'} Error: ${fullErrorCode} - ${errorMessage}`,
+            error,
+            false
+        );
+
+    } catch (e) {
+        Logger.printError(
+            'WHATAP-ERROR-HANDLER',
+            'Error in common error handler',
+            e,
+            false
+        );
+    }
+}
+
+/**
+ * 오류 무시 여부 확인
+ * @param {string} errorCode - 오류 코드
+ * @param {string} errorMessage - 오류 메시지
+ * @returns {boolean} - 오류를 처리할지 여부
+ */
+function shouldIgnoreError(errorCode, errorMessage) {
+    try {
+        // 오류 클래스 무시 설정 확인
+        if (conf._is_trace_ignore_err_cls_contains === true &&
+            conf.trace_ignore_err_cls_contains &&
+            errorCode &&
+            errorCode.indexOf(conf.trace_ignore_err_cls_contains) >= 0) {
+            return false;
+        }
+
+        // 오류 메시지 무시 설정 확인
+        if (conf._is_trace_ignore_err_msg_contains === true &&
+            conf.trace_ignore_err_msg_contains &&
+            errorMessage &&
+            errorMessage.indexOf(conf.trace_ignore_err_msg_contains) >= 0) {
+            return false;
+        }
+
+        // 기본적으로 오류 처리
+        return true;
+    } catch (e) {
+        Logger.printError(
+            'WHATAP-ERROR-IGNORE-CHECK',
+            'Error checking ignore conditions',
+            e,
+            false
+        );
+        return true;
+    }
+}
+
+/**
+ * MySQL 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - SQL Step
+ * @param {Error} error - MySQL 오류 객체
+ */
+function handleMysqlError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'mysql-',
+        textType: TextTypes.SQL,
+        defaultMessage: 'mysql error'
+    });
+}
+
+/**
+ * PostgreSQL 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - SQL Step
+ * @param {Error} error - PostgreSQL 오류 객체
+ */
+function handlePgsqlError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'pgsql-',
+        textType: TextTypes.SQL,
+        defaultMessage: 'postgresql error'
+    });
+}
+
+/**
+ * Oracle 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - SQL Step
+ * @param {Error} error - Oracle 오류 객체
+ */
+function handleOracleError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'oracle-',
+        textType: TextTypes.SQL,
+        defaultMessage: 'Oracle error'
+    });
+}
+
+/**
+ * MongoDB 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - SQL Step
+ * @param {Error} error - MongoDB 오류 객체
+ */
+function handleMongoError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'mongodb-',
+        textType: TextTypes.SQL,
+        defaultMessage: 'MongoDB error'
+    });
+}
+
+/**
+ * Redis 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - SQL Step
+ * @param {Error} error - Redis 오류 객체
+ */
+function handleRedisError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'redis-',
+        textType: TextTypes.SQL,
+        defaultMessage: 'Redis error'
+    });
+}
+
+/**
+ * HTTP 클라이언트 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - HTTP Step
+ * @param {Error} error - HTTP 오류 객체
+ */
+function handleHttpcError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'httpc-',
+        textType: TextTypes.HTTPC_URL,
+        defaultMessage: 'HTTP client error',
+        includeStack: false,
+        logPrefix: 'HTTPC-ERROR'
+    });
+}
+
+/**
+ * 일반적인 시스템 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - Step 객체 (선택사항)
+ * @param {Error} error - 오류 객체
+ * @param {string} prefix - 오류 접두사
+ */
+function handleSystemError(ctx, step, error, prefix = 'system') {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: `${prefix}-`,
+        textType: null, // 자동 추론
+        defaultMessage: 'System error',
+        includeStack: true,
+        logPrefix: 'SYSTEM-ERROR'
+    });
+}
+
+/**
+ * gRPC 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - gRPC Step (MessageStep)
+ * @param {Error} error - gRPC 오류 객체
+ */
+function handleGrpcError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'grpc-',
+        textType: TextTypes.MESSAGE,
+        defaultMessage: 'gRPC error',
+        includeStack: true,
+        logPrefix: 'GRPC-ERROR'
+    });
+}
+
+/**
+ * WebSocket 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - WebSocket Step (SocketStep)
+ * @param {Error} error - WebSocket 오류 객체
+ */
+function handleWebsocketError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'websocket-',
+        textType: TextTypes.MESSAGE,
+        defaultMessage: 'WebSocket error',
+        includeStack: false,
+        logPrefix: 'WEBSOCKET-ERROR'
+    });
+}
+
+/**
+ * Socket.IO 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - Socket.IO Step (SocketStep)
+ * @param {Error} error - Socket.IO 오류 객체
+ */
+function handleSocketIOError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'socketio-',
+        textType: TextTypes.MESSAGE,
+        defaultMessage: 'Socket.IO error',
+        includeStack: false,
+        logPrefix: 'SOCKETIO-ERROR'
+    });
+}
+
+/**
+ * Global Fetch API 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - HTTP Step (HttpStepX)
+ * @param {Error|Response} error - Fetch 오류 객체 또는 Response 객체
+ */
+function handleFetchError(ctx, step, error) {
+    // Response 객체인 경우와 Error 객체인 경우 구분
+    const isResponse = error && typeof error.status === 'number';
+
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'fetch-',
+        textType: TextTypes.HTTPC_URL,
+        defaultMessage: isResponse ? 'Fetch HTTP error' : 'Fetch network error',
+        includeStack: !isResponse, // Response 객체는 스택 없음
+        logPrefix: 'FETCH-ERROR'
+    });
+}
+
+/**
+ * Process 관련 오류 처리 (stdout/stderr)
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - Process Step
+ * @param {Error} error - Process 오류 객체
+ */
+function handleProcessError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'process-',
+        textType: TextTypes.MESSAGE,
+        defaultMessage: 'Process error',
+        includeStack: true,
+        logPrefix: 'PROCESS-ERROR'
+    });
+}
+
+/**
+ * Prisma 전용 오류 처리
+ * @param {Object} ctx - Trace Context
+ * @param {Object} step - SQL Step
+ * @param {Error} error - Prisma 오류 객체
+ */
+function handlePrismaError(ctx, step, error) {
+    setCommonErrorInfo(ctx, step, error, {
+        prefix: 'prisma-',
+        textType: TextTypes.SQL,
+        defaultMessage: 'Prisma error'
+    });
+}
+
+module.exports = {
+    setCommonErrorInfo,
+    shouldIgnoreError,
+    inferTextTypeFromStep,
+    setContextErrorOnly,
+
+    // 데이터베이스 관련
+    handleMysqlError,
+    handlePgsqlError,
+    handleOracleError,
+    handleMongoError,
+    handleRedisError,
+    handlePrismaError,
+
+    // 네트워크/통신 관련
+    handleHttpcError,
+    handleFetchError,
+    handleGrpcError,
+    handleWebsocketError,
+    handleSocketIOError,
+
+    // 시스템 관련
+    handleSystemError,
+    handleProcessError,
+    handleMessageError
+};

package/lib/io/data-inputx.js

@@ -4,7 +4,17 @@
  * can be found in the LICENSE file.
  */
 
-var ValueEnum       = require('../value/valueenum'),
+var constants       = require('../env/constants'),
+    ValueEnum       = require('../value/valueenum'),
+    CounterPack     = require('../pack/counter-pack'),
+    ActiveStackPack = require('../pack/activestack-pack'),
+    TextPack        = require('../pack/text-pack').TextPack,
+    StatHttpcPack   = require('../pack/stathttpc-pack').StatHttpcPack,
+    StatErrorPack   = require('../pack/staterror-pack').StatErrorPack,
+    ParamPack       = require('../pack/param-pack').ParamPack,
+    ErrorSnapPack1  = require('../pack/errorsnap-pack').ErrorSnapPack1,
+    EventPack       = require('../pack/event-pack').Instance,
+    HitMapPack1     = require('../pack/hitmap-pack1'),
     PackEnum        = require('../pack/packenum'),
     Long            = require('long');
 
@@ -121,7 +131,7 @@ DataInputX.prototype.readByte = function() {
 DataInputX.prototype.readIntBytes = function(max) {
     var len = this.readInt32BE();
     if(len < 0 || len > max){
-        Logger.printError('WHATAP-501', "read byte is overflowed max:" + max + " len:" + len, new Error());
+        Logger.printError('WHATAP-210', "read byte is overflowed max:" + max + " len:" + len, new Error());
     }
     return this.read(len);
 };
@@ -228,7 +238,7 @@ DataInputX.prototype.readPack = function () {
         }
         return null;
     } catch (e) {
-        require('../logger').printError('WHATAP-502', "DataInputX (readPack)", e , false);
+        require('../logger').printError('WHATAP-211', "DataInputX (readPack)", e , false);
         return null;
     }
 };