whatap 1.0.1 → 1.0.2

package/lib/observers/http-observer.js

@@ -4,41 +4,66 @@
  * can be found in the LICENSE file.
  */
 
+const { reqlog_x_txid } = require('../conf/config-default');
+const controlHandler = require('../control/control-handler');
+const RequestLog = require('../requestlog');
 var TraceContextManager = require('../trace/trace-context-manager'),
-    URLPatternDetector = require('../trace/serviceurl-pattern-detector').Detector,
-    conf = require('../conf/configure'),
-    HashUtil = require('../util/hashutil'),
-    Hexa32 = require('../util/hexa32'),
-    KeyGen = require('../util/keygen'),
-    TraceHelper = require('../util/trace-helper'),
-    Logger = require('../logger'),
-    AsyncSender = require('../udp/async_sender'),
-    PacketTypeEnum = require('../udp/packet_type_enum'),
-    TraceHttpc = require('../trace/trace-httpc');
+    URLPatternDetector  = require('../trace/serviceurl-pattern-detector').Detector,
+    DataTextAgent       = require('../data/datatext-agent'),
+    DataProfileAgent    = require('../data/dataprofile-agent'),
+    EventLevel          = require('../data/event-level'),
+    HttpStepX           = require('../step/http-stepx'),
+    TxRecord            = require('../service/tx-record'),
+    StatError           = require('../stat/stat-error'),
+    StatHttpc           = require('../stat/stat-httpc'),
+    StatTranxMtCaller   = require('../stat/stat-tx-caller'),
+    ProfilePack         = require('../pack/profile-pack'),
+    DataInputX          = require('../io/data-inputx'),
+    SecurityMaster      = require('../net/security-master'),
+    MeterService        = require('../counter/meter/meter-service').MeterService,
+    MeterUsers          = require('../counter/meter/meter-users'),
+    MeterHttpC          = require('../counter/meter/meter-httpc'),
+    conf                = require('../conf/configure'),
+    HashUtil            = require('../util/hashutil'),
+    UserIdUtil          = require('../util/userid-util'),
+    DateUtil            = require('../util/dateutil'),
+    IPUtil              = require('../util/iputil'),
+    Hexa32              = require('../util/hexa32'),
+    ResourceProfile     = require('../util/resourceprofile'),
+    TextTypes           = require('../lang/text-types'),
+    MessageStep         = require('../step/message-step'),
+    SecureMsgStep = require('../step/securemsg-step'),
+    PluginLoaderManager = require('../plugin/plugin-loadermanager'),
+    Long                = require('long'),
+    KeyGen              = require('../util/keygen'),
+    Logger              = require('../logger'),
+    IntKeyLinkedMap = require("../util/intkey-linkedmap");
+const ParamSecurity = require("../util/paramsecurity");
 const {Buffer} = require("buffer");
 const shimmer = require('../core/shimmer');
-const os = require('os');
-const Transfer = require('../util/transfer');
+const TraceHttpc = require('../trace/trace-httpc');
 
-var _exts = new Set([".css", ".js", ".png", ".htm", ".html", ".gif", ".jpg", ".css", ".txt", ".ico"]);
+var _exts=new Set([".css",".js",".png", ".htm", ".html", ".gif", ".jpg", ".css", ".txt", ".ico"]);
 
 var configIpHeaderKey = conf.getProperty('trace_http_client_ip_header_key', 'x-forwarded-for');
 var configUserAgentKey = conf.getProperty('trace_user_agent_header_key', '');
 var configRefererKey = conf.getProperty('trace_referer_header_key', '');
+var trace_origin_url = conf.getProperty('trace_origin_url', false);
 var ignore_http_method = conf.getProperty('ignore_http_method', 'PATCH,OPTIONS,HEAD,TRACE');
 var transaction_status_error_enable = conf.getProperty('transaction_status_error_enable', true);
 var status_ignore = conf.getProperty('status_ignore', '');
 var httpc_status_ignore = conf.getProperty('httpc_status_ignore', '');
 var status_ignore_set = conf.getProperty('status_ignore_set', '');
 var httpc_status_ignore_set = conf.getProperty('httpc_status_ignore_set', '');
+var profile_error_step_enabled = conf.getProperty('profile_error_step_enabled', '');
+var httpc_not_found_ignore = conf.getProperty('httpc_not_found_ignore', false);
+var httpc_not_found_ignore_time = conf.getProperty('httpc_not_found_ignore_time', 300000);
 var profile_http_header_ignore_keys = conf.getProperty('profile_http_header_ignore_keys', 'Cookie,cookie,accept,user-agent,referer');
 var profile_http_parameter_enabled = conf.getProperty('profile_http_parameter_enabled', true);
 var profile_http_parameter_keys = conf.getProperty('profile_http_parameter_keys', '');
 var ignore_build_file_enabled = conf.getProperty('ignore_build_file_enabled', true);
 var ignore_build_file_path = conf.getProperty('ignore_build_file_path', '/_next/');
-
-// Configuration event handlers
-conf.on('trace_http_client_ip_header_key', function (newProperty) {
+conf.on('trace_http_client_ip_header_key', function(newProperty) {
     configIpHeaderKey = newProperty;
 });
 conf.on('trace_normalize_urls', function (newProps) {
@@ -56,6 +81,9 @@ conf.on('trace_user_agent_header_key', function (newProps) {
 conf.on('trace_referer_header_key', function (newProps) {
     configRefererKey = newProps;
 });
+conf.on('trace_origin_url', function (newProps) {
+    trace_origin_url = newProps;
+})
 conf.on('ignore_http_method', function (newProps) {
     ignore_http_method = newProps;
 })
@@ -74,6 +102,15 @@ conf.on('status_ignore_set', function (newProps) {
 conf.on('httpc_status_ignore_set', function (newProps) {
     httpc_status_ignore_set = newProps;
 })
+conf.on('profile_error_step_enabled', function (newProps) {
+    profile_error_step_enabled = newProps;
+})
+conf.on('httpc_not_found_ignore', function (newProps) {
+    httpc_not_found_ignore = newProps;
+})
+conf.on('httpc_not_found_ignore_time', function (newProps) {
+    httpc_not_found_ignore_time = newProps;
+})
 conf.on('profile_http_header_ignore_keys', function (newProps) {
     profile_http_header_ignore_keys = newProps;
 })
@@ -90,66 +127,127 @@ conf.on('ignore_build_file_path', function (newProps) {
     ignore_build_file_path = newProps;
 })
 
+let _trace_mtrace_traceparent_key = conf.getProperty('trace_mtrace_traceparent_key', 'traceparent');
+let custom_trace_header_enabled = conf.getProperty('custom_trace_header_enabled', false);
+let custom_trace_header_key = conf.getProperty('custom_trace_header_key', null);
+
+// conf 리스너 추가
+conf.on('trace_mtrace_traceparent_key', function(newProperty) {
+    _trace_mtrace_traceparent_key = newProperty;
+});
+
+conf.on('custom_trace_header_enabled', function(newProperty) {
+    custom_trace_header_enabled = newProperty;
+});
+
+conf.on('custom_trace_header_key', function(newProperty) {
+    custom_trace_header_key = newProperty;
+});
 var staticConents = function (newProps) {
-    var x = new Set();
-    var words = !newProps ? [] : newProps.split(',');
-    for (var i = 0; i < words.length; i++) {
+    var x=new Set();
+    var words = !newProps?[]:newProps.split(',');
+    for(var i = 0 ; i < words.length ; i++) {
         var ex = words[i].trim();
-        if (ex.length > 0) {
-            if (ex.startsWith(".")) {
+        if(ex.length>0){
+            if(ex.startsWith(".")){
                 x.add(ex);
-            } else {
-                x.add("." + ex);
+            }else{
+                x.add("."+ex );
             }
         }
     }
-    _exts = x;
+
+    _exts =x;
 };
 conf.on('web_static_content_extensions', staticConents);
 staticConents(conf["web_static_content_extensions"]);
 
-var HttpObserver = function (agent) {
+var httpc_not_found_ignore_map = null;
+var httpc_not_found_ignore_url = new Set();
+
+var HttpObserver = function(agent){
     this.agent = agent;
-    this.packages = ['http', 'https'];
+    this.packages = ['http','https'];
     URLPatternDetector.build(true);
 };
 
-HttpObserver.prototype.__createTransactionObserver = function (callback, isHttps, server) {
+HttpObserver.prototype.__createTransactionObserver = function(callback, isHttps, server) {
     var self = this;
     var aop = this.agent.aop;
 
     return function (req, res) {
         TraceContextManager._asyncLocalStorage.run(initCtx(req, res), () => {
             var ctx = TraceContextManager._asyncLocalStorage.getStore();
-            if (!ctx) {
+            if(!ctx) {
                 return callback(req, res);
             }
 
-            ctx.service_name = req.url ? req.url : "";
-            let hostname = ctx.host && ctx.host.includes(':') ? ctx.host.split(':')[0] : '';
-            let datas = [
-                hostname,
-                ctx.service_name,
-                ctx.remoteIp,
-                ctx.userAgentString,
-                ctx.referer,
-                String(ctx.userid),
-                String(ctx.isStaticContents),
-                req.method
-            ];
-
-            aop.after(res, 'end', function (obj, args) {
-                if (ctx == null) {
-                    return;
+            PluginLoaderManager.do('httpservicestart', ctx, req, res);
+            if(trace_origin_url === true){
+                var originUrlHash = HashUtil.hashFromString('Origin url');
+                var step = new MessageStep();
+                step.hash = originUrlHash;
+                step.start_time = ctx.getElapsedTime();
+                // step.desc = req.url;
+
+                DataTextAgent.MESSAGE.add(originUrlHash, 'Origin url');
+                ctx.profile.add(step);
+            }
+
+            // res.on('aborted', () => {
+            //     self.__endTransaction(null, ctx, req, res)
+            // })
+
+            res.on('close', () => {
+                // 강제로 종료할 경우
+                self.__endTransaction(null, ctx, req, res)
+            });
+
+            aop.after(res, 'end', function(obj, args) {
+                if(ctx == null) { return; }
+                PluginLoaderManager.do('httpserviceend', ctx, req, res);
+
+                var not_found_ignore = httpc_not_found_ignore_url.has(req.url);
+                if (httpc_not_found_ignore && !not_found_ignore && obj.statusCode === 404) {
+                    const url_hash = HashUtil.hashFromString(req.url);
+                    const currentTime = new Date().getTime();
+
+                    if (!httpc_not_found_ignore_map) {
+                        httpc_not_found_ignore_map = new IntKeyLinkedMap(500, 1).setMax(500);
+                    }
+                    updateNotFoundIgnoreMap(req.url, url_hash, currentTime);
+                }
+
+                ctx.isStaticContents = isStatic(req.url);
+                ctx.http_method = req.method;
+                if(conf.profile_http_querystring_enabled){
+                    ctx.http_query = JSON.stringify(req.query);
+                }
+                ctx.http_content_type = (req.headers['content_type'] || '');
+                ctx.status = Math.floor(obj.statusCode / 100);
+
+                // 에러가 발생했지만 스택수집안했을 경우
+                if (transaction_status_error_enable && ctx.status >= 4) {
+                    if(ctx.error.isZero())
+                        ctx.error = StatError.addError(obj.statusCode, obj.statusMessage, ctx.service_hash);
+                    ctx.statusCode = obj.statusCode;
+                    ctx.statusTitle = obj.statusMessage;
+                    ctx.statusMessage = ctx.error_message;
+                }
+
+                var shouldEndTransaction = shouldEndCurrentTransaction(not_found_ignore, ctx, res, req.route ? req.route.path : '');
+                if (shouldEndTransaction) {
+                    self.__endTransaction(null, ctx, req, res);
+                } else {
+                    TraceContextManager.end(ctx._id);
+                    ctx = null;
                 }
-                self.__endTransaction(null, ctx, req, res);
             });
 
             try {
-                AsyncSender.send_packet(PacketTypeEnum.TX_START, ctx, datas)
                 return callback.apply(this, arguments);
             } catch (e) {
-                Logger.printError("WHATAP-235", 'Hooking request failed..', e, false);
+                Logger.printError("WHATAP-606", 'Hooking request failed..', e, false);
                 self.__endTransaction(e, ctx, req, res);
                 throw e;
             }
@@ -157,14 +255,13 @@ HttpObserver.prototype.__createTransactionObserver = function (callback, isHttps
     };
 };
 
-function isStatic(u) {
+function isStatic(u){
     var x = u.lastIndexOf('.');
-    if (x <= 0) return false;
+    if(x<=0) return false;
 
     var ext = u.substring(x);
     return _exts.has(ext);
 }
-
 function initCtx(req, res) {
     /*url이 없으면 추적하지 않는다*/
     if(!req.url) { return null; }
@@ -177,11 +274,70 @@ function initCtx(req, res) {
     }
 
     var ctx = TraceContextManager.start();
-    if (ctx == null) {
-        return null;
+    if(ctx == null) { return null; }
+
+    req.__ctx_id__ = ctx._id;
+
+    var url = req.url,
+        index = url.indexOf('?');
+    if(index >= 0) {
+        url = url.slice(0, index);
+    }
+    ctx.service_name = URLPatternDetector.normalize(url);
+    try {
+        if(conf.trace_service_port_enabled === true && server.address()) {
+            ctx.service_name += ' {' + server.address().port + '}';
+        }
+    } catch(e){
+    }
+
+    try {
+        if(conf.trace_transaction_name_header_key != null) {
+            ctx.service_name = req.headers[conf.trace_transaction_name_header_key] + ctx.service_name;
+        }
+    } catch(e) {
+    }
+
+    ctx.service_hash = HashUtil.hashFromString(ctx.service_name);
+    // DataTextAgent.SERVICE.add(ctx.service_hash, ctx.service_name);
+
+    ctx.isStaticContents = isStatic(req.url);
+
+    var referer = undefined;
+    if(configRefererKey && req.headers[configRefererKey]){
+        referer = req.headers[configRefererKey];
+    }else if(req.headers.referer){
+        referer = req.headers.referer;
+    }
+    if (referer) {
+        ctx.referer = HashUtil.hashFromString(referer);
+        DataTextAgent.REFERER.add(ctx.referer, referer);
+    }
+
+    if(configUserAgentKey && req.headers[configUserAgentKey]){
+        ctx.userAgentString = req.headers[configUserAgentKey];
+    }else if(req.headers['user-agent']){
+        ctx.userAgentString = req.headers['user-agent'];
+    }
+    if(ctx.userAgentString){
+        ctx.userAgent = HashUtil.hashFromString(ctx.userAgentString);
+        DataTextAgent.USERAGENT.add(ctx.userAgent, ctx.userAgentString);
+    }
+
+    ctx.http_host=req.headers.host;
+    if (ctx.http_host) {
+        ctx.http_host_hash = HashUtil.hashFromString(ctx.http_host);
+        DataTextAgent.HTTP_DOMAIN.add(ctx.http_host_hash, ctx.http_host);
     }
 
-    // RemoteIP
+    ctx.originUrl = req.url;
+    ctx.start_malloc = ResourceProfile.getUsedHeapSize();
+    ctx.start_cpu = ResourceProfile.getCPUTime();
+    ctx.http_method = req.method;
+    if(conf.profile_http_querystring_enabled){
+        ctx.http_query = JSON.stringify(req.query);
+    }
+    ctx.http_content_type = (req.headers['content_type'] || '');
     var remote_addr;
     try {
         remote_addr = req.headers[configIpHeaderKey] || req.connection.remoteAddress || "0.0.0.0";
@@ -203,27 +359,9 @@ function initCtx(req, res) {
                 MeterUsers.add(ctx.userid);
                 break;
         }
-    } catch (e) {
-    }
 
-    // Referer
-    var referer = undefined;
-    if(configRefererKey && req.headers[configRefererKey]){
-        ctx.referer = req.headers[configRefererKey];
-    }else if(req.headers.referer){
-        ctx.referer = req.headers.referer;
-    }
-
-    // UserAgent
-    if(configUserAgentKey && req.headers[configUserAgentKey]){
-        ctx.userAgentString = req.headers[configUserAgentKey];
-    }else if(req.headers['user-agent']){
-        ctx.userAgentString = req.headers['user-agent'];
+    } catch (e) {
     }
-
-    // Host
-    ctx.host = req.headers.host;
-
     /************************************/
     /*   Header / param Trace           */
     /************************************/
@@ -235,26 +373,19 @@ function initCtx(req, res) {
             header_enabled = false;
         }
     }
-    /************************************/
-    /*   Header / param Trace           */
-    /************************************/
-    var header_enabled = false;
-    if (conf.profile_http_header_enabled === true && req.headers) {
-        header_enabled = true;
-        var prefix = conf.profile_header_url_prefix;
-        if (prefix && ctx.service_name.indexOf(prefix) < 0) {
-            header_enabled = false;
-        }
-    }
 
-    if (header_enabled) {
+    if(header_enabled) {
+        var step = new MessageStep();
+        step.hash = HashUtil.hashFromString("HTTP-HEADERS");
+        step.start_time = ctx.getElapsedTime();
+
         var header_ignore_key_set = new Set();
-        if (profile_http_header_ignore_keys) {
+        if(profile_http_header_ignore_keys) {
             header_ignore_key_set = new Set(profile_http_header_ignore_keys.split(','));
         }
 
-        try {
-            var headerDesc = Object.keys(req.headers).map(function (key) {
+        try{
+            step.desc = Object.keys(req.headers).map(function (key) {
                 if (!header_ignore_key_set.has(key)) {
                     return `${key}=${req.headers[key]}`;
                 }
@@ -262,40 +393,81 @@ function initCtx(req, res) {
                 if (element) return element
             }).join('\n');
 
-            if (headerDesc) {
-                let headerDatas = ['HTTP-HEADERS', 'HTTP-HEADERS', headerDesc];
-                ctx.start_time = Date.now();
-                AsyncSender.send_packet(PacketTypeEnum.TX_MSG, ctx, headerDatas);
-            }
-        } catch (e) {
-            Logger.printError('WHATAP-236', 'Header parsing error', e, false);
+            DataTextAgent.MESSAGE.add(step.hash, "HTTP-HEADERS");
+            ctx.profile.push(step);
+        }catch (e) {
+            step = null;
+            header_ignore_key_set = null;
+            Logger.printError('WHATAP-614', 'Header parsing error', e, false);
         }
     }
-
     /************************************/
     /*   Multi Server Transaction Trace */
     /************************************/
-    if (conf.getProperty('mtrace_enabled', false)) {
-        var poid = req.headers['x-wtap-po'];
+    if(conf.getProperty('mtrace_enabled', false)) {
+        // traceparent 헤더 처리 (W3C Trace Context 표준)
+        const traceparentKey = req.headers[conf._trace_mtrace_traceparent_key];
+        let hasTraceparentKey = false;
+        let traceparentStepId = 0;
+
+        if (traceparentKey) {
+            hasTraceparentKey = true;
+            const parts = traceparentKey.trim().split('-');
+
+            if (parts.length >= 4) {
+                // parts[0]: version (00)
+                // parts[1]: 32자리 16진수 trace ID
+                // parts[2]: 16자리 16진수 parent ID (span ID)
+                // parts[3]: 2자리 16진수 trace flags
+
+                const mcallerTraceIdValue = parts[1];
+                ctx.mcallerTraceparentValue = mcallerTraceIdValue;
+
+                try {
+                    // trace ID의 뒤 16자리를 가져와서 long으로 변환
+                    const mtidHex = mcallerTraceIdValue.substring(16);
+                    const mtidLong = Long.fromString(mtidHex, true, 16);
+                    ctx.mtid = mtidLong;
+
+                    if (!ctx.mtid.isZero()) {
+                        ctx.mtid_build_checked = true;
+                    }
+                } catch (e) {
+                    Logger.printError('WHATAP-851', 'traceparent parse error', e, true);
+                }
+
+                try {
+                    // parent span ID 추출
+                    const parentSpanId = Long.fromString(parts[2], true, 16);
+                    ctx.mcaller_stepId = parentSpanId;
+                    traceparentStepId = parentSpanId;
+                } catch (e) {
+                    Logger.printError('WHATAP-851', 'traceparent span id parse error', e, true);
+                }
+            }
+        }
+
+        // WhaTap 고유 헤더 처리
+        const poid = req.headers['x-wtap-po'];
         if (poid != null) {
             ctx.setCallerPOID(poid);
+
             try {
-                var mt_caller = req.headers[conf._trace_mtrace_caller_key];
-                if (mt_caller) {
-                    var x = mt_caller.indexOf(',');
+                const mt_caller = req.headers[conf._trace_mtrace_caller_key];
+                if(mt_caller) {
+                    const x = mt_caller.indexOf(',');
                     if (x > 0) {
                         ctx.mtid = Hexa32.toLong32(mt_caller.substring(0, x));
                         ctx.mtid_build_checked = true;
-                        var y = mt_caller.indexOf(',', x + 1);
+                        const y = mt_caller.indexOf(',', x + 1);
                         if (y > 0) {
                             ctx.mdepth = parseInt(mt_caller.substring(x + 1, y));
-                            var z = mt_caller.indexOf(',', y + 1);
+                            const z = mt_caller.indexOf(',', y + 1);
                             if (z < 0) {
                                 ctx.mcaller_txid = Hexa32.toLong32(mt_caller.substring(y + 1));
                             } else {
                                 ctx.mcaller_txid = Hexa32.toLong32(mt_caller.substring(y + 1, z));
-
-                                var z2 = mt_caller.indexOf(',', z + 1);
+                                const z2 = mt_caller.indexOf(',', z + 1);
                                 if (z2 < 0) {
                                     ctx.mcaller_stepId = Hexa32.toLong32(mt_caller.substring(z + 1));
                                 } else {
@@ -305,57 +477,91 @@ function initCtx(req, res) {
                         }
                     }
                 }
+
                 if (conf.stat_mtrace_enabled) {
-                    var inf = req.headers[conf._trace_mtrace_spec_key1];
-                    if (inf != null && inf.length > 0) {
-                        var px = inf.indexOf(',');
+                    const inf = req.headers[conf._trace_mtrace_spec_key1];
+                    if(inf != null && inf.length > 0) {
+                        const px = inf.indexOf(',');
                         ctx.mcaller_spec = inf.substring(0, px);
                         ctx.mcaller_url = inf.substring(px + 1);
-                        ctx.mcaller_url_hash = HashUtil.hashFromString(ctx.mcaller_url);
                     }
                 }
-            } catch (e) {
-                Logger.printError('WHATAP-237', 'Multi Server Transaction ', e, true);
+            } catch(e) {
+                Logger.printError('WHATAP-850', 'Multi Server Transaction', e, true);
+            }
+        }
+
+        // Gateway가 header를 그대로 전달하는 경우 처리
+        if (hasTraceparentKey) {
+            if (!traceparentStepId.equals(ctx.mcaller_stepId)) {
+                ctx.mcaller_stepId = traceparentStepId;
+                ctx.mcaller_txid = Long.ZERO;
             }
         }
     }
 
-    return ctx;
-};
+    var poid=req.headers['x-wtap-po'];
+    if (poid != null) {
+        ctx.setCallerPOID(poid);
+        try{
+            var mt_caller=req.headers[conf._trace_mtrace_caller_key];
+            if(mt_caller){
+                var x = mt_caller.indexOf(',');
+                if (x > 0) {
+                    ctx.mtid = Hexa32.toLong32(mt_caller.substring(0, x));
+                    ctx.mtid_build_checked = true;
+                    var y = mt_caller.indexOf(',', x + 1);
+                    if (y > 0) {
+                        ctx.mdepth = parseInt(mt_caller.substring(x + 1, y));
+                        var z = mt_caller.indexOf(',', y + 1);
+                        if (z < 0) {
+                            ctx.mcaller_txid = Hexa32.toLong32(mt_caller.substring(y + 1));
+                        } else {
+                            ctx.mcaller_txid = Hexa32.toLong32(mt_caller.substring(y + 1, z));
 
-function interceptorError(statusCode, error, ctx) {
-    if (!ctx) {
-        return;
+                            var z2 = mt_caller.indexOf(',', z + 1);
+                            if (z2 < 0) {
+                                ctx.mcaller_stepId = Hexa32.toLong32(mt_caller.substring(z + 1));
+                            } else {
+                                ctx.mcaller_stepId = Hexa32.toLong32(mt_caller.substring(z + 1, z2));
+                            }
+                        }
+                    }
+                }
+            }
+            if (conf.stat_mtrace_enabled) {
+                var inf=req.headers[conf._trace_mtrace_spec_key1];
+                if(inf != null && inf.length > 0){
+                    var px = inf.indexOf(',');
+                    ctx.mcaller_spec =  inf.substring(0, px);
+                    ctx.mcaller_url = inf.substring(px + 1);
+                }
+            }
+        } catch(e) {
+            Logger.printError('WHATAP-850', 'Multi Server Transaction ', e, true);
+        }
     }
 
-    ctx.status = statusCode;
-    let errors = [];
-    let error_message = 'Request failed with status code ';
-    if (statusCode >= 400 && !ctx.error) {
-        ctx.error = 1;
-
-        errors.push(error.class)
-        if(error.message)
-            errors.push(error.message)
-        else
-            errors.push(error_message + statusCode);
+    return ctx;
+};
 
-        AsyncSender.send_packet(PacketTypeEnum.TX_ERROR, ctx, errors);
-    }
-}
+HttpObserver.prototype.__endTransaction = function(error, ctx, req, res) {
+    if(ctx == null || TraceContextManager.isExist(ctx._id) === false) { return; }
 
-HttpObserver.prototype.__endTransaction = function (error, ctx, req, res) {
     var param_enabled = false;
-    if (conf.profile_http_parameter_enabled === true) {
+    if(conf.profile_http_parameter_enabled === true) {
         param_enabled = true;
         var prefix = conf.profile_http_parameter_url_prefix;
-        if (prefix && ctx.service_name.indexOf(prefix) < 0) {
+        if(prefix && ctx.service_name.indexOf(prefix) < 0) {
             param_enabled = false;
         }
     }
 
     if (param_enabled && req.query && Object.keys(req.query).length > 0) {
         const query = req.query;
+        let secureMsgStep = new SecureMsgStep(ctx.getElapsedTime());
+        secureMsgStep.hash = HashUtil.hashFromString("HTTP-PARAMETERS");
+
         const profileHttpParameterKeysSet = profile_http_parameter_keys ? new Set(profile_http_parameter_keys.split(',')) : null;
 
         try {
@@ -365,67 +571,218 @@ HttpObserver.prototype.__endTransaction = function (error, ctx, req, res) {
                 .join('\n');
 
             if (desc) {
-                let paramDatas = ['HTTP-PARAMETERS', req.method, desc];
-                AsyncSender.send_packet(PacketTypeEnum.TX_SECURE_MSG, ctx, paramDatas);
+                const crc = { value: 0 };
+                secureMsgStep.value = toParamBytes(desc, crc);
+                secureMsgStep.crc = crc.value;
+
+                DataTextAgent.MESSAGE.add(secureMsgStep.hash, "HTTP-PARAMETERS");
+                ctx.profile.push(secureMsgStep);
+            } else {
+                secureMsgStep = null;
             }
         } catch (e) {
-            Logger.printError('WHATAP-238', 'Parameter parsing error', e, false);
+            secureMsgStep = null;
+            Logger.printError('WHATAP-613', 'Parameter parsing error', e, false);
         }
     }
 
-    if (error) {
-        TraceContextManager.end(ctx != null ? ctx.id : null);
-        ctx = null;
-        return;
+    if(profile_error_step_enabled && ctx.statusMessage){
+        var step = new MessageStep();
+        var title = ctx.statusTitle ? ctx.statusTitle : "EXCEPTION";
+        step.hash = HashUtil.hashFromString(title);
+        step.start_time = ctx.getElapsedTime();
+        step.desc = ctx.statusMessage;
+        DataTextAgent.MESSAGE.add(step.hash, title);
+        ctx.profile.push(step);
     }
 
-    if (ctx == null || TraceContextManager.isExist(ctx.id) === false) {
+    if(error) {
+        TraceContextManager.end(ctx != null ? ctx._id : null);
+        ctx = null;
         return;
     }
 
-    if (ctx.isStaticContents !== true && conf._trace_ignore_url_set[ctx.service_hash]) {
+    // if(ctx == null || TraceContextManager.isExist(ctx._id) === false) { return; }
+
+    if(ctx.isStaticContents !== true && conf._trace_ignore_url_set[ctx.service_hash]){
         ctx.isStaticContents = true;
     }
 
-    if (conf._is_trace_ignore_url_prefix === true && ctx.service_name.startsWith(conf.trace_ignore_url_prefix) === true) {
+    if(conf._is_trace_ignore_url_prefix === true && ctx.service_name.startsWith(conf.trace_ignore_url_prefix) === true){
         ctx.isStaticContents = true;
     }
 
-    if (ctx.isStaticContents) {
-        TraceContextManager.end(ctx.id);
+    if(ctx.isStaticContents){
+        TraceContextManager.end(ctx._id);
         ctx = null;
         return;
     }
 
     try {
-        ctx.start_time = Date.now();
-        let datas = [os.hostname(), ctx.service_name, ctx.mtid, ctx.mdepth, ctx.mcaller_txid,
-            ctx.mcaller_pcode, ctx.mcaller_spec, String(ctx.mcaller_url_hash), ctx.status];
-        ctx.elapsed = Date.now() - ctx.start_time;
-        AsyncSender.send_packet(PacketTypeEnum.TX_END, ctx, datas);
+        var profile = new ProfilePack();
+        var wtx = new TxRecord();
+        // profile.time = ctx.start_time;
+        wtx.endTime = DateUtil.currentTime();
+        profile.time  = wtx.endTime;
+        wtx.elapsed = ctx.getElapsedTime();
+
+        ctx.service_hash = HashUtil.hashFromString(ctx.service_name);
+        DataTextAgent.SERVICE.add(ctx.service_hash, ctx.service_name);
+
+        wtx.service = ctx.service_hash;
+        wtx.cpuTime = ResourceProfile.getCPUTime() - ctx.start_cpu;
+        wtx.malloc = ResourceProfile.getUsedHeapSize()-ctx.start_malloc;
+        if(wtx.malloc < 0) { wtx.malloc = 0; }
+        wtx.originUrl = ctx.originUrl;
+
+        wtx.cipher = HashUtil.hash(ParamSecurity.key);
+        wtx.seq = ctx.txid;
+        wtx.sqlCount = ctx.sql_count;
+        wtx.sqlTime = ctx.sql_time;
+        wtx.sqlFetchCount = ctx.rs_count;
+        wtx.sqlFetchTime = parseInt(ctx.rs_time);
+        wtx.ipaddr = ctx.remoteIp;
+        wtx.userid = ctx.userid;
+
+        if (ctx.error.isZero() === false) {
+            wtx.error = ctx.error;
+            wtx.errorLevel = EventLevel.WARNING;
+        }
+        wtx.userAgent = ctx.userAgent;
+        wtx.referer = ctx.referer;
+
+        wtx.httpcCount = ctx.httpc_count;
+        wtx.httpcTime = ctx.httpc_time;
+        wtx.status = ctx.status;
+
+        wtx.http_method=TxRecord.HTTP_METHOD[ctx.http_method] || TxRecord.WEB_GET;
+        wtx.http_host = ctx.http_host;
+        wtx.mtid=ctx.mtid;
+        wtx.mdepth=ctx.mdepth;
+        wtx.mcaller=ctx.mcaller_txid;
+        wtx.mcallerStepId = ctx.mcaller_stepId;
+        wtx.mcaller_pcode = ctx.mcaller_pcode;
+        wtx.mcaller_okind = ctx.mcaller_okind;
+        wtx.mcaller_oid = ctx.mcaller_oid;
+
+        wtx.appctx = "";
+        wtx.txName = ctx.service_name;
+        if(ctx.error_class) wtx.error_class = ctx.error_class;
+        if(ctx.error_message) wtx.error_message = ctx.error_message;
+
+        if(conf.getProperty('txtext_txname_enabled', true)){
+            wtx.txName = ctx.service_name;
+        }
+
+        MeterService.add(wtx.service, wtx.elapsed,
+            wtx.errorLevel, ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_oid);
+
+        profile.oid = SecurityMaster.OID;
+        profile.service = wtx;
+        if(Boolean(conf.reqlog_enabled) == true) {
+            ctx.endTime = wtx.endTime;
+            ctx.elapsed = wtx.elapsed;
+            RequestLog.setRecord(ctx);
+        }
+        //duplicated executed... so end() first
+        TraceContextManager.end(ctx._id);
+        setTimeout(function () {
+            DataProfileAgent.sendProfile(ctx, profile, false);
+            TraceContextManager.end(ctx._id);
+            ctx = null;
+        }, 100);
 
-        TraceContextManager.end(ctx.id);
     } catch (e) {
-        Logger.printError('WHATAP-239', 'End transaction error..', e, false);
-        TraceContextManager.end(ctx.id);
+        Logger.printError('WHATAP-607', 'End transaction error..', e, false);
+        TraceContextManager.end(ctx._id);
         ctx = null;
     }
+
 };
 
-// interTxTraceAutoOn function moved to util/trace-helper.js
+var transfer_poid;
+function transferPOID(ctx) {
+    if (transfer_poid)
+        return transfer_poid;
+    transfer_poid=Hexa32.toString32(SecurityMaster.PCODE)+','
+        +Hexa32.toString32(SecurityMaster.OKIND)+','+Hexa32.toString32(SecurityMaster.OID);
+    return transfer_poid;
+}
 
-HttpObserver.prototype.inject = function (mod, moduleName) {
+function transferMTID_CALLERTX(ctx) {
+    if (ctx.transfer_id)
+        return ctx.transfer_id;
+    var x = Hexa32.toString32(ctx.mtid) + ',' + (ctx.mdepth + 1) + ',' + Hexa32.toString32(ctx.txid);
+    ctx.transfer_id = x;
+    return ctx.transfer_id;
+}
+function transferSPEC_URL(ctx) {
+    if (ctx.transfer_info)
+        return ctx.transfer_info;
+    var x =  conf.mtrace_spec + ',' + ctx.service_hash;
+    ctx.transfer_info = x;
+    return ctx.transfer_info;
+}
+var check_seq = 1;
+function interTxTraceAutoOn(ctx) {
+    if (conf.mtrace_enabled ==false || ctx.httpc_checked || ctx.mtid.isZero()===false)
+        return;
+    ctx.httpc_checked = true;
+    if(conf.mtrace_rate>=100){
+        ctx.mtid = KeyGen.next();
+        return;
+    }
+    check_seq++;
+    switch (Math.floor(conf.mtrace_rate / 10)) {
+        case 10:
+            ctx.mtid = KeyGen.next();
+            break;
+        case 9:
+            if (check_seq % 10 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 8:
+            if (check_seq % 5 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 7:
+            if (check_seq % 4 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 6:
+            if (check_seq % 3 !== 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 5:
+            if (check_seq % 2 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 4:
+            if (check_seq % 3 === 0 || check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 3:
+            if (check_seq % 4 === 0 || check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 2:
+            if (check_seq % 5 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+        case 1:
+            if (check_seq % 10 === 0)
+                ctx.mtid = KeyGen.next();
+            break;
+    }
+}
+HttpObserver.prototype.inject = function( mod, moduleName ) {
     var self = this;
     var aop = self.agent.aop;
 
-    if (mod.__whatap_observe__) {
-        return;
-    }
+    if(mod.__whatap_observe__) { return; }
     mod.__whatap_observe__ = true;
     Logger.initPrint("HttpObserver");
-    if (conf.getProperty('profile_enabled', true) === false) {
-        return;
-    }
+    if( conf.getProperty('profile_enabled', true) === false ) { return; }
 
     aop.after(mod, 'createServer', function (obj, args, ret) {
         aop.before(ret, 'listen', function (obj, args) {
@@ -448,83 +805,102 @@ HttpObserver.prototype.inject = function (mod, moduleName) {
         }
     );
 
-    if (conf.getProperty('httpc_enabled', true) === false) {
-        return;
-    }
+    if( conf.getProperty('httpc_enabled', true) === false ) { return; }
 
     shimmer.wrap(mod, 'request', function (original) {
         return function wrappedRequest(options, callback) {
             var ctx = TraceContextManager.getCurrentContext();
-            if (!ctx || (!options.host && !options.hostname)) {
+            if(!ctx || (!options.host && !options.hostname)){
                 return original.apply(this, arguments);
             }
 
             var isHttpRepeat = false;
-            if (moduleName === 'https') {
+            if(moduleName === 'https'){
                 options.__isHttps = true;
             }
-            if (moduleName === 'http' && options.__isHttps) {
+            if(moduleName === 'http' && options.__isHttps){
                 isHttpRepeat = true;
             }
-            ctx.start_time = Date.now();
 
-            if (!isHttpRepeat) {
-                if (options.method === 'OPTION') {
+            var step = new HttpStepX();
+            step.start_time = ctx.getElapsedTime();
+
+            if(!isHttpRepeat){
+                if(options.method === 'OPTION'){
                     return original.apply(this, arguments);
                 }
-                try {
-                    TraceHelper.interTxTraceAutoOn(ctx);
-                    if (conf.getProperty('mtrace_enabled', false)) {
-                        if (options.headers) {
-                            options.headers['x-wtap-po'] = Transfer.POID();
-                        } else {
-                            options.headers = {'x-wtap-po': Transfer.POID()};
+                try{
+                    interTxTraceAutoOn(ctx);
+                    if(conf.getProperty('mtrace_enabled', false)){
+                        if(options.headers){
+                            options.headers['x-wtap-po'] = transferPOID(ctx);
+                        }else{
+                            options.headers = {'x-wtap-po': transferPOID(ctx)};
                         }
-                        if (conf.stat_mtrace_enabled) {
-                            options.headers[conf._trace_mtrace_spec_key1] = Transfer.SPEC_URL(ctx);
+                        if(conf.stat_mtrace_enabled){
+                            options.headers[conf._trace_mtrace_spec_key1]=transferSPEC_URL(ctx);
                         }
-                        if (!ctx.mtid.isZero()) {
-                            options.headers[conf._trace_mtrace_caller_key] = Transfer.MTID_CALLERTX(ctx);
+                        if(ctx.mtid.isZero()===false){
+                            options.headers[conf._trace_mtrace_caller_key]=transferMTID_CALLERTX(ctx);
                         }
-
-                        ctx.mcallee = KeyGen.next();
-                        options.headers[conf._trace_mtrace_callee_key] = Hexa32.toString32(ctx.mcallee);
                     }
 
                     ctx.httpc_url = options.path || '/';
                     ctx.httpc_host = options.host || options.hostname || '';
                     ctx.httpc_port = options.port || -1;
-                    ctx.active_httpc_hash = true;
-                    
-                    if (ctx.httpc_port < 0) {
-                        ctx.httpc_port = 80
+
+                    ctx.footprint('Http Call Start');
+
+                    if (ctx.httpc_port < 0) { ctx.httpc_port = 80 };
+
+                    if(conf.getProperty('profile_httpc_start_step_enabled', false)){
+                        step.elapsed = ctx.getElapsedTime() - step.start_time;
+                        step.url = HashUtil.hashFromString(ctx.httpc_url);
+                        DataTextAgent.HTTPC_URL.add(step.url, ctx.httpc_url);
+                        step.host = HashUtil.hashFromString(ctx.httpc_host);
+                        DataTextAgent.HTTPC_HOST.add(step.host, ctx.httpc_host);
+                        step.port = ctx.httpc_port;
+
+                        ctx.active_httpc_hash = step.url;
+                        ctx.profile.push(step);
+                        endHttpc(ctx, step);
                     }
-                } catch (e) {
-                    Logger.printError('WHATAP-240', 'Http Setup Error', e, true);
+                }catch (e) {
+                    Logger.printError('WHATAP-852', 'Http Repeat ', e, true);
                     return original.apply(this, arguments);
                 }
-            } else {
+            }else{
                 return original.apply(this, arguments);
             }
 
             var wrappedCallback;
             if (typeof callback === 'function') {
-                wrappedCallback = function (response) {
-                    if (TraceContextManager.resume(ctx.id) === null) {
+                wrappedCallback = function(response) {
+                    if (TraceContextManager.resume(ctx._id) === null) {
                         return callback.apply(this, arguments);
                     }
 
-                    response.on('end', function () {
-                        // HTTP 클라이언트 에러 처리 - Python 스타일로 단순화
+                    response.on('end', function() {
+                        step.elapsed = ctx.getElapsedTime() - step.start_time;
+                        step.url = HashUtil.hashFromString(ctx.httpc_url);
+                        DataTextAgent.HTTPC_URL.add(step.url, ctx.httpc_url);
+                        step.host = HashUtil.hashFromString(ctx.httpc_host);
+                        DataTextAgent.HTTPC_HOST.add(step.host, ctx.httpc_host);
+                        step.port = ctx.httpc_port;
+
                         if (response.statusCode >= 400 && transaction_status_error_enable) {
-                            let error = {
-                                class: response.statusMessage,
-                                message: ''
+                            step.error = StatError.addError(response.statusCode, response.statusMessage,
+                                ctx.service_hash, TextTypes.HTTPC_URL, step.url);
+                            if (ctx.error.isZero()) {
+                                ctx.error = step.error;
+                                ctx.statusCode = response.statusCode;
+                                ctx.statusMessage = response.statusMessage;
                             }
-                            interceptorError(response.statusCode, error, ctx);
                         }
 
-                        endHttpc(ctx);
+                        ctx.active_httpc_hash = step.url;
+                        ctx.profile.push(step);
+                        endHttpc(ctx, step);
                     });
 
                     return callback.apply(this, arguments);
@@ -532,32 +908,36 @@ HttpObserver.prototype.inject = function (mod, moduleName) {
             }
 
             var req = original.apply(this, [options, wrappedCallback]);
+            var httpc_url = options.path
 
             req.on('error', function (err) {
-                if (TraceContextManager.resume(ctx.id) === null) {
-                    return;
-                }
-
-                const isIgnoreHttpc = shouldIgnoreError(err.code, ctx.httpc_url, httpc_status_ignore, httpc_status_ignore_set);
-
-                const networkErrorToStatusCode = {
-                    'ECONNREFUSED': 503,
-                    'ETIMEDOUT': 504,
-                    'ENOTFOUND': 502,
-                    'ECONNRESET': 503,
-                    'EPIPE': 503,
-                    'EHOSTUNREACH': 503,
-                };
-                const statusCode = networkErrorToStatusCode[err.code] || 500;
+                if (TraceContextManager.resume(ctx._id) === null) { return; }
+                const isIgnoreHttpc = shouldIgnoreError(
+                    err.code,
+                    httpc_url,
+                    httpc_status_ignore,
+                    httpc_status_ignore_set);
+
+                if (!isIgnoreHttpc && transaction_status_error_enable && step.error.isZero()) {
+                    ctx.is_httpc_error = true;
+
+                    step.error = StatError.addError(
+                        err.code,
+                        err.message,
+                        ctx.service_hash,
+                        TextTypes.HTTPC_URL,
+                        step.url
+                    );
+
+                    if (ctx.error.isZero()) {
+                        ctx.error = step.error;
+                        ctx.statusCode = err.code;
+                        ctx.statusMessage = ctx.error_message = err.message;
 
-                if (!isIgnoreHttpc && transaction_status_error_enable) {
-                    let error = {
-                        class: err.code,
-                        message: err.message || ''
                     }
-                    interceptorError(statusCode, error, ctx);
                 }
-                endHttpc(ctx);
+
+                endHttpc(ctx, step);
             });
 
             shimmer.wrap(req, 'write', function (original) {
@@ -573,15 +953,18 @@ HttpObserver.prototype.inject = function (mod, moduleName) {
                                 if (arguments[0] instanceof Buffer) {
                                     bodyData = arguments[0].toString('utf8');
                                 }
-                                if (bodyData) {
-                                    console.log(`body Data: ${bodyData}`)
-                                    let paramDatas = ['HTTP-PARAMETERS', options.method, bodyData];
-                                    AsyncSender.send_packet(PacketTypeEnum.TX_SECURE_MSG, ctx, paramDatas);
+                                if(bodyData){
+                                    var step = new MessageStep();
+                                    step.hash = HashUtil.hashFromString("HTTPC-REQUEST-BODY");
+                                    step.start_time = ctx.getElapsedTime();
+                                    step.desc = bodyData;
+                                    DataTextAgent.MESSAGE.add(step.hash, "HTTPC-REQUEST-BODY");
+                                    ctx.profile.push(step);
                                 }
                             }
                         }
                     } catch (e) {
-                        Logger.printError('WHATAP-241', 'HTTP Write hook failed', e, false);
+                        Logger.printError('WHATAP-615', 'HTTP Write hook failed', e, false);
                     }
                     return original.apply(this, arguments);
                 }
@@ -591,18 +974,18 @@ HttpObserver.prototype.inject = function (mod, moduleName) {
         }
     });
 
-    // HTTP 클라이언트 종료 처리 - 단순화
-    function endHttpc(ctx) {
-        if (ctx == null) {
-            return;
-        }
+    function endHttpc(ctx, step) {
+        if(ctx == null || step == null) { return; }
 
-        ctx.active_httpc_hash = false;
-        let urls = `${ctx.httpc_host}:${ctx.httpc_port}${ctx.httpc_url}`;
-        let httpcDatas = [urls, ctx.mcallee];
-        ctx.elapsed = Date.now() - ctx.start_time;
-        TraceHttpc.isSlowHttpc(ctx);
-        AsyncSender.send_packet(PacketTypeEnum.TX_HTTPC, ctx, httpcDatas);
+        step.elapsed = ctx.getElapsedTime() - step.start_time;
+        TraceHttpc.isSlowHttpc(ctx, step);
+
+        ctx.httpc_count++;
+        ctx.httpc_time += step.elapsed;
+
+        MeterHttpC.add(step.host, step.elapsed, step.error.isZero()===false);
+        StatHttpc.addHttpcTime(ctx.service_hash, step.url, step.host, step.port, step.elapsed, step.error.isZero()===false);
+        ctx.footprint('Http Call Done');
     }
 };
 
@@ -625,4 +1008,51 @@ function shouldIgnoreError(statusCode, url, statusCodeIgnore, statusIgnoreSet) {
     return false;
 }
 
+function updateNotFoundIgnoreMap(url, url_hash, currentTime) {
+    var ignore_value = httpc_not_found_ignore_map.get(url_hash) || {
+        count: 0, start_time: currentTime, last_time: currentTime, url: url
+    };
+
+    ignore_value.count++;
+    ignore_value.last_time = currentTime;
+
+    if ((ignore_value.last_time - ignore_value.start_time) > httpc_not_found_ignore_time) {
+        ignore_value = { count: 1, start_time: currentTime, last_time: currentTime, url: url };
+    }
+
+    if (ignore_value.count >= 50) {
+        httpc_not_found_ignore_url.add(url);
+        httpc_not_found_ignore_map.remove(url_hash);
+        Logger.print('WHATAP-610', "The ignore option for URL " + url + " is applied.", false);
+    }
+
+    httpc_not_found_ignore_map.put(url_hash, ignore_value);
+}
+
+function shouldEndCurrentTransaction(not_found_ignore, ctx, res, requestPath) {
+    if (not_found_ignore) return false;
+    const is_ignore = shouldIgnoreError(res.statusCode, requestPath, status_ignore, status_ignore_set);
+    return !is_ignore;
+}
+
+var toParamBytes = function (p, crc) {
+    if (p == null || p.length === 0) {
+        return null;
+    }
+    try {
+        return ParamSecurity.encrypt(Buffer.from(p, 'utf8'), crc);
+    } catch (e) {
+        return null;
+    }
+};
+
+function createMtraceId(userAgentString) {
+    // MTraceHelper.createId 구현
+    if (userAgentString) {
+        // userAgent 기반으로 ID 생성
+        return KeyGen.next();
+    }
+    return KeyGen.next();
+}
+
 exports.HttpObserver = HttpObserver;