whatap-python 2.1.0__py3-none-any.whl

whatap/trace/mod/httpc/django.py

@@ -0,0 +1,31 @@
+from whatap.trace.trace_handler import trace_handler
+from whatap.trace.mod.httpc.util import transfer, \
+    interceptor_httpc_request
+
+
+def instrument_revproxy_views(module):
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            callback = fn(*args, **kwargs)
+            
+            # set mtid header
+            callback = transfer(callback)
+            return callback
+        return trace
+    
+    module.ProxyView.get_proxy_request_headers = wrapper(module.ProxyView.get_proxy_request_headers)
+
+    
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            # set httpc_url
+            httpc_url = args[0].upstream
+            callback = interceptor_httpc_request(fn, httpc_url, *args, **kwargs)
+            return callback
+
+        return trace
+    
+    module.ProxyView.dispatch = wrapper(module.ProxyView.dispatch)
+

whatap/trace/mod/httpc/httplib.py

@@ -0,0 +1,70 @@
+from whatap.trace.trace_handler import trace_handler
+from whatap.trace.mod.httpc.util import transfer, \
+    interceptor_httpc_request, interceptor_sock_connect
+
+request_injection_processed = False
+def instrument_httplib(module):
+    global request_injection_processed
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            # set mtid header
+            kwargs['headers'] = transfer(kwargs.get('headers', {}))
+            newargs = []
+            for arg in args:
+                if arg and isinstance(arg, dict) and 'host' in [x.lower() for x in arg.keys()]:
+                    arg = transfer(arg)
+                newargs.append(arg)
+            # set httpc_url
+            httpc_url = args[2]
+            if hasattr(args[0], 'host'):
+                httpc_url = getattr(args[0], 'host') + httpc_url
+            callback = interceptor_httpc_request(fn, httpc_url, *newargs, **kwargs)
+            return callback
+        
+        return trace
+    if not request_injection_processed:
+        module.HTTPConnection.request = wrapper(module.HTTPConnection.request)
+        request_injection_processed = True
+
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            callback = interceptor_sock_connect(fn, *args, **kwargs)
+            return callback
+        
+        return trace
+    module.HTTPConnection.connect = wrapper(module.HTTPConnection.connect)
+
+def instrument_httplib2(module):
+    global request_injection_processed
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            # set mtid header
+            kwargs['headers'] = transfer(kwargs.get('headers', {}))
+            newargs = []
+            for arg in args:
+                if arg and isinstance(arg, dict) and 'host' in [x.lower() for x in arg.keys()]:
+                    arg = transfer(arg)
+                newargs.append(arg)
+            # set httpc_url
+            httpc_url = args[2]
+            if hasattr(args[0], 'host'):
+                httpc_url = getattr(args[0], 'host') + httpc_url
+            callback = interceptor_httpc_request(fn, httpc_url, *newargs, **kwargs)
+            return callback
+
+        return trace
+    if not request_injection_processed:
+        module.Http.request = wrapper(module.Http.request)
+        request_injection_processed = True
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            callback = interceptor_sock_connect(fn, *args, **kwargs)
+            return callback
+        return trace
+
+    module.HTTPConnectionWithTimeout.connect = wrapper(module.HTTPConnectionWithTimeout.connect)
+    module.HTTPSConnectionWithTimeout.connect = wrapper(module.HTTPSConnectionWithTimeout.connect)

whatap/trace/mod/httpc/httpx.py

@@ -0,0 +1,62 @@
+from whatap.trace.trace_handler import trace_handler, async_trace_handler
+from whatap.trace.mod.httpc.util import transfer, \
+    interceptor_httpc_request, async_interceptor_httpc_request
+
+
+def instrument_httpx(module):
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            if len(args) >= 2 and hasattr(args[1], 'headers') and hasattr(args[1], 'url'):
+                request = args[1]
+                request.headers = transfer(request.headers)
+                httpc_url = str(request.url)
+
+            # 2. stream call: send(request=request, ...)
+            elif len(args) == 1 and 'request' in kwargs:
+                request = kwargs['request']
+                if hasattr(request, 'headers') and hasattr(request, 'url'):
+                    request.headers = transfer(request.headers)
+                    httpc_url = str(request.url)
+                else:
+                    httpc_url = "invalid_request_object"
+
+            # 3. 예상치 못한 패턴
+            else:
+                httpc_url = "httpx_unknown_pattern"
+
+            return interceptor_httpc_request(fn, httpc_url, *args, **kwargs)
+
+        return trace
+
+    def async_wrapper(fn):
+        @async_trace_handler(fn)
+        async def trace(*args, **kwargs):
+            if len(args) >= 2 and hasattr(args[1], 'headers') and hasattr(args[1], 'url'):
+                request = args[1]
+                request.headers = transfer(request.headers)
+                httpc_url = str(request.url)
+
+            # 2. stream call: send(request=request, ...)
+            elif len(args) == 1 and 'request' in kwargs:
+                request = kwargs['request']
+                if hasattr(request, 'headers') and hasattr(request, 'url'):
+                    request.headers = transfer(request.headers)
+                    httpc_url = str(request.url)
+                else:
+                    httpc_url = "invalid_request_object"
+
+            # 3. 예상치 못한 패턴
+            else:
+                httpc_url = "httpx_unknown_pattern"
+
+            return await async_interceptor_httpc_request(fn, httpc_url, *args, **kwargs)
+
+        return trace
+
+    if hasattr(module, 'Client') and hasattr(module.Client, 'send'):
+        module.Client.send = wrapper(module.Client.send)
+
+
+    if hasattr(module, 'AsyncClient') and hasattr(module.AsyncClient, 'send'):
+        module.AsyncClient.send = async_wrapper(module.AsyncClient.send)

whatap/trace/mod/httpc/requests.py

@@ -0,0 +1,20 @@
+from whatap.trace.trace_handler import trace_handler
+from whatap.trace.mod.httpc.util import transfer, \
+    interceptor_httpc_request
+
+
+def instrument_requests(module):
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            # set mtid header
+            args[1].headers = transfer(args[1].headers)
+            
+            # set httpc_url
+            httpc_url = args[1].url
+            callback = interceptor_httpc_request(fn, httpc_url, *args, **kwargs)
+            return callback
+        
+        return trace
+    
+    module.Session.send = wrapper(module.Session.send)

whatap/trace/mod/httpc/urllib3.py

@@ -0,0 +1,27 @@
+from whatap.trace.trace_handler import trace_handler
+from whatap.trace.mod.httpc.util import transfer, interceptor_httpc_request, \
+    interceptor_sock_connect
+
+
+def instrument_urllib3(module):
+    def wrapper(fn):
+        @trace_handler(fn)
+        def trace(*args, **kwargs):
+            # set mtid header
+            kwargs['headers'] = transfer(kwargs.get('headers', {}))
+            newargs = []
+            for arg in args:
+                if arg and isinstance(arg, dict) and 'host' in [x.lower() for x in arg.keys()]:
+                    arg = transfer(arg)
+                newargs.append(arg)
+            # set httpc_url
+            httpc_url = args[2]
+            if hasattr(args[0], 'host'):
+                httpc_url = getattr(args[0], 'host') + httpc_url
+            callback = interceptor_httpc_request(fn, httpc_url, *newargs, **kwargs)
+            return callback
+        
+        return trace
+    if hasattr(module, 'RequestMethods'):
+        module.RequestMethods.request = wrapper(module.RequestMethods.request)
+    

whatap/trace/mod/httpc/util.py

@@ -0,0 +1,388 @@
+import random
+
+from whatap import logging
+from whatap.conf.configure import Configure as conf
+from whatap.net.packet_type_enum import PacketTypeEnum
+import whatap.net.async_sender as async_sender
+from whatap.trace.trace_context_manager import TraceContextManager
+from whatap.trace.trace_error import interceptor_step_error
+from whatap.llm.definitions import URL_OPERATION_MAP
+from whatap.util.date_util import DateUtil
+from whatap.util.hexa32 import Hexa32
+
+
+def _update_api_status_stats(ctx, status_code):
+    try:
+        from whatap.counter.tasks.llm_stat_task import LlmStatTask
+        stat = LlmStatTask.get_stat('ApiStatusStat')
+        if not stat:
+            return
+        httpc_url = getattr(ctx, '_llm_httpc_url', '') or ''
+        url = ''
+        if httpc_url:
+            if '://' in httpc_url:
+                httpc_url = httpc_url.split('://', 1)[1]
+            if '/' in httpc_url:
+                url = '/' + httpc_url.split('/', 1)[1]
+        # operation_type / prompt_version 은 prompt_meta scope 우선 (사용자 라벨링).
+        # scope 미적용 시 default ('default', 'v1').
+        try:
+            from whatap.llm.prompt_meta import get_prompt_meta
+            op_type, prompt_version = get_prompt_meta()
+        except Exception:
+            op_type = getattr(ctx, '_llm_operation_type', None) or 'unknown'
+            prompt_version = 'v1'
+        stat.update_stats(
+            getattr(ctx, '_llm_model', None) or 'unknown',
+            getattr(ctx, '_llm_provider', None) or '',
+            op_type,
+            status_code,
+            url=url,
+            prompt_version=prompt_version,
+        )
+    except Exception:
+        pass
+
+
+def _extract_host(url):
+    """URL에서 호스트 추출"""
+    host = url
+    if '://' in host:
+        host = host.split('://', 1)[1]
+    return host.split('/', 1)[0]
+
+
+def _match_llm_api_host(httpc_url):
+    """whatap.conf의 llm_api_hosts 설정과 호스트 매칭"""
+    hosts = conf.llm_api_hosts
+    if not hosts:
+        return False
+    try:
+        host = _extract_host(httpc_url)
+        if not host:
+            return False
+        if isinstance(hosts, str):
+            hosts = [h.strip() for h in hosts.split(',') if h.strip()]
+        for h in hosts:
+            if host == h or host.endswith('.' + h):
+                return True
+    except Exception:
+        pass
+    return False
+
+
+def _detect_llm_api(ctx, httpc_url):
+    """LLM API 감지: 플래그 기반, URL 패턴 기반, 호스트 설정 기반"""
+    op_type = 'unknown'
+    for pattern, ot in URL_OPERATION_MAP:
+        if pattern in httpc_url:
+            op_type = ot
+            break
+    if getattr(ctx, '_llm_httpc_pending', False) and (op_type != 'unknown' or _match_llm_api_host(httpc_url)):
+        ctx.driver = 'LLM API'
+        ctx.is_llm = 1
+        ctx._llm_httpc_pending = False
+        ctx._llm_operation_type = op_type
+        ctx._llm_httpc_url = httpc_url
+
+        # HTTPC URL에서 호스트/경로 추출 → ctx 속성 + active stat 즉시 등록
+        try:
+            host = _extract_host(httpc_url)
+            url_path = ''
+            raw = httpc_url
+            if '://' in raw:
+                raw = raw.split('://', 1)[1]
+            if '/' in raw:
+                url_path = '/' + raw.split('/', 1)[1]
+            if host:
+                ctx._llm_provider = host
+                model = getattr(ctx, '_llm_model', None)
+                if model:
+                    from whatap.counter.tasks.llm_stat_task import LlmStatTask
+                    stat = LlmStatTask.get_stat('ActiveStat')
+                    if stat:
+                        stat.set_host(model, host, url_path)
+        except Exception:
+            pass
+        return True
+    return False
+
+
+def interceptor_httpc_request(fn, httpc_url, *args, **kwargs):
+    ctx = TraceContextManager.getLocalContext()
+    if not ctx or ctx.active_httpc_hash:
+        try:
+            return fn(*args, **kwargs)
+        except Exception as e:
+            interceptor_step_error(e, ctx=ctx)
+            raise
+
+    param = None
+    method = None
+    if httpc_url.find('?') > -1:
+        httpc_url, param = httpc_url.split('?')
+
+    is_llm = _detect_llm_api(ctx, httpc_url)
+    if not ctx.step_id:
+        seq = getattr(ctx, '_httpc_seq', 1)
+        ctx.step_id = seq
+        ctx._httpc_seq = seq + 1
+
+    start_time = DateUtil.nowSystem()
+    ctx.start_time = start_time
+    ctx.httpc_url = httpc_url
+    ctx.active_httpc_hash = ctx.httpc_url
+    ctx.active_httpc_start_time = DateUtil.nowSystem()
+
+    callback = None
+    try:
+        try:
+            callback = fn(*args, **kwargs)
+            return callback
+        except TypeError as e:
+            callback = fn(*args)
+            return callback
+    except Exception as e:
+        interceptor_step_error(e, ctx=ctx)
+        raise
+    finally:
+        try:
+            if ctx.driver == 'LLM API' and callback is not None:
+                status_code = getattr(callback, 'status_code', None)
+                if status_code:
+                    _update_api_status_stats(ctx, status_code)
+                    if status_code >= 400:
+                        try:
+                            read_fn = getattr(callback, 'read', None)
+                            if callable(read_fn):
+                                try:
+                                    read_fn()
+                                except Exception:
+                                    pass
+                            error_text = callback.text or ''
+                        except Exception as e:
+                            logging.warning('[HTTPC] LLM API error text read failed: %s' % e, extra={'id': 'HTTPC001'})
+                            error_text = ''
+                        try:
+                            interceptor_step_error(
+                                Exception('[LLM API Error] HTTP {} {}'.format(status_code, error_text)),
+                                ctx=ctx
+                            )
+                        except Exception as e:
+                            logging.warning('[HTTPC] interceptor_step_error failed: %s' % e, extra={'id': 'HTTPC002'})
+        except Exception as e:
+            logging.warning('[HTTPC] LLM API error handling failed: %s' % e, extra={'id': 'HTTPC003'})
+
+        datas = [ctx.httpc_url, ctx.step_id, ctx.driver]
+        ctx.elapsed = DateUtil.nowSystem() - start_time
+        async_sender.send_packet(PacketTypeEnum.TX_HTTPC, ctx, datas)
+
+        if conf.profile_http_parameter_enabled and len(args) > 1:
+            try:
+                if type(args[1]) == dict:
+                    param = (args[1].body if 'body' in args[1] else args[1]) or param
+                    method = args[1].method if 'method' in args[1] else args[1]
+
+                if param:
+                    datas = ['HTTP-PARAMETERS', method, param]
+                    ctx.start_time = DateUtil.nowSystem()
+                    async_sender.send_packet(PacketTypeEnum.TX_SECURE_MSG, ctx, datas)
+            except Exception as _e:
+                # finally 가 어떤 일이 있어도 return 값을 exception 으로 둔갑시키지 않도록 방어
+                logging.warning(
+                    '[HTTPC] profile_http_parameter_enabled extract failed: %s' % _e,
+                    extra={'id': 'HTTPC004'}
+                )
+
+        if is_llm:
+            ctx._llm_step_id = ctx.step_id
+        ctx.active_httpc_hash = 0
+        ctx.active_httpc_start_time = 0
+        ctx.httpc_url = None
+        ctx.step_id = 0
+        ctx.driver = ''
+
+
+async def async_interceptor_httpc_request(fn, httpc_url, *args, **kwargs):
+    ctx = TraceContextManager.getLocalContext()
+    if not ctx or ctx.active_httpc_hash:
+        try:
+            return await fn(*args, **kwargs)
+        except Exception as e:
+            interceptor_step_error(e, ctx=ctx)
+            raise
+
+    param = None
+    method = None
+    if httpc_url.find('?') > -1:
+        httpc_url, param = httpc_url.split('?')
+
+    is_llm = _detect_llm_api(ctx, httpc_url)
+    if not ctx.step_id:
+        seq = getattr(ctx, '_httpc_seq', 1)
+        ctx.step_id = seq
+        ctx._httpc_seq = seq + 1
+
+    start_time = DateUtil.nowSystem()
+    ctx.start_time = start_time
+    ctx.httpc_url = httpc_url
+    ctx.active_httpc_hash = ctx.httpc_url
+    ctx.active_httpc_start_time = DateUtil.nowSystem()
+
+    callback = None
+    try:
+        callback = await fn(*args, **kwargs)
+        return callback
+    except Exception as e:
+        interceptor_step_error(e, ctx=ctx)
+        raise
+    finally:
+        try:
+            if ctx.driver == 'LLM API' and callback is not None:
+                status_code = getattr(callback, 'status_code', None)
+                if status_code:
+                    _update_api_status_stats(ctx, status_code)
+                    if status_code >= 400:
+                        try:
+                            aread_fn = getattr(callback, 'aread', None)
+                            if callable(aread_fn):
+                                try:
+                                    await aread_fn()
+                                except Exception:
+                                    pass
+                            else:
+                                read_fn = getattr(callback, 'read', None)
+                                if callable(read_fn):
+                                    try:
+                                        read_fn()
+                                    except Exception:
+                                        pass
+                            error_text = callback.text or ''
+                        except Exception as e:
+                            logging.warning('[HTTPC] LLM API error text read failed: %s' % e, extra={'id': 'HTTPC001'})
+                            error_text = ''
+                        try:
+                            interceptor_step_error(
+                                Exception('[LLM API Error] HTTP {} {}'.format(status_code, error_text)),
+                                ctx=ctx
+                            )
+                        except Exception as e:
+                            logging.warning('[HTTPC] interceptor_step_error failed: %s' % e, extra={'id': 'HTTPC002'})
+        except Exception as e:
+            logging.warning('[HTTPC] LLM API error handling failed: %s' % e, extra={'id': 'HTTPC003'})
+
+        datas = [ctx.httpc_url, ctx.step_id, ctx.driver]
+        ctx.elapsed = DateUtil.nowSystem() - start_time
+        async_sender.send_packet(PacketTypeEnum.TX_HTTPC, ctx, datas)
+
+        if conf.profile_http_parameter_enabled and len(args) > 1:
+            try:
+                if type(args[1]) == dict:
+                    param = (args[1].body if 'body' in args[1] else args[1]) or param
+                    method = args[1].method if 'method' in args[1] else args[1]
+
+                if param:
+                    datas = ['HTTP-PARAMETERS', method, param]
+                    ctx.start_time = DateUtil.nowSystem()
+                    async_sender.send_packet(PacketTypeEnum.TX_SECURE_MSG, ctx, datas)
+            except Exception as _e:
+                # finally 가 어떤 일이 있어도 return 값을 exception 으로 둔갑시키지 않도록 방어
+                logging.warning(
+                    '[HTTPC] profile_http_parameter_enabled extract failed: %s' % _e,
+                    extra={'id': 'HTTPC004'}
+                )
+
+        if is_llm:
+            ctx._llm_step_id = ctx.step_id
+        ctx.active_httpc_hash = 0
+        ctx.active_httpc_start_time = 0
+        ctx.httpc_url = None
+        ctx.step_id = 0
+        ctx.driver = ''
+
+
+def interceptor_sock_connect(fn, *args, **kwargs):
+    ctx = TraceContextManager.getLocalContext()
+    if not ctx:
+        return fn(*args, **kwargs)
+
+    try:
+        ctx.socket_connecting = True
+        callback = fn(*args, **kwargs)
+        return callback
+    except Exception as e:
+        interceptor_step_error(e)
+    finally:
+        ctx.socket_connecting = False
+
+
+def inter_tx_trace_auto_on(ctx):
+    try:
+        if isinstance(conf.mtrace_rate, str):
+            conf.mtrace_rate = int(conf.mtrace_rate)
+    except ValueError:
+        conf.mtrace_rate = 0
+    finally:
+        if conf.mtrace_rate <= 0 or ctx.httpc_checked or ctx.mtid != 0:
+            return
+
+        ctx.httpc_checked = True
+
+        try:
+            inter_tx_trace_auto_on.check_seq += 1
+        except AttributeError:
+            inter_tx_trace_auto_on.check_seq = 1
+        finally:
+            check_seq = inter_tx_trace_auto_on.check_seq
+
+            rate = int(conf.mtrace_rate / 10)
+            if rate == 10:
+                ctx.mtid = TraceContextManager.getId()
+            elif rate == 9:
+                if check_seq % 10 != 0:
+                    ctx.mtid = TraceContextManager.getId()
+            elif rate == 8:
+                if check_seq % 5 != 0:
+                    ctx.mtid = TraceContextManager.getId()
+            elif rate == 7:
+                if check_seq % 4 != 0:
+                    ctx.mtid = TraceContextManager.getId()
+            elif rate == 6:
+                if check_seq % 3 != 0:
+                    ctx.mtid = TraceContextManager.getId()
+            elif rate == 5:
+                if check_seq % 2 == 0:
+                    ctx.mtid = TraceContextManager.getId()
+            elif rate == 4:
+                if check_seq % 3 == 0 or check_seq % 5 == 0:
+                    ctx.mtid = TraceContextManager.getId()
+            elif rate == 3:
+                if check_seq % 4 == 0 or check_seq % 5 == 0:
+                    ctx.mtid = TraceContextManager.getId()
+            elif rate == 2:
+                if check_seq % 5 == 0:
+                    ctx.mtid = TraceContextManager.getId()
+            elif rate == 1:
+                if check_seq % 10 == 0:
+                    ctx.mtid = TraceContextManager.getId()
+
+
+def transfer(headers):
+    ctx = TraceContextManager.getLocalContext()
+
+    if not ctx.mtid:
+        inter_tx_trace_auto_on(ctx)
+
+    if ctx.mtid:
+        if not ctx.step_id:
+            seq = getattr(ctx, '_httpc_seq', 1)
+            ctx.step_id = seq
+            ctx._httpc_seq = seq + 1
+        headers[conf._trace_mtrace_caller_key] = ctx.transfer()
+        if conf.stat_mtrace_enabled:
+            headers[conf._trace_mtrace_info_key] = ctx.transferInfo()
+
+        ctx.mcallee = TraceContextManager.getId()
+        headers[conf._trace_mtrace_callee_key] = Hexa32.toString32(ctx.mcallee)
+        headers[conf._trace_mtrace_caller_poid_key] = ctx.transferPOID()
+    return headers