whatap-python 2.1.0__py3-none-any.whl

whatap/net/udp_session.py

@@ -0,0 +1,394 @@
+import threading
+
+import sys, os
+import traceback
+import time
+import logging
+import socket
+import errno
+import socket
+from . import stackhelper
+
+from whatap.conf.configure import Configure as conf
+from whatap.io import DataOutputX
+from whatap.net.packet_enum import PacketEnum
+from whatap.net.packet_type_enum import PacketTypeEnum
+from whatap.net.param_def import ParamDef
+from whatap.trace import TraceContextManager
+try:
+    from importlib.metadata import distributions as _distributions
+    def _get_installed_packages():
+        return ', '.join('{} {}'.format(d.metadata['Name'], d.version) for d in _distributions())
+except ImportError:
+    import pkg_resources
+    def _get_installed_packages():
+        return ', '.join(str(d) for d in pkg_resources.working_set)
+
+class UdpSession(object):
+    s = None
+    llm_s = None
+    buffer_arr = []
+    llm_buffer_arr = []
+    thread_lock = threading.Lock()
+    llm_thread_lock = threading.Lock()
+    socket_lock = threading.Lock()
+    read_timeout = 5
+
+    @classmethod
+    def udp(cls):
+        
+        home = 'WHATAP_HOME'
+        whatap_home = os.environ.get(home,'')
+
+        # Unix sockets are not available on Windows
+        if conf.unix_socket_enabled == True and sys.platform != 'win32':
+            unix_socket_path = os.path.join(whatap_home, conf.unix_socket)
+            while not os.path.exists(unix_socket_path):
+                time.sleep(0.1)
+            with cls.socket_lock:
+                tmp_socket_path = os.path.join(whatap_home,'run', 'whatap.{}.sock'.format(os.getpid()))
+                if os.path.exists(tmp_socket_path):
+                    os.remove(tmp_socket_path)
+                prefix=os.path.split(tmp_socket_path)[0]
+                if not os.path.exists(prefix):
+                    os.makedirs(prefix)
+                sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, PacketEnum.PACKET_SEND_BUFFER_SIZE)
+                sock.bind(tmp_socket_path)
+                sock.connect(unix_socket_path)
+                sock.settimeout(cls.read_timeout)
+                if cls.s:
+                    try:
+                        cls.s.close()
+                    except Exception as e:
+                        print(e)
+                cls.s = sock
+        else:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, PacketEnum.PACKET_SEND_BUFFER_SIZE)
+            sock.connect((PacketEnum.SERVER, int(conf.net_udp_port)))
+            sock.settimeout(cls.read_timeout)
+            if cls.s:
+                try:
+                    cls.s.close()
+                except Exception as e:
+                    print(e)
+            cls.s = sock
+        return cls.s
+
+    @classmethod
+    def udp_llm(cls):
+        """Initialize UDP socket for LLM Go Agent."""
+        llm_port = int(getattr(conf, 'llm_net_udp_port', 0))
+        if not llm_port:
+            return None
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, PacketEnum.PACKET_SEND_BUFFER_SIZE)
+            sock.connect((PacketEnum.SERVER, llm_port))
+            sock.settimeout(cls.read_timeout)
+            if cls.llm_s:
+                try:
+                    cls.llm_s.close()
+                except Exception:
+                    pass
+            cls.llm_s = sock
+        except Exception as e:
+            logging.debug('WHATAP: LLM UDP init failed: %s' % e, extra={'id': 'WA920'}, exc_info=True)
+        return cls.llm_s
+
+    @classmethod
+    def send_packet(cls, packet_type, ctx, datas=[]):
+        import whatap.net.udp_thread
+        try:
+            if sum( [ len(x) for x in cls.buffer_arr]) + sum( [len(x) if isinstance(x, (str, bytes)) else 4 for x in datas ] ) +PacketEnum.PACKET_BODY_REQUIRED_SIZE + 1 > PacketEnum.PACKET_BUFFER_SIZE:
+                cls.send(packet_type, ctx)
+
+            if not datas:
+                return
+
+            dout = DataOutputX()
+            start_header_buffer_size = dout.size()
+
+            # header
+            dout.writeByte(packet_type)
+            dout.writeInt(PacketEnum.PACKET_VERSION)
+            dout.writeInt(0)
+
+            # body
+            start_body_buffer_size = dout.size()
+
+            if ctx:
+                dout.writeLong(ctx.id)
+                dout.writeLong(ctx.start_time)
+                dout.writeInt(ctx.elapsed)
+                dout.writeLong(ctx.getCpuTime())
+                dout.writeLong(ctx.getMemory())
+                dout.writeInt(os.getpid())
+                dout.writeLong(ctx.thread_id)
+
+            diff = PacketEnum.PACKET_BUFFER_SIZE \
+                   - sum( [len(x) for x  in cls.buffer_arr]) - (PacketEnum.PACKET_BODY_REQUIRED_SIZE + 1)
+            for data in datas:
+                data = str(data)[:PacketEnum.DATA_SIZE_LIMIT]
+                diff -= len(data)
+                if diff < 0:
+                    data = ' '
+                    logging.debug('message too long.', extra={'id': 'WA999'},
+                                  exc_info=False)
+                dout.writeUTF(data)
+
+            with cls.thread_lock:
+                dout.writeToPos(
+                    start_header_buffer_size + PacketEnum.PACKET_HEADER_LEN_POS,
+                    dout.size() - start_body_buffer_size)
+
+                cls.buffer_arr.append(dout.buffer.getvalue())
+
+                if not ctx or packet_type == PacketTypeEnum.TX_START \
+                        or packet_type == PacketTypeEnum.TX_END:
+                    cls.send(packet_type, ctx)
+        except Exception as e:
+            logging.debug(e, extra={'id': 'WA913'}, exc_info=True)
+
+    @classmethod
+    def send(cls, packet_type, ctx):
+        if not cls.buffer_arr:
+            if packet_type == PacketTypeEnum.TX_END and ctx:
+                TraceContextManager.end(ctx.id)
+            return
+
+        sendbuf = b''.join(cls.buffer_arr)
+
+        # APM Go Agent 전송
+        if cls.s:
+            try:
+                cls.s.send(sendbuf)
+            except ConnectionRefusedError as e:
+                logging.debug(e, extra={'id': 'WA911'}, exc_info=True)
+                cls.udp()
+            except Exception as e:
+                logging.debug(e, extra={'id': 'WA912'}, exc_info=True)
+                if len(cls.buffer_arr) > 1:
+                    overflow = cls.buffer_arr.pop()
+                    cls.send(packet_type, ctx)
+                    cls.buffer_arr.append(overflow)
+                    return
+
+        # LLM Go Agent에도 APM 트랜잭션 데이터 전송 (tx correlation용)
+        if not cls.llm_s and getattr(conf, 'llm_enabled', False):
+            llm_port = int(getattr(conf, 'llm_net_udp_port', 0))
+            if llm_port:
+                cls.udp_llm()
+        if cls.llm_s:
+            try:
+                cls.llm_s.send(sendbuf)
+            except ConnectionRefusedError:
+                cls.udp_llm()
+            except Exception:
+                pass
+
+        if packet_type == PacketTypeEnum.TX_END and ctx:
+            TraceContextManager.end(ctx.id)
+        cls.buffer_arr = []
+
+    @classmethod
+    def get(cls):
+        from whatap.conf.configure import Configure as conf
+        sockets = [s for s in (cls.s, cls.llm_s) if s]
+        if not sockets:
+            return
+        try:
+            import select
+            readable, _, _ = select.select(sockets, [], [], cls.read_timeout)
+            for sock in readable:
+                try:
+                    received = sock.recv(1024)
+                    if received:
+                        cls.handle(received.decode().split(','))
+                except socket.error:
+                    pass
+        except socket.error as serr:
+            if serr.errno == errno.EAGAIN:
+                conf.init(display=False)
+            else:
+                raise serr
+    
+    @classmethod    
+    def getThreadStackFaultHandler(cls, thread_id):
+        stackhelper.StackParser.parse()
+        for line, line_num, method_name in stackhelper.StackParser.find(thread_id):
+            if 'whatap' + os.sep + 'trace' in line or 'threading.py' in line:
+                continue
+            yield '{} ({}:{})\n'.format(method_name, line, line_num)
+
+    
+    @classmethod    
+    def getThreadStack(cls, frame):
+        for stack in traceback.extract_stack(frame):
+            line = stack[0]
+            line_num = stack[1]
+            method_name = stack[2]
+
+            if 'whatap' + os.sep + 'trace' in line or 'threading.py' in line:
+                continue
+            yield '{} ({}:{})\n'.format(method_name, line, line_num)
+
+    
+    @classmethod
+    def handle(cls, received):
+        param_id, request, extra = tuple(received)
+        param_id = int(param_id)
+        thread_id = 0
+        pid = 0
+        if(extra != ''):
+            extra_datas = extra.replace(' ', ', ').split(', ')
+            thread_id, pid = TraceContextManager.parseThreadId(int(extra_datas[1]))
+
+        data = ''
+        if not param_id:
+            # active stack pack
+            datas = []
+            data = extra.replace(' ', ', ')
+            
+            frame = sys._current_frames().get(thread_id)
+            if not frame:
+                return
+
+            if conf.threadstack_faulthandler:
+                for stack in cls.getThreadStackFaultHandler(thread_id):
+                    data += stack
+                datas.append(data)
+            else:
+                for stack in cls.getThreadStack(frame):
+                    data += stack
+                datas.append(data)
+
+            cls.send_packet(PacketTypeEnum.ACTIVE_STACK, None, datas)
+        else:
+            # param pack
+            # format: "[packetType], [ctx], [datas: xxxx xxxx xxxx]"
+            datas = [str(param_id), request]
+            if param_id == ParamDef.MODULE_DEPENDENCY:
+                
+                data = _get_installed_packages()
+                datas.append(data)
+            elif param_id == ParamDef.GET_ACTIVE_TRANSACTION_DETAIL:
+                data = extra.replace(' ', ', ')
+
+                frame = sys._current_frames().get(thread_id)
+                if not frame:
+                    return
+
+                for stack in traceback.extract_stack(frame):
+                    line = stack[0]
+                    line_num = stack[1]
+                    method_name = stack[2]
+
+                    if line.find('whatap/trace') > -1:
+                        continue
+                    data += '{} ({}:{})\n'.format(method_name, line, line_num)
+
+                datas.append(data)
+            elif param_id == ParamDef.GET_ACTIVE_STATS:
+                stats, httpc_detail = TraceContextManager.getActiveStats()
+                result = ','.join([str(x) for x in stats])
+                for host_hash, d in httpc_detail.items():
+                    result += '|{}:{}:{}:{}:{}'.format(
+                        host_hash, d['actx'],
+                        d['acts'][0], d['acts'][1], d['acts'][2])
+                # LLM meter count
+                from whatap.conf.configure import Configure as conf
+                if getattr(conf, 'llm_enabled', False):
+                    from whatap.llm.stats.meter import Meter
+                    llm_count = Meter.get_and_reset()
+                    if llm_count > 0:
+                        result += '|LLM:{}'.format(llm_count)
+                datas = [result]
+                cls.send_packet(PacketTypeEnum.ACTIVE_STATS, None, datas)
+                return
+            elif param_id == ParamDef.GET_DBCONN_POOL:
+                
+                stats = TraceContextManager.getDBConnPool()
+                if stats:
+                    pid = os.getpid()
+                    strdbpool = '{}|{{}}|{{}}|{{}}'.format(pid)
+                    datas = [','.join([strdbpool.format( k,act,inact) for k,(act,inact) in stats.items()])]
+                    cls.send_packet(PacketTypeEnum.DBCONN_POOL, None, datas)
+
+                return
+
+            cls.send_packet(PacketTypeEnum.TX_PARAM, None, datas)
+
+    @classmethod
+    def send_relaypack(cls, packbytes):
+        packet_type = PacketTypeEnum.RELAY_PACK
+        import whatap.net.udp_thread
+        try:
+            if sum( [ len(x) for x in cls.buffer_arr]) + PacketEnum.PACKET_BODY_REQUIRED_SIZE + 1 > PacketEnum.PACKET_BUFFER_SIZE:
+                cls.send(packet_type, None)
+
+            if not packbytes:
+                return
+
+            dout = DataOutputX()
+            
+            # header
+            dout.writeByte(packet_type)
+            dout.writeInt(PacketEnum.PACKET_VERSION)
+            dout.writeIntBytes(packbytes)
+
+            with cls.thread_lock:
+                cls.buffer_arr.append(dout.toByteArray())
+        except Exception as e:
+            print(e)
+            logging.debug(e, extra={'id': 'WA262'}, exc_info=True)
+
+    @classmethod
+    def send_llm_relaypack(cls, packbytes):
+        """LLM 데이터를 LLM Go Agent 소켓으로 버퍼링 후 전송. 48KB 누적 시 자동 flush.
+        명시적 flush는 flush_llm_buffer()를 사용한다."""
+        if not packbytes:
+            return
+        if not cls.llm_s:
+            llm_port = int(getattr(conf, 'llm_net_udp_port', 0))
+            if llm_port:
+                cls.udp_llm()
+        if not cls.llm_s:
+            return
+        try:
+            dout = DataOutputX()
+            dout.writeByte(PacketTypeEnum.RELAY_PACK)
+            dout.writeInt(PacketEnum.PACKET_VERSION)
+            dout.writeIntBytes(packbytes)
+            entry = dout.toByteArray()
+
+            with cls.llm_thread_lock:
+                current_size = sum(len(x) for x in cls.llm_buffer_arr)
+                if cls.llm_buffer_arr and current_size + len(entry) > PacketEnum.PACKET_BUFFER_SIZE:
+                    cls._flush_llm_buffer_locked()
+                cls.llm_buffer_arr.append(entry)
+        except Exception as e:
+            logging.debug(e, extra={'id': 'WA921'}, exc_info=True)
+
+    @classmethod
+    def flush_llm_buffer(cls):
+        """LLM 버퍼에 남아있는 데이터를 즉시 전송한다. idle/트랜잭션 경계에서 호출."""
+        if not cls.llm_s:
+            return
+        with cls.llm_thread_lock:
+            cls._flush_llm_buffer_locked()
+
+    @classmethod
+    def _flush_llm_buffer_locked(cls):
+        """llm_thread_lock을 이미 보유한 상태에서 호출되어야 한다."""
+        if not cls.llm_buffer_arr:
+            return
+        sendbuf = b''.join(cls.llm_buffer_arr)
+        cls.llm_buffer_arr = []
+        try:
+            cls.llm_s.send(sendbuf)
+        except ConnectionRefusedError:
+            cls.udp_llm()
+        except Exception as e:
+            logging.debug(e, extra={'id': 'WA921'}, exc_info=True)

whatap/net/udp_thread.py

@@ -0,0 +1,54 @@
+import threading, time
+import logging as logging_module
+
+logging = logging_module.getLogger(__name__)
+
+def startWhatapThread():
+    def __whatap_jobs():
+        from whatap.conf.configure import Configure as conf
+        from whatap.net.udp_session import UdpSession
+        UdpSession.read_timeout = conf.session_read_timeout
+        UdpSession.udp()
+        # LLM Go Agent UDP 소켓 초기화
+        if getattr(conf, 'llm_enabled', False):
+            llm_port = int(getattr(conf, 'llm_net_udp_port', 0))
+            if llm_port:
+                UdpSession.udp_llm()
+
+        def __udp_recv():
+            UdpSession.get()
+
+        def __config_load():
+            now = time.time()
+            if now - conf.last_loaded > conf.init_load_interval / 1000:
+                conf.init(False)
+
+        while True:
+            for job in (__udp_recv, __config_load):
+                try:
+                    job()
+                    time.sleep(0)
+                except Exception as e:
+                    if conf.debug:
+                        logging.debug(e, extra={'id': 'WA914'}, exc_info=True)
+                    time.sleep(0.05)
+
+    t = threading.Thread(target=__whatap_jobs)
+    t.daemon = True
+    t.start()
+
+
+    def __debug():
+        from whatap.conf.configure import Configure as conf
+        import whatap.util.debug_util as debug_util
+        while True:
+            
+            if conf.debug_stack_enabled:
+                debug_util.printAllStack()
+            time.sleep(10)
+    t = threading.Thread(target=__debug)
+    t.daemon = True
+    t.start()
+
+
+startWhatapThread()

whatap/pack/logSinkPack.py

@@ -0,0 +1,77 @@
+from whatap.pack.pack import Pack
+from whatap.pack.pack_enum import PackEnum
+from whatap.value.map_value import MapValue
+from whatap.io.data_outputx import DataOutputX
+from whatap.util.hash_util import HashUtil as hashutil
+
+class LogSinkPack(Pack):
+    def __init__(self):
+        super(LogSinkPack, self).__init__()
+        self.tagHash = 0 #  int64
+        self.tags = MapValue()
+        self.line = 0 #     int64
+        self.content = None # string
+        self.fields = MapValue()   
+        
+    def getPackType(self):
+        return PackEnum.LOGSINK
+
+    def write(self, dout):
+        super(LogSinkPack, self).write(dout)
+
+        dout.writeByte(0)
+        dout.writeText(self.Category)
+        if self.tagHash == 0 and self.tags.size() > 0:
+            tagHash, tagBytes = self.resetTagHash()
+            self.tagHash = tagHash
+            dout.writeDecimal(tagHash)
+            dout.write(tagBytes)
+        else:
+            dout.writeDecimal(self.tagHash)
+            dout.writeValue(self.tags)
+        dout.writeDecimal(self.line)
+        dout.writeText(self.content)
+        if self.fields and self.fields.size() > 0:
+            dout.writeBoolean(True)
+            dout.writeValue(self.fields)
+        else:
+            dout.writeBoolean(False)
+
+    def resetTagHash(self) :
+        dout = DataOutputX()
+        dout.writeValue(self.tags)
+        tagBytes = dout.toByteArray()
+        tagHash = hashutil.hash(tagBytes)
+        return tagHash, tagBytes
+        
+    def read(self, din):
+        super(LogSinkPack, self).read(din)
+        din.readByte()
+        self.Category = din.readText()
+        self.tagHash = din.ReadDecimal()
+        self.tags = din.readValue()
+        self.line = din.readDecimal()
+        self.content = din.readText()
+        if din.readBool():
+            self.fields = din.readValue()
+        
+        return self
+
+
+def getLogSinkPack( t = 0,
+        category = None,
+        tags = {},
+        fields = {},
+        line=0,
+        content = None):
+    pack = LogSinkPack()
+    pack.time = t
+    pack.Category = category
+    for k, v in tags.items():
+        pack.tags.putString(k, str(v))
+    for k, v in fields.items():
+        pack.fields.putString(k, str(v))
+    pack.line = line
+    pack.content = content
+    return pack
+    

whatap/pack/pack.py

@@ -0,0 +1,34 @@
+from datetime import datetime
+
+
+class Pack(object):
+    def __init__(self):
+        self.pcode = 0
+        self.oid = 0
+        self.time = 0
+    
+    def __repr__(self):
+        to_str = '{0}: '.format(type(self).__name__)
+        
+        data = self.__dict__.copy()
+        for key, value in data.items():
+            if key == 'time':
+                value = datetime.fromtimestamp(value / 1000) \
+                    .strftime('%Y-%m-%d %H:%M:%S')
+                
+            to_str += '{0}={1}, '.format(key, value)
+        return to_str
+    
+    def getPackType(self):
+        return 0
+    
+    def write(self, dout):
+        dout.writeDecimal(self.pcode)
+        dout.writeInt(self.oid)
+        dout.writeLong(self.time)
+    
+    def read(self, din):
+        self.pcode = din.readDecimal()
+        self.oid = din.readInt()
+        self.time = din.readLong()
+        return self

whatap/pack/pack_enum.py

@@ -0,0 +1,41 @@
+class PackEnum(object):
+    PARAMETER = 0x0100
+    COUNTER = 0x0200
+    COUNTER_1 = 0x0201
+    PROFILE = 0x0300
+    ACTIVESTACK = 0x0400
+    TEXT = 0x0700
+    ERROR_SNAP = 0x0800
+    ERROR_SNAP_1 = 0x0801
+    REALTIME_USER = 0x0f00
+    REALTIME_USER_1 = 0x0f01
+
+    STAT_SERVICE = 0x0900
+    STAT_GENERAL = 0x0910
+    STAT_SQL = 0x0a00
+    STAT_HTTPC = 0x0b00
+    STAT_ERROR = 0x0c00
+    STAT_METHOD = 0x0e00
+    STAT_TOP_SERVICE = 0x1000
+    STAT_REMOTE_IP = 0x1100
+    STAT_USER_AGENT = 0x1200
+    STAT_USER_AGENT_1 = 0x1201
+
+    EVENT = 0x1400
+    HITMAP = 0x1500
+    HITMAP_1 = 0x1501
+    EXTENSION = 0x1600
+
+    COMPOSITE = 0x1700
+
+    LOGSINK = 0x170a
+    TAG_COUNT = 0x1601
+    @staticmethod
+    def create(p):
+        return None
+
+class EventLevel(object):
+    FATAL = 30
+    WARNING = 20
+    INFO = 10
+    NONE = 0

whatap/pack/tagCountPack.py

@@ -0,0 +1,61 @@
+from whatap.pack.pack import Pack
+from whatap.pack.pack_enum import PackEnum
+from whatap.value.map_value import MapValue
+from whatap.io.data_outputx import DataOutputX
+from whatap.util.hash_util import HashUtil as hashutil
+
+class TagCountPack(Pack):
+    def __init__(self):
+        super(TagCountPack, self).__init__()
+        self.tagHash = 0 #  int64
+        self.tags = MapValue()
+        self.fields = MapValue()   
+        
+    def getPackType(self):
+        return PackEnum.TAG_COUNT
+
+    def write(self, dout):
+        super(TagCountPack, self).write(dout)
+
+        dout.writeByte(0)
+        dout.writeText(self.Category)
+        if self.tagHash == 0 and self.tags.size() > 0:
+            tagHash, tagBytes = self.resetTagHash()
+            self.tagHash = tagHash
+            dout.writeDecimal(tagHash)
+            dout.write(tagBytes)
+        else:
+            dout.writeDecimal(self.tagHash)
+            dout.writeValue(self.tags)
+        dout.writeValue(self.fields)
+        
+    def resetTagHash(self) :
+        dout = DataOutputX()
+        dout.writeValue(self.tags)
+        tagBytes = dout.toByteArray()
+        tagHash = hashutil.hash(tagBytes)
+        return tagHash, tagBytes
+        
+    def read(self, din):
+        super(TagCountPack, self).read(din)
+        din.readByte()
+        self.Category = din.readText()
+        self.tagHash = din.ReadDecimal()
+        self.tags = din.readValue()
+        self.fields = din.readValue()
+        
+        return self
+
+def getTagCountPack( t = 0,
+        category = None,
+        tags = {},
+        fields = {}):
+    pack = TagCountPack()
+    pack.time = t
+    pack.Category = category
+    for k, v in tags.items():
+        pack.tags.putAuto(k, v)
+    for k, v in fields.items():
+        pack.fields.putAuto(k, v)
+    return pack
+