whatap-python 2.1.0__py3-none-any.whl

whatap/trace/trace_import.py

@@ -0,0 +1,91 @@
+import sys
+
+try:
+    from importlib.util import find_spec
+    from importlib.abc import MetaPathFinder
+except ImportError:
+    find_spec = None
+
+from whatap.trace.trace_module_definition import IMPORT_HOOKS, PLUGIN
+
+from whatap import logging
+from whatap.conf.configure import Configure as conf
+
+
+def load_module(module, fullname):
+    # if conf.dev:
+    #     logging.debug(fullname)
+
+    try:
+        if fullname in IMPORT_HOOKS:
+            wfullname = IMPORT_HOOKS[fullname]['module']
+            wmodule = sys.modules.get(wfullname)
+            if not wmodule:
+                __import__(wfullname)
+                wmodule = sys.modules.get(wfullname)
+
+            if wfullname.endswith('plugin'):
+                module = {'module': module, 'class_defs': PLUGIN[fullname]}
+
+            getattr(wmodule,
+                    IMPORT_HOOKS[fullname]['def'])(module)
+            logging.info("successfully injected %s as %s", fullname, wfullname)
+        else:
+            if conf.debug:
+                logging.info("non-injected module %s ", fullname)
+    except Exception as e:
+        logging.debug(e, exc_info=True)
+    finally:
+        return module
+
+
+class _ImportHookLoader(object):
+    def load_module(self, fullname):
+        module = sys.modules[fullname]
+        return load_module(module, fullname)
+
+
+class _ImportHookChainedLoader(object):
+    def __init__(self, loader):
+        self.loader = loader
+
+    def load_module(self, fullname):
+        module = self.loader.load_module(fullname)
+        return load_module(module, fullname)
+
+
+class ImportFinder(MetaPathFinder):
+    def __init__(self):
+        self._hooks = {}
+
+    def find_spec(self, fullname, path=None, target=None):
+        if fullname not in IMPORT_HOOKS \
+                or fullname.startswith('whatap') \
+                or fullname.startswith('pip'):
+
+            if conf.debug:
+                logging.info("non-injected module %s ", fullname)
+
+            return None
+
+        if fullname in self._hooks:
+            return None
+        self._hooks[fullname] = True
+
+        # if conf.dev:
+        #     logging.debug(fullname)
+
+        try:
+            if find_spec:
+                spec = find_spec(fullname, path)
+                if spec and spec.loader:
+                    spec.loader = _ImportHookChainedLoader(spec.loader)
+                    return spec
+            else:
+                __import__(fullname)
+                return _ImportHookLoader()
+
+        except Exception as e:
+            if conf.dev:
+                print(e)
+            return

whatap/trace/trace_module_definition.py

@@ -0,0 +1,156 @@
+PLUGIN = {}
+IMPORT_HOOKS = {}
+
+
+
+DEFINITION = {
+    'plugin': [
+        ('', 'instrument_plugin'),
+    ],
+    'standalone': [
+        ('', 'instrument_standalone_single'),
+        ('', 'instrument_standalone_multiple')
+    ],
+    'email.smtp': [
+        ('smtplib', 'instrument_smtp'),
+    ],
+    'amqp.pika': [
+        ('pika.channel', 'instrument_pika'),
+    ],
+    'llm.openai': [
+        ('openai', 'instrument_openai', 'whatap.llm.providers.openai')
+    ],
+    'llm.anthropic': [
+        ('anthropic', 'instrument_anthropic', 'whatap.llm.providers.anthropic')
+    ],
+    'logging': [
+        ('logging.handlers', 'instrument_logging'),
+        ('loguru._handler', 'instrument_loguru'),
+    ],
+
+
+    'httpc.httplib': [
+        ('httplib', 'instrument_httplib'),
+        ('http.client', 'instrument_httplib'),
+        ('httplib2', 'instrument_httplib2'),
+    ],
+    'httpc.requests': [
+        ('requests.sessions', 'instrument_requests'),
+    ],
+    'httpc.urllib3': [
+        ('urllib3.request', 'instrument_urllib3'),
+    ],
+    'httpc.django': [
+        ('revproxy.views', 'instrument_revproxy_views'),
+    ],
+    'httpc.httpx' : [
+        ('httpx', 'instrument_httpx')
+    ],
+
+
+    'database.mysql': [
+        ('MySQLdb', 'instrument_MySQLdb'),
+        ('MySQLdb.cursors', 'instrument_MySQLdb_cursors'),
+        ('pymysql', 'instrument_pymysql'),
+        ('pymysql.cursors', 'instrument_pymysql_cursors'),
+    ],
+    'database.psycopg2': [
+        ('psycopg2', 'instrument_psycopg2'),
+        ('psycopg2._psycopg', 'instrument_psycopg2_connection'),
+        ('psycopg2.extensions', 'instrument_psycopg2_extensions'),
+    ],
+    'database.psycopg3': [
+        ('psycopg', 'instrument_psycopg'),
+        ('psycopg_pool', 'instrument_psycopg_pool'),
+    ],
+
+    'database.neo4j': [
+        ('neo4j', 'instrument_neo4j')
+    ],
+    'database.sqlite3': [
+        ('sqlite3', 'instrument_sqlite3')
+    ],
+    'database.cxoracle':[
+        ('cx_Oracle', 'instrument_oracle_client'),
+    ],
+    'database.redis': [
+        ('redis', 'instrument_redis_connection'),
+    ],
+    'database.mongo': [
+        ('pymongo', 'instrument_mongo_client'),
+    ],
+    'database.sqlalchemy': [
+        ('sqlalchemy.orm.session', 'instrument_sqlalchemy'),
+        ('sqlalchemy.engine.default', 'instrument_sqlalchemy_engine'),
+        ('sqlalchemy.engine.create', 'instrument_sqlalchemy_engine_basic'),
+        ('sqlalchemy.engine', 'instrument_sqlalchemy_engine_basic'),
+    ],
+    'database.util': [
+        ('', ''),
+    ],
+
+
+    'application.starlette' : [
+        ('starlette.websockets' , 'instrument_starlette_websocket'),
+    ],
+    'application.wsgi': [
+        ('', ''),
+    ],
+    'application.bottle': [
+        ('bottle', 'instrument'),
+    ],
+    'application.cherrypy': [
+        ('cherrypy', 'instrument'),
+    ],
+    'application.django': [
+        ('django.core.handlers.wsgi', 'instrument'),
+        ('django.core.handlers.base', 'instrument_handlers_base'),
+        ('django.views.generic.base', 'instrument_generic_base'),
+        ('django.contrib.staticfiles.handlers', 'instrument_handlers_static'),
+        ('channels.http', 'instrument_handlers_channels'),
+
+        # Django==1.10
+        ('django.urls.resolvers', 'instrument_url_resolvers', False),
+        ('django.urls.base', 'instrument_urls_base', False),
+        ('django.core.handlers.exception', 'instrument_handlers_exception',
+         False),
+
+    ],
+    'application.django_asgi': [
+        ('django.core.handlers.asgi', 'instrument_asgi'),
+    ],
+    'application.flask': [
+        ('flask', 'instrument'),
+    ],
+    'application.tornado': [
+        ('tornado.web', 'instrument'),
+    ],
+    'application.celery': [
+        ('celery.execute.trace', 'instrument_celery_execute_trace'),
+        ('celery.task.trace', 'instrument_celery_execute_trace'),
+        ('celery.app.trace', 'instrument_celery_execute_trace'),
+    ],
+    'application.nameko': [
+        ('nameko.containers', 'instrument_nameko_spawn_worker'),
+        ('spsengine.containers', 'instrument_nameko_spawn_worker'),
+    ],
+    'application.graphql':[
+        #graphen-core 3.x~
+        ('graphql.execution.execute','instrument_graphql'),
+
+        #graphen-core 2.x
+        ('graphql.execution.executor','instrument_graphql'),
+    ],
+    'application.fastapi': [
+        ('fastapi.applications', 'instrument_applications'),
+        ('fastapi.routing', 'instrument'),
+        ('fastapi.dependencies.utils', 'instrument_util'),
+
+    ],
+    'application.frappe': [
+        ('frappe.app', 'instrument'),
+    ],
+    'application.odoo': [
+        ('odoo', 'instrument'),
+    ],
+}

whatap/util/bit_util.py

@@ -0,0 +1,49 @@
+BYTE_MIN_VALUE = -128
+BYTE_MAX_VALUE = 127
+SHORT_MIN_VALUE = -32768
+SHORT_MAX_VALUE = 32767
+INT3_MIN_VALUE = -0x800000
+INT3_MAX_VALUE = 0x007fffff
+INT_MIN_VALUE = -0x80000000
+INT_MAX_VALUE = 0x7fffffff
+LONG5_MIN_VALUE = -0x8000000000
+LONG5_MAX_VALUE = 0x0000007fffffffff
+LONG_MIN_VALUE = -0x8000000000000000
+LONG_MAX_VALUE = 0x7fffffffffffffff
+
+
+class BitUtil(object):
+    @staticmethod
+    def composite(hkey, wkey):
+        if BYTE_MIN_VALUE <= wkey <= BYTE_MAX_VALUE:
+            return (hkey << 8) | (wkey & 0xff)
+        elif SHORT_MIN_VALUE <= wkey <= SHORT_MAX_VALUE:
+            return (hkey << 16) | (wkey & 0xffff)
+        elif INT_MIN_VALUE <= wkey <= INT_MAX_VALUE:
+            return (hkey << 32) | (wkey & 0xffffffff)
+
+    @staticmethod
+    def setHigh(src, hkey):
+        return (src & 0x00000000ffffffff) | (hkey << 32)
+
+    @staticmethod
+    def setLow(src, wkey):
+        return (src & 0xffffffff00000000) | (wkey & 0xffffffff)
+
+    @staticmethod
+    def getHigh(key):
+        if SHORT_MIN_VALUE <= key <= SHORT_MAX_VALUE:
+            return ((key >> 8) % 0x100000000) & 0xff
+        elif INT_MIN_VALUE <= key <= INT_MAX_VALUE:
+            return ((key >> 16) % 0x100000000) & 0xffff
+        elif LONG_MIN_VALUE <= key <= LONG_MAX_VALUE:
+            return ((key >> 32) % 0x100000000) & 0xffffffff
+
+    @staticmethod
+    def getLow(key):
+        if SHORT_MIN_VALUE <= key <= SHORT_MAX_VALUE:
+            return key & 0xff
+        elif INT_MIN_VALUE <= key <= INT_MAX_VALUE:
+            return key & 0xffff
+        elif LONG_MIN_VALUE <= key <= LONG_MAX_VALUE:
+            return key & 0xffffffff

whatap/util/cardinality/hyperloglog.py

@@ -0,0 +1,84 @@
+import math
+
+from whatap.io.data_outputx import DataOutputX
+from whatap.util.cardinality.murmurhash import MurMurHash
+from whatap.util.cardinality.registerset import RegisterSet
+
+
+def getAlphaMM(p, m):
+    if p == 4:
+        return 0.673 * m * m
+    elif p == 5:
+        return 0.697 * m * m
+    elif p == 6:
+        return 0.709 * m * m
+    else:
+        return (0.7213 / (1 + 1.079 / m)) * m * m
+
+
+def linearCounting(m, V):
+    return m * math.log(m / V)
+
+
+def numberOfLeadingZeros(i):
+    if not i:
+        return 32
+
+    n = 1
+    if (i % 0x100000000) >> 16 == 0:
+        n += 16
+        i <<= 16
+    if (i % 0x100000000) >> 24 == 0:
+        n += 8
+        i <<= 8
+    if (i % 0x100000000) >> 28 == 0:
+        n += 4
+        i <<= 4
+    if (i % 0x100000000) >> 30 == 0:
+        n += 2
+        i <<= 2
+
+    n -= (i % 0x100000000) >> 31
+    return n
+
+
+class HyperLogLog(object):
+    def __init__(self):
+        self.log2m = 10
+        self.registerSet = RegisterSet(1 << self.log2m)
+        m = 1 << self.log2m
+        self.alphaMM = getAlphaMM(self.log2m, m)
+
+    def offerHashed(self, hashedValue):
+        j = (hashedValue % 0x100000000) >> (32 - self.log2m)
+        r = numberOfLeadingZeros(
+            (hashedValue << self.log2m) | (1 << (self.log2m - 1)) + 1) + 1
+        return self.registerSet.updateIfGreater(j, r)
+
+    def offer(self, o):
+        x = MurMurHash.hash(o)
+        return self.offerHashed(x)
+
+    def cardinality(self):
+        registerSum = 0
+        count = self.registerSet.count
+        zeros = 0.0
+        for j, _ in enumerate(count):
+            val = self.registerSet.get(j)
+            registerSum += 1.0 / (1 << val)
+            if int(val) == 0:
+                zeros += 1
+
+        estimate = self.alphaMM * (1 / registerSum)
+        if estimate <= (5.0 / 2.0) * count:
+            return math.round(linearCounting(count, zeros))
+        else:
+            return math.round(estimate)
+
+    def getBytes(self):
+        dout = DataOutputX()
+        dout.writeInt(self.log2m)
+        dout.writeInt(self.registerSet.size)
+        for m in self.registerSet.readOnlyBits():
+            dout.writeInt(m)
+        return dout.toByteArray()

whatap/util/cardinality/murmurhash.py

@@ -0,0 +1,20 @@
+class MurMurHash(object):
+    @classmethod
+    def hash(cls, data):
+        m = 0x5bd1e995
+        r = 24
+        h = 0
+        k = (data * m) & 0xffffffff
+        k ^= (k % 0x100000000) >> r
+        h ^= (k * m) & 0xffffffff
+        
+        k = ((data >> 32) * m) & 0xffffffff
+        k ^= (k % 0x100000000) >> r
+        h *= m
+        h ^= k * m
+        
+        h ^= (h % 0x100000000) >> 13
+        h *= m
+        h ^= (h % 0x100000000) >> 15
+        
+        return h

whatap/util/cardinality/registerset.py

@@ -0,0 +1,60 @@
+LOG2_BITS_PER_WORD = 6
+REGISTER_SIZE = 5
+
+
+def getBits(count):
+    return int(count / LOG2_BITS_PER_WORD)
+
+
+def getSizeForCount(count):
+    bits = getBits(count)
+    if not bits:
+        return 1
+    elif not (bits % 32):
+        return bits
+    else:
+        return bits + 1
+
+
+class RegisterSet(object):
+    def __init__(self, count):
+        self.count = count
+        self.M = [0 for _ in range(getSizeForCount(self.count))]
+        self.size = len(self.M)
+
+    def set(self, position, value):
+        bucketPos = int(position / LOG2_BITS_PER_WORD)
+        shift = REGISTER_SIZE * (position - (bucketPos * LOG2_BITS_PER_WORD))
+        self.M[bucketPos] = (
+            self.M[bucketPos] & ~(0x1f << shift) | (value << shift))
+
+    def get(self, position):
+        bucketPos = int(position / LOG2_BITS_PER_WORD)
+        shift = REGISTER_SIZE * (position - (bucketPos * LOG2_BITS_PER_WORD))
+        return ((self.M[bucketPos] & (0x1f << shift)) % 0x100000000) >> shift
+
+    def updateIfGreater(self, position, value):
+        bucket = int(position / LOG2_BITS_PER_WORD)
+        shift = REGISTER_SIZE * (position - (bucket * LOG2_BITS_PER_WORD))
+        mask = 0x1f << shift
+
+        curVal = self.M[bucket] & mask
+        newVal = value << shift
+        if curVal < newVal:
+            self.M[bucket] = (self.M[bucket] & ~mask) | newVal
+            return True
+        else:
+            return False
+
+    def merge(self, that):
+        for bucket, m in enumerate(self.M):
+            word = 0
+            for j in range(LOG2_BITS_PER_WORD):
+                mask = 0x1f << (REGISTER_SIZE * j)
+                thisVal = (self.M[bucket] & mask)
+                thatVal = (that.M[bucket] & mask)
+                word |= thatVal if thisVal < thatVal else thisVal
+            self.M[bucket] = word
+
+    def readOnlyBits(self):
+        return self.M

whatap/util/compare_util.py

@@ -0,0 +1,19 @@
+class CompareUtil(object):
+
+    @staticmethod
+    def compareTo(l, r):
+        if not l and not r:
+            return 0
+        elif not l:
+            return -1
+        elif not r:
+            return 1
+
+        l_ln = len(l)
+        r_ln = len(r)
+        for i in range(l_ln):
+            if l[i] > r[i]:
+                return 1
+            if l[i] < r[i]:
+                return -1
+        return l_ln - r_ln

whatap/util/date_util.py

@@ -0,0 +1,55 @@
+import time
+
+from datetime import datetime
+
+
+class DateUtil(object):
+    delta = 0
+
+    MILLIS_PER_SECOND = 1000
+    MILLIS_PER_MINUTE = 60 * MILLIS_PER_SECOND
+    MILLIS_PER_FIVE_MINUTE = 5 * 60 * MILLIS_PER_SECOND
+    MILLIS_PER_TEN_MINUTE = 10 * MILLIS_PER_MINUTE
+    MILLIS_PER_HOUR = 60 * MILLIS_PER_MINUTE
+    MILLIS_PER_DAY = 24 * MILLIS_PER_HOUR
+
+    @classmethod
+    def setServerTime(cls, time, syncfactor):
+        now = cls.now()
+        delta = time - now
+        if delta:
+            delta *= syncfactor
+        cls.delta = delta
+        return cls.delta
+
+    @classmethod
+    def currentTime(cls):
+        return int(round(time.time() * 1000)) + cls.delta
+
+    @classmethod
+    def getServerDelta(cls):
+        return cls.delta
+
+    @classmethod
+    def now(cls):
+        return cls.currentTime()
+
+    @classmethod
+    def nowSystem(cls):
+        return int(round(time.time() * 1000))
+
+    @classmethod
+    def getFiveMinUnit(cls):
+        return int(int(
+            cls.currentTime() / cls.MILLIS_PER_FIVE_MINUTE) * cls.MILLIS_PER_FIVE_MINUTE)
+
+    @classmethod
+    def datetime(cls):
+        return datetime.now()
+
+    @classmethod
+    def yyyymmdd(cls, time=None):
+        if not time:
+            time = cls.currentTime()
+        return datetime \
+            .fromtimestamp(time / 1000).strftime('%Y%m%d')

whatap/util/debug_util.py

@@ -0,0 +1,73 @@
+import threading, traceback, time, sys
+from datetime import datetime, timedelta
+
+
+def dump_stack(thread_id,output_path, timeout):
+    with open(output_path,'a+') as f:
+        started = datetime.now()
+        while datetime.now()  - started < timedelta(seconds=timeout):
+            frame = sys._current_frames().get(thread_id)
+            if not frame:
+                return
+            f.write(str(datetime.now()))
+            f.write('\n')
+            for stack in traceback.extract_stack(frame):
+                line = stack[0]
+                line_num = stack[1]
+                method_name = stack[2]
+
+                stack = '{} ({}:{})\n'.format(method_name, line, line_num)
+                f.write(stack)
+                f.write('\n')
+            time.sleep(0.5)
+
+def dumpThreadStack(timeout, output_path):
+    thread_id = threading.get_ident()
+
+    thr = threading.Thread(target=dump_stack, args=(thread_id,output_path, timeout), kwargs={})
+    thr.setDaemon(True)
+    thr.start()
+
+
+def printStack():
+    thread_id = threading.get_ident()
+
+    frame = sys._current_frames().get(thread_id)
+    if not frame:
+        return
+    import io
+    f = io.StringIO()
+    f.write(str(datetime.now()))
+    f.write('\n')
+    for stack in traceback.extract_stack(frame):
+        line = stack[0]
+        line_num = stack[1]
+        method_name = stack[2]
+
+        stack = '{} ({}:{})\n'.format(method_name, line, line_num)
+        f.write(stack)
+        f.write('\n')
+    print(f.getvalue())
+
+
+def printAllStack():
+    for t in threading.enumerate():
+        thread_id = t.ident
+
+        frame = sys._current_frames().get(thread_id)
+        if not frame:
+            return
+            
+        import io
+        f = io.StringIO()
+        f.write(str(datetime.now()))
+        f.write('\n')
+        for stack in traceback.extract_stack(frame):
+            line = stack[0]
+            line_num = stack[1]
+            method_name = stack[2]
+
+            stack = '{} ({}:{})\n'.format(method_name, line, line_num)
+            f.write(stack)
+            f.write('\n')
+        print(f.getvalue())