whatap-python 2.1.0__py3-none-any.whl

whatap/util/string_util.py

@@ -0,0 +1,20 @@
+
+
+
+def tokenizer(target, delim):
+    if not target:
+        return None
+
+    tokens = []
+    for t in target.split(delim):
+        t = t.strip()
+        if t:
+            tokens.append(t)
+
+    return tokens
+
+def trimEmpty(src):
+    if src:
+        return src.strip()
+    else:
+        return ''

whatap/util/throttle_util.py

@@ -0,0 +1,99 @@
+from whatap.conf.configure import Configure as conf
+from whatap.pack.pack_enum import EventLevel
+import whatap.net.async_sender as async_sender
+from whatap.util.hash_util import HashUtil
+from whatap.util.int_set import IntSet
+import whatap.util.string_util as string_util
+from whatap.util.ip_util import IPUtil
+from whatap.net.packet_type_enum import PacketTypeEnum
+
+def getThrottleStringToHashSet(value, deli):
+    intset = IntSet()
+    if value is not None:
+        vv = string_util.tokenizer(value, deli)
+        if vv is None:
+            return intset
+        for x in vv:
+            intset.put(HashUtil.hash(x))
+
+    return intset
+
+def getThrottleIP(value, deli):
+    intset = IntSet()
+    if value is not None:
+        vv = string_util.tokenizer(value, deli)
+        if not vv :
+            return intset
+        for x in vv:
+            ip = IPUtil.toInt(x)
+            if ip != 0:
+                intset.put(ip)
+
+    return intset
+
+def getThrottleIgnorePrefixl(value, deli):
+    strset = set()
+    if value:
+        vv = StringUtil.tokenizer(value, deli)
+        if not vv:
+            return []
+        for x in vv:
+            if x:
+                strset.add(x)
+
+    return list(strset)
+
+
+passingUrlSet = getThrottleStringToHashSet(conf.throttle_passing_url, ",")
+passingPrefix = getThrottleIgnorePrefixl(conf.throttle_passing_url_prefix, ",")
+blockingUrlSet = getThrottleStringToHashSet(conf.throttle_blocking_url, ",")
+blockingIPSet = getThrottleIP(conf.throttle_blocking_ip, ",")
+
+blocking_enabled = blockingUrlSet.size() > 0 or blockingIPSet.size() > 0
+
+def getHash():
+    k1 = string_util.trimEmpty(conf.throttle_passing_url)
+    k2 = string_util.trimEmpty(conf.throttle_passing_url_prefix)
+    k3 = string_util.trimEmpty(conf.throttle_blocking_url)
+    k4 = string_util.trimEmpty(conf.throttle_blocking_ip)
+    return hash(k1) ^ hash(k2) ^ hash(k3) ^ hash(k4)
+
+
+def isblocking(str_ip, path):
+    remote_ip = IPUtil.toInt(str_ip)
+    if not blocking_enabled:
+        return False
+    if blockingIPSet.contains(remote_ip):
+        return True
+    urlhash = HashUtil.hashFromString(path)
+    if blockingUrlSet.contains(urlhash):
+        return True
+    return False
+
+def sendrejectevent(ctx, path, ip):
+    pathHash = HashUtil.hashFromString(path)
+    datas = ("REJECTED_URL", EventLevel.WARNING,
+             "Rejected " + path,
+             pathHash,
+             path,
+             ip,
+    )
+    async_sender.send_packet(PacketTypeEnum.EVENT, ctx, datas)
+    
+valueHash = 0
+def updateConfig():
+    global passingUrlSet, passingPrefix, blockingUrlSet, blockingIPSet, blocking_enabled, ignoreContext, ignoreDomain, valueHash
+    newHash = getHash()
+    if valueHash != newHash:
+        passingUrlSet = getThrottleStringToHashSet(conf.throttle_passing_url, ",")
+        passingPrefix = getThrottleIgnorePrefixl(conf.throttle_passing_url_prefix, ",")
+        blockingUrlSet = getThrottleStringToHashSet(conf.throttle_blocking_url, ",")
+        blockingIPSet = getThrottleIP(conf.throttle_blocking_ip, ",")
+
+        blocking_enabled = len(blockingUrlSet) > 0 or len(blockingIPSet) > 0
+
+        ignoreContext = StringUtil.tokenizer(conf.throttle_blocking_ignore_context, ",")
+        ignoreDomain = getThrottleStringToHashSet(conf.throttle_blocking_ignore_domain, ",")
+
+        valueHash = getHash()
+conf.addObserver(updateConfig)

whatap/util/userid_util.py

@@ -0,0 +1,134 @@
+from whatap.util.hash_util import HashUtil as hash_util
+from whatap.util.hexa32 import Hexa32 as hexa32
+from whatap.util.keygen import KeyGen
+import whatap.util.bit_util as bit_util
+from whatap.conf.configure import Configure as conf
+import logging as logging_module
+import sys,  traceback
+
+logger = logging_module.getLogger(__name__)
+
+def toDjangoHeaderName(src):
+
+    return 'HTTP_' + src.upper().replace('-','_')
+
+class UseridUtil(object):
+    WHATAP_R = "WHATAP"
+
+    @staticmethod
+    def getUserId(req,  defValue):
+        try:
+
+            if conf.user_header_ticket:
+                ticket = UseridUtil.getHeader(req, conf.user_header_ticket)
+                if  ticket:
+                    return hash_util.hashFromString(ticket), ticket
+                return 0, ""
+            cookie = UseridUtil.getHeader(req, "Cookie")
+            if cookie:
+                if len(cookie) >= conf.trace_user_cookie_limit :
+                    return hash_util.hashFromString(defValue) if defValue else 0, defValue or ""
+
+                x1 = cookie.find(UseridUtil.WHATAP_R)
+                if x1 >= 0:
+                    x2 = cookie.find(';', x1)
+                    if x2 > 0:
+                        value = cookie[x1 + len(UseridUtil.WHATAP_R) + 1: x2]
+                    else:
+                        value = cookie[x1 + len(UseridUtil.WHATAP_R) + 1:]
+                    return hexa32.toLong32(value), value
+            userid = KeyGen.next()
+            return userid, hexa32.toString32(userid)
+        except Exception:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            logger.debug("A502", 10, str(exc_value))
+            return hash_util.hashFromString(defValue) if defValue else 0, defValue or ""
+
+    @staticmethod
+    def setUserId(req, res, cookieValue):
+        try:
+            if not conf.user_header_ticket:
+                cookie = UseridUtil.getHeaderEx(req, "Cookie")
+                if not cookie or cookie.find(UseridUtil.WHATAP_R) < 0 or cookie.find(cookieValue) < 0:
+                    UseridUtil.setCookie(res, UseridUtil.WHATAP_R, cookieValue, bit_util.INT_MAX_VALUE, "/", conf.trace_user_cookie_domain )
+        except Exception:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            logger.debug("A503", 10, str(exc_value))
+
+    @staticmethod
+    def DEPRECATED_getUserid(req, res, defValue):
+        try:
+            if conf.user_header_ticket_enabled:
+                ticket = UseridUtil.getHeader(req, conf.user_header_ticket)
+                if  ticket:
+                    return hash_util.hash(ticket)
+                return 0
+            cookie = UseridUtil.getHeader(req, "Cookie")
+            if cookie:
+                if len(cookie) >= conf.trace_user_cookie_limit :
+                    return defValue
+
+                x1 = cookie.find(UseridUtil.WHATAP_R)
+                if x1 >= 0:
+                    x2 = cookie.find(';', x1)
+                    if x2 > 0:
+                        value = cookie[x1 + len(UseridUtil.WHATAP_R) + 1: x2]
+                    else:
+                        value = cookie[x1 + len(UseridUtil.WHATAP_R) + 1:]
+                    return hexa32.toLong32(value)
+            UseridUtil.setCookie(res, UseridUtil.WHATAP_R, hexa32.toString32(KeyGen.next()), bit_util.INT_MAX_VALUE, "/", conf.trace_user_cookie_domain )
+
+        except Exception:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            logger.debug("A502",10, str(exc_value))
+
+        return defValue
+
+    META_DICT = {'django': lambda x, name: x.get(toDjangoHeaderName(name))}
+
+    @staticmethod
+    def getHeader(args, name):
+        for arg in args:
+            if isinstance(arg, dict):
+                if name in arg:
+                    return arg[name]
+                else:
+                    for  fn in UseridUtil.META_DICT.values():
+                        v = fn(arg, name)
+                        if v:
+                            return v
+
+        return None
+
+    GET_HEADER={'META': lambda x : x.META.get}
+    @staticmethod
+    def getHeaderEx(req, name):
+        name = toDjangoHeaderName(name)
+
+        for k, fn in UseridUtil.GET_HEADER.items():
+            if hasattr(req, k):
+                return fn(req)(name)
+
+        return None
+
+    SETCOOKIE_METHODS={
+       "set_cookie": lambda res, key, value, max_age, path, domain : res.set_cookie(key, value, max_age = max_age, path=path, domain=domain )
+    }
+    @staticmethod
+    def setCookie(res, key, value, max_age, path, domain):
+        for mname, fn in UseridUtil.SETCOOKIE_METHODS.items():
+            if hasattr(res, mname):
+                fn(res, key, value, max_age, path, domain)
+                break
+
+    @staticmethod
+    def getRemoteAddr(args):
+        if conf.trace_http_client_ip_header_key:
+            header_val = UseridUtil.getHeader(args, conf.trace_http_client_ip_header_key)
+            if header_val:
+                return header_val.split(',')[0].strip()
+        x_forwarded_for = UseridUtil.getHeader(args, "x-forwarded-for")
+        if x_forwarded_for:
+            return x_forwarded_for.split(',')[0].strip()
+        return UseridUtil.getHeader(args, "REMOTE_ADDR")
+        #return UseridUtil.getHeader(args, "REMOTE_ADDR")

whatap/value/__init__.py

@@ -0,0 +1 @@
+from whatap.value.list_value import ListValue

whatap/value/blob_value.py

@@ -0,0 +1,38 @@
+from whatap.value.value import Value
+from whatap.value.value_enum import ValueEnum
+
+from whatap.util.compare_util import CompareUtil
+from whatap.util.hash_util import HashUtil
+
+
+class BlobValue(Value):
+    def __init__(self, value=None):
+        super(BlobValue, self).__init__()
+        self.value = value
+
+    def compareTo(self, object):
+        if isinstance(object, BlobValue):
+            return CompareUtil.compareTo(self.value, object.value)
+        return 1
+
+    def equals(self, object):
+        if isinstance(object, BlobValue):
+            if not self.value:
+                return object.value is None
+            return self.value == object.value
+        return False
+
+    def hashCode(self):
+        if not self.value:
+            return 0
+        return HashUtil.hash(self.value)
+
+    def write(self, dout):
+        dout.writeBlob(self.value)
+
+    def read(self, din):
+        self.value = din.readBlob()
+        return self
+
+    def getValueType(self):
+        return ValueEnum.BLOB

whatap/value/boolean_value.py

@@ -0,0 +1,33 @@
+from whatap.value.value import Value
+from whatap.value.value_enum import ValueEnum
+
+
+class BooleanValue(Value):
+    def __init__(self, value=None):
+        super(BooleanValue, self).__init__()
+        self.value = value if isinstance(value, bool) else False
+
+    def compareTo(self, object):
+        if isinstance(object, BooleanValue):
+            if self.value == object.value:
+                return 0
+            return 1 if self.value else -1
+        return 1
+
+    def equals(self, object):
+        if isinstance(object, BooleanValue):
+            return self.value == object.value
+        return False
+
+    def hashCode(self):
+        return 1 if self.value else 0
+
+    def write(self, dout):
+        dout.writeBoolean(self.value)
+
+    def read(self, din):
+        self.value = din.readBoolean()
+        return self
+
+    def getValueType(self):
+        return ValueEnum.BOOLEAN

whatap/value/decimal_value.py

@@ -0,0 +1,36 @@
+from whatap.value.number_value import NumberValue
+from whatap.value.value_enum import ValueEnum
+
+
+class DecimalValue(NumberValue):
+    def __init__(self, value=None):
+        super(DecimalValue, self).__init__()
+        self.value = value
+
+    def compareTo(self, object):
+        if isinstance(object, DecimalValue):
+            if self.value < object.value:
+                return -1
+            elif self.value == object.value:
+                return 0
+            else:
+                return 1
+        return 1
+
+    def equals(self, object):
+        if isinstance(object, DecimalValue):
+            return self.value == object.value
+        return False
+
+    def hashCode(self):
+        return self.value ^ (self.value >> 32)
+
+    def write(self, dout):
+        dout.writeDecimal(self.value)
+
+    def read(self, din):
+        self.value = din.readDecimal()
+        return self
+
+    def getValueType(self):
+        return ValueEnum.DECIMAL

whatap/value/double_summary.py

@@ -0,0 +1,86 @@
+from whatap.value.summary_value import SummaryValue
+from whatap.value.value_enum import ValueEnum
+
+
+class DoubleSummary(SummaryValue):
+    def __init__(self):
+        super(DoubleSummary. self).__init__()
+        self.sum = 0
+        self.count = 0
+        self.min = 0
+        self.max = 0
+
+    def getValueType(self):
+        return ValueEnum.DOUBLE_SUMMARY
+
+    def write(self, dout):
+        dout.writeDouble(self.sum)
+        dout.writeInt(self.count)
+        dout.writeDouble(self.min)
+        dout.writeDouble(self.max)
+
+    def read(self, din):
+        self.sum = din.readDouble()
+        self.count = din.readInt()
+        self.min = din.readDouble()
+        self.max = din.readDouble()
+        return self
+
+    def toObject(self):
+        return self
+
+    def addCount(self):
+        self.count += 1
+
+    def add(self, value):
+        if not value:
+            return self
+
+        if isinstance(value, int):
+            if not self.count:
+                self.sum = value
+                self.count = 1
+                self.max = value
+                self.min = value
+            else:
+                self.sum += value
+                self.count += 1
+                self.max = max(self.max, value)
+                self.min = min(self.min, value)
+
+        elif isinstance(value, SummaryValue):
+            if not value.getCount():
+                return self
+
+            self.count += value.getCount()
+            self.sum += value.doubleSum()
+            self.min = min(self.min, value.doubleMin())
+            self.max = max(self.max, value.doubleMax())
+        return self
+
+    def longSum(self):
+        return self.sum
+
+    def longMin(self):
+        return self.min
+
+    def longMax(self):
+        return self.max
+
+    def longAvg(self):
+        return 0 if not self.count else self.sum / self.count
+
+    def doubleSum(self):
+        return self.sum
+
+    def doubleMin(self):
+        return self.min
+
+    def doubleMax(self):
+        return self.max
+
+    def doubleAvg(self):
+        return 0 if not self.count else self.sum / self.count
+
+    def getCount(self):
+        return self.count

whatap/value/double_value.py

@@ -0,0 +1,33 @@
+from whatap.value.number_value import NumberValue
+from whatap.value.value_enum import ValueEnum
+
+
+class DoubleValue(NumberValue):
+    def __init__(self, value=None):
+        super(DoubleValue, self).__init__()
+        self.value = value
+
+    def compareTo(self, object):
+        if isinstance(object, DoubleValue):
+            if self.value != object.value:
+                return 1 if self.value > object.value else -1
+            return 0
+        return 1
+
+    def equals(self, object):
+        if isinstance(object, DoubleValue):
+            return self.value == object.value
+        return False
+
+    def hashCode(self):
+        return self.value ^ (self.value >> 32)
+
+    def write(self, dout):
+        dout.writeDouble(self.value)
+
+    def read(self, din):
+        self.value = din.readDouble()
+        return self
+
+    def getValueType(self):
+        return ValueEnum.DOUBLE

whatap/value/float_array.py

@@ -0,0 +1,42 @@
+from whatap.value.value import Value
+from whatap.value.value_enum import ValueEnum
+
+from whatap.util.compare_util import CompareUtil
+
+
+class FloatArray(Value):
+    def __init__(self, value=None):
+        super(FloatArray, self).__init__()
+        self.value = value if isinstance(value, list) else []
+        self._hash = 0
+
+    def compareTo(self, object):
+        if isinstance(object, FloatArray):
+            return CompareUtil.compareTo(self.value, object.value)
+        return 1
+
+    def equals(self, object):
+        if isinstance(object, FloatArray):
+            return self.value == object.value
+        return False
+
+    def hashCode(self):
+        if not self._hash:
+            if not self.value:
+                return 0
+
+            result = 1
+            for v in self.value:
+                result = 31 * result + round(v)
+            self._hash = result
+        return self._hash
+
+    def write(self, dout):
+        dout.writeFloatArray(self.value)
+
+    def read(self, din):
+        self.value = din.readFloatArray()
+        return self
+
+    def getValueType(self):
+        return ValueEnum.ARRAY_FLOAT

whatap/value/float_value.py

@@ -0,0 +1,34 @@
+from whatap.value.number_value import NumberValue
+from whatap.value.value_enum import ValueEnum
+
+from whatap.util.compare_util import CompareUtil
+
+
+class FloatValue(NumberValue):
+    def __init__(self, value=0.0):
+
+        super(FloatValue, self).__init__()
+        self.value = value
+
+    def compareTo(self, object):
+        if isinstance(object, FloatValue):
+            return CompareUtil.compareTo(self.value, object.value)
+        return 1
+
+    def equals(self, object):
+        if isinstance(object, FloatValue):
+            return self.value == object.value
+        return False
+
+    def hashCode(self):
+        return round(self.value)
+
+    def write(self, dout):
+        dout.writeFloat(self.value)
+
+    def read(self, din):
+        self.value = din.readFloat()
+        return self
+
+    def getValueType(self):
+        return ValueEnum.FLOAT

whatap/value/int_array.py

@@ -0,0 +1,42 @@
+from whatap.value.value import Value
+from whatap.value.value_enum import ValueEnum
+
+from whatap.util.compare_util import CompareUtil
+
+
+class IntArray(Value):
+    def __init__(self, value=None):
+        super(IntArray, self).__init__()
+        self.value = value if isinstance(value, list) else []
+        self._hash = 0
+
+    def compareTo(self, object):
+        if isinstance(object, IntArray):
+            return CompareUtil.compareTo(self.value, object.value)
+        return 1
+
+    def equals(self, object):
+        if isinstance(object, IntArray):
+            return self.value == object.value
+        return False
+
+    def hashCode(self):
+        if not self._hash:
+            if not self.value:
+                return 0
+
+            result = 1
+            for v in self.value:
+                result = 31 * result + v
+            self._hash = result
+        return self._hash
+
+    def write(self, dout):
+        dout.writeIntArray(self.value)
+
+    def read(self, din):
+        self.value = din.readIntArray()
+        return self
+
+    def getValueType(self):
+        return ValueEnum.ARRAY_INT

whatap/value/ip4_value.py

@@ -0,0 +1,50 @@
+from whatap.io.data_inputx import DataInputX
+from whatap.value.value import Value
+from whatap.value.value_enum import ValueEnum
+
+from whatap.util.compare_util import CompareUtil
+from whatap.util.ip_util import IPUtil
+
+
+class IP4Value(Value):
+    def __init__(self, value=None):
+        super(IP4Value, self).__init__()
+        if isinstance(object, str):
+            self.value = IPUtil.toBytes(value)
+        else:
+            self.value = None
+        self.empty = bytearray(4)
+
+    def compareTo(self, object):
+        if isinstance(object, IP4Value):
+            return CompareUtil.compareTo(self.value, object.value)
+        return 1
+
+    def equals(self, object):
+        if isinstance(object, IP4Value):
+            return self.value == object.value
+        return False
+
+    def hashCode(self):
+        if self.value == self.empty:
+            return 0
+        return DataInputX.toInt(self.value, 0)
+
+    def write(self, dout):
+        if not self.value:
+            self.value = self.empty
+        dout.write(self.value)
+
+    def read(self, din):
+        if not self.value:
+            self.value = self.empty
+        self.value = din.read(4)
+        return self
+
+    def toString(self):
+        if not self.value:
+            self.value = self.empty
+        return IPUtil.toString(self.value)
+
+    def getValueType(self):
+        return ValueEnum.IP4ADDR