whatap-python 0.1.120__tar.gz → 0.3.35__tar.gz

whatap_python-0.3.35/PKG-INFO

@@ -0,0 +1,143 @@
+Metadata-Version: 2.1
+Name: whatap-python
+Version: 0.3.35
+Summary: Monitoring and Profiling Service
+Home-page: https://www.whatap.io
+Author: whatap
+Author-email: admin@whatap.io
+License: Whatap License
+
+
+.. image:: https://www.whatap.io/img/common/whatap_logo_re.svg
+
+.. _WhaTap: https://www.whatap.io/
+
+WhaTap_ for python
+==================
+
+- Whatap allows for application performance monitoring.
+- Support: WSGI server application & Batch job & Specific method profiling.
+- Python version : 2.7 + & 3.3+
+
+Installation
+------------
+
+  .. code:: bash
+
+    $ pip install whatap-python
+
+Application Monitoring
+----------------------
+
+Supported web frameworks such as Django, Flask, Bottle, Cherrypy, Tornado and WSGI Server Application.
+
+Configuration
+~~~~~~~~~~~~~
+
+  .. code:: bash
+
+    $ export WHATAP_HOME=[PATH]
+    $ whatap-setting-config --host [HOST_ADDR]
+                            --license [LICENSE_KEY]
+                            --app_name [APPLICATION_NAME]
+                            --app_process_name [APP_PROCESS_NAME]
+
+Usage
+~~~~~
+
+  .. code:: bash
+
+    $ whatap-start-agent [YOUR_APPLICATION_START_COMMAND]
+
+    ...
+
+Unsupported web frameworks WSGI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want WSGI Application monitoring, include the @register_app decorator.
+
+  .. code:: python
+
+    import whatap
+
+    @whatap.register_app
+    def simple_app(environ, start_response):
+    """Simplest possible application object"""
+        status = '200 OK'
+        response_headers = [('Content-type', 'text/plain')]
+        start_response(status, response_headers)
+        return ['Hello world!\n']
+
+Method Profiling
+~~~~~~~~~~~~~~~~
+
+If you want method profiling, include the @method_profiling decorator.
+
+  .. code:: python
+
+    from whatap import method_profiling
+
+    @method_profiling
+    def db_connection():
+        db.connect('mysql:// ..')
+
+    @method_profiling
+    def query():
+        db.select('select * from ..')
+
+      ....
+
+Batch Monitoring
+----------------
+
+for Batch job.
+
+Configuration
+~~~~~~~~~~~~~
+
+Set Environment valiable configuration.
+
+  .. code:: bash
+
+    $ export WHATAP_BATCH_HOME=[PATH]
+    $ cat >> $WHATAP_BATCH_HOME/whatap.conf << EOF
+    license=[LICENSE_KEY]
+    whatap.server.host=[HOST_ADDR]
+
+    app_name=batch
+    app_process_name=batch
+    EOF
+
+
+Usage
+~~~~~
+
+Start bach agent.
+
+  .. code:: bash
+
+    $ whatap-start-batch
+
+Example code
+~~~~~~~~~~~~
+
+  .. code:: python
+
+    from whatap import method_profiling
+
+    class Command(BaseCommand):
+
+        @batch_profiling
+        def handle(self, *args, **options):
+            // batch code..
+            ....
+
+Restart
+-------
+
+Your Application restart.
+
+Copyright
+---------
+
+Copyright (c) 2017 Whatap, Inc. All rights reserved.

{whatap-python-0.1.120 → whatap_python-0.3.35}/whatap/__init__.py

@@ -39,12 +39,21 @@ class ContextFilter(logging_module.Filter):
             record.id = ''
             return True
 
+    
 
 from whatap.conf.configure import Configure as conf
-
 CONFIG_FILE_NAME = 'whatap.conf'
 LOG_FILE_NAME = 'whatap-hook.log'
 
+isFrappeCommands = "get-frappe-commands" in sys.argv if hasattr(sys, "argv") else False
+
+def whatap_print(*args):
+    if isFrappeCommands:
+        logging.info(*args)
+    else:
+        if len(args) > 0:
+            message = " ".join(args)
+            print(message)
 
 class Logger(object):
     def __init__(self):
@@ -62,8 +71,8 @@ class Logger(object):
                 os.mkdir(os.environ['WHATAP_LOGS'])
             
             except Exception as e:
-                print('WHATAP: LOG FILE WRITE ERROR.')
-                print(
+                whatap_print('WHATAP: LOG FILE WRITE ERROR.')
+                whatap_print(
                     'WHATAP: Try to execute command. \n  {}'.format(
                         'sudo mkdir -m 777 -p $WHATAP_HOME/logs`'))
         
@@ -86,7 +95,7 @@ class Logger(object):
             
             self.logger.setLevel(logging_module.DEBUG)
         except Exception as e:
-            print('WHATAP: LOGGING ERROR: {}'.format(e))
+            whatap_print('WHATAP: LOGGING ERROR: {}'.format(e))
         else:
             self.print_whatap()
     
@@ -105,7 +114,7 @@ class Logger(object):
                                                 os.environ['WHATAP_CONFIG']))
         str += '{0}: {1}\n\n'.format('Logs', os.environ['WHATAP_LOGS'])
         
-        print(str)
+        whatap_print(str)
         logging.debug(str)
 
 
@@ -113,7 +122,7 @@ def read_file(home, file_name):
     data = ''
     try:
         f = open(os.path.join(os.environ.get(home), file_name), 'r+')
-        data = f.readline()
+        data = str(f.readline()).strip()
         f.close()
     finally:
         return data
@@ -125,8 +134,8 @@ def write_file(home, file_name, value):
         f.write(value)
         f.close()
     except Exception as e:
-        print('WHATAP: WHATAP HOME ERROR. (path: {})'.format(os.path.join(os.environ.get(home))))
-        print(
+        whatap_print('WHATAP: WHATAP HOME ERROR. (path: {})'.format(os.path.join(os.environ.get(home))))
+        whatap_print(
             'WHATAP: Try to execute command. \n  {}'.format(
                 '`sudo chmod -R 777 $WHATAP_HOME`'))
         return False
@@ -137,7 +146,7 @@ def write_file(home, file_name, value):
 def check_whatap_home(target='WHATAP_HOME'):
     whatap_home = os.environ.get(target)
     if not whatap_home:
-        print('WHATAP: ${} is empty'.format(target))
+        whatap_print('WHATAP: ${} is empty'.format(target))
     
     return whatap_home
 
@@ -150,10 +159,10 @@ def init_config(home):
             whatap_home = os.getcwd()
             os.environ[home] = whatap_home
             
-            print('WHATAP: WHATAP_HOME is empty')
-            print(
+            whatap_print('WHATAP: WHATAP_HOME is empty')
+            whatap_print(
                 'WHATAP: WHATAP_HOME set default CURRENT_WORKING_DIRECTORY value')
-            print('CURRENT_WORKING_DIRECTORY is {}\n'.format(whatap_home))
+            whatap_print('CURRENT_WORKING_DIRECTORY is {}\n'.format(whatap_home))
     
     if not write_file(home, home.lower(), whatap_home):
         return False
@@ -172,8 +181,8 @@ def init_config(home):
                 with open(config_file, 'w+') as new_f:
                     new_f.write(content)
             except Exception as e:
-                print('WHATAP: PERMISSION ERROR: {}'.format(e))
-                print(
+                whatap_print('WHATAP: PERMISSION ERROR: {}'.format(e))
+                whatap_print(
                     'WHATAP: Try to execute command. \n  {}'.format(
                         '`sudo chmod -R 777 $WHATAP_HOME`'))
                 return False
@@ -184,7 +193,6 @@ def init_config(home):
 def update_config(home, opt_key, opt_value):
     config_file = os.path.join(os.environ[home],
                                CONFIG_FILE_NAME)
-    
     try:
         with open(config_file, 'r+') as f:
             is_update = False
@@ -198,11 +206,11 @@ def update_config(home, opt_key, opt_value):
                     
                     content += line
             if not is_update:
-                content += '{0}={1}\n'.format(opt_key, opt_value)
+                content += '\n{0}={1}\n'.format(opt_key, opt_value)
             open(config_file, 'w+').write(content)
     
     except Exception as e:
-        print('WHATAP: OPTION ERROR: {}'.format(e))
+        whatap_print('WHATAP: OPTION ERROR: {}'.format(e))
 
 
 def config(home):
@@ -241,9 +249,13 @@ def hooks(home):
         logging.debug(e, extra={'id': 'MODULE ERROR'})
     finally:
         try:
+            if conf.trace_logging_enabled:
+                logging_module = sys.modules.get("logging")
+                from whatap.trace.mod.logging import instrument_logging
+                instrument_logging(logging_module)
+
             if conf.hook_method_patterns:
                 from whatap.trace.mod.plugin import instrument_plugin
-            
                 patterns = conf.hook_method_patterns.split(',')
                 for pattern in patterns:
                     pattern=pattern.strip()
@@ -281,10 +293,10 @@ def agent():
             whatap_home = os.getcwd()
             os.environ[home] = whatap_home
             
-            print('WHATAP: WHATAP_HOME is empty')
-            print(
+            whatap_print('WHATAP: WHATAP_HOME is empty')
+            whatap_print(
                 'WHATAP: WHATAP_HOME set default CURRENT_WORKING_DIRECTORY value')
-            print('CURRENT_WORKING_DIRECTORY is {}\n'.format(whatap_home))
+            whatap_print('CURRENT_WORKING_DIRECTORY is {}\n'.format(whatap_home))
     
     if write_file(home, home.lower(), whatap_home):
         os.environ['WHATAP_HOME'] = whatap_home
@@ -300,13 +312,15 @@ ARCH = {
     'x86_32': '386',
     'ARM': 'arm',
     'AArch64': 'arm64',
+    'arm64': 'arm64',
+    'aarch64': 'arm64'
 }
 
 AGENT_NAME = 'whatap_python'
 
 
 
-def go(batch=False):
+def go(batch=False, opts={}):
     newenv=os.environ.copy()
     newenv['WHATAP_VERSION'] = build.version
     newenv['whatap.start'] = str(DateUtil.now())
@@ -317,6 +331,7 @@ def go(batch=False):
     newenv['sys.version_info'] = str(sys.version_info)
     newenv['sys.executable'] = sys.executable
     newenv['sys.path'] = str(sys.path)
+    newenv.update(opts)
 
     if not batch:
         home = 'WHATAP_HOME'
@@ -331,11 +346,12 @@ def go(batch=False):
             with open(cmdlinepath) as f:
                 content = f.read()
                 if content:
-                    return content.encode('utf8')
+                    return content.strip()
         return ''
     pid = read_file(home, file_name)
-    if pid and 'whatap_python' in get_pname(pid):
-        return
+    if pid and get_pname(pid).find('whatap_python') >= 0:
+        os.kill(int(pid), signal.SIGKILL)
+        
     try:
         home_path= os.environ.get(home)
         if os.path.exists(os.path.join(home_path, AGENT_NAME)):
@@ -344,28 +360,32 @@ def go(batch=False):
         source_cwd = os.path.join(os.path.join(os.path.dirname(__file__), 'agent'), platform.system().lower(),
                                   ARCH[platform.machine()])
 
-
         os.symlink(os.path.join(source_cwd, AGENT_NAME),
                    os.path.join(home_path, AGENT_NAME))
 
+        sockfile_path = os.path.join(home_path, 'run')
+        if not os.path.exists(sockfile_path):
+            os.mkdir(sockfile_path)
         newenv['whatap.enabled'] = 'True'
-        def preexec_function():
-            signal.signal(signal.SIGINT, signal.SIG_IGN)
-        process = subprocess.Popen('./{0}'.format(AGENT_NAME),
-                                   cwd=home_path,env=newenv, preexec_fn=preexec_function)
-        # Write PID file
-        write_file(home, file_name, str(process.pid))
+        newenv['WHATAP_PID_FILE'] = file_name
+        newenv['PYTHON_PARENT_APP_PID'] = str(os.getpid())
+        process = subprocess.Popen(['./{0}'.format(AGENT_NAME), '-t', '3', '-d', '1'],
+                                   cwd=home_path,env=newenv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdouts, errs = process.communicate()
+        whatap_print("executed golang module ", str(stdouts,"utf8"), str(errs, "utf8"))
     except Exception as e:
-        print('WHATAP: AGENT ERROR: {}'.format(e))
+        import traceback
+        traceback.print_exc()
+        whatap_print('WHATAP: AGENT ERROR: {}'.format(e))
     else:
-        print('WHATAP: AGENT UP! (process name: {})\n'.format(AGENT_NAME))
+        whatap_print('WHATAP: AGENT UP! (process name: {})\n'.format(AGENT_NAME))
 
 import signal
 
 from whatap.trace.mod.application_wsgi import interceptor, start_interceptor, \
-    end_interceptor, trace_handler
-
-from whatap.trace.trace_context import TraceContext
+    end_interceptor, trace_handler, interceptor_step_error
+from whatap.trace.mod.application_fastapi import interceptor_error_log
+from whatap.trace.trace_context import TraceContext, TraceContextManager
 
 def register_app(fn):
     @trace_handler(fn, True)
@@ -392,14 +412,18 @@ def method_profiling(fn):
         try:
             ctx = TraceContext()
             ctx.service_name=fn.__name__
-            ctx = start_interceptor(ctx)
+            start_interceptor(ctx)
             callback = fn(*args, **kwargs)
-            end_interceptor(thread_id= ctx.thread_id)
         except Exception as e:
+            ctx = TraceContextManager.getLocalContext()
+            interceptor_step_error(e, ctx=ctx)
+            interceptor_error_log(ctx.id, e, fn, args, kwargs)
             logging.debug('WHATAP(@method_profiling): ' + str(e),
                           extra={'id': 'WA776'}, exc_info=True)
         finally:
-            return callback if callback else fn(*args, **kwargs)
+            ctx = TraceContextManager.getLocalContext()
+            end_interceptor(ctx=ctx)
+            return callback
     
     if not os.environ.get('whatap.enabled'):
         agent()
@@ -412,7 +436,7 @@ def batch_agent():
     batch_home = os.environ.get(home)
     if not batch_home:
         if not read_file(home, home.lower()):
-            print('WHATAP: WHATAP_HOME_BATCH is empty')
+            whatap_print('WHATAP: WHATAP_HOME_BATCH is empty')
             return
     
     if write_file(home, home.lower(), batch_home):
@@ -431,7 +455,7 @@ def batch_profiling(fn):
             home = 'WHATAP_HOME_BATCH'
             batch_home = read_file(home, home.lower())
             if not batch_home:
-                print(
+                whatap_print(
                     'WHATAP(@batch_profiling): try, whatap-start-batch')
                 return fn(*args, **kwargs)
             else:
@@ -455,3 +479,78 @@ def batch_profiling(fn):
             return callback if callback else fn(*args, **kwargs)
     
     return trace
+
+
+import fcntl, os, time
+import errno
+def openPortFile(filepath=os.environ.get('WHATAP_LOCK_FILE', '/tmp/whatap-python.lock')):
+    f = None
+    i=0
+    while f == None and i < 100:
+        try:
+            f = open(filepath, 'r+')
+        except IOError as e:
+            if e.errno == 2:
+                prefix = os.path.split(filepath)[0]
+                try:
+                    if not os.path.exists(prefix):
+                        os.makedirs(prefix)
+                    f = open(filepath, 'w+')
+                except:
+                    pass
+        i += 1
+
+    if f:
+        try:
+            fcntl.lockf(f, fcntl.LOCK_EX)
+            return f
+        except Exception as e:
+            whatap_print(e)
+            try:
+                f.close()
+            except:
+                pass
+
+def get_port_number(port=6600, home=os.environ.get('WHATAP_HOME')):
+    if not home:
+        return None
+
+    for i in range(100):
+        f = openPortFile()
+        if not f:
+            if i > 50:
+                time.sleep(0.1)
+            continue
+    if f:
+        lastPortFound = None
+        for l in f.readlines():
+            l = l.strip()
+            try:
+                (portFound, portHome) = l.split()
+                portFound = int(portFound)
+            except:
+                continue
+            if home == portHome:
+                return portFound
+            if not lastPortFound or lastPortFound < portFound:
+                lastPortFound = int(portFound)
+        if not lastPortFound:
+            lastPortFound = port
+        else:
+            lastPortFound += 1
+        f.write(str(lastPortFound))
+        f.write('\t')    
+        f.write(home)
+        f.write('\n')    
+        fcntl.lockf(f, fcntl.LOCK_UN)
+        f.close()
+        return lastPortFound
+            
+    return port
+
+
+def configPort():
+    port = get_port_number()
+    if port:
+        update_config('WHATAP_HOME', 'net_udp_port', str(port))
+        return port

whatap_python-0.3.35/whatap/bootstrap/sitecustomize.py

@@ -0,0 +1,19 @@
+import sys, os
+
+try:
+    from whatap import agent
+    agent()
+except ModuleNotFoundError:
+    pythonpath = os.environ.get('PYTHONPATH')
+    if pythonpath.endswith("/bootstrap"):
+        pythonpath,_ = os.path.split(pythonpath)
+        pythonpath,_ = os.path.split(pythonpath)
+        
+        sys.path.append(pythonpath)
+        try:
+            from whatap import agent
+            agent()
+        except ModuleNotFoundError:
+            #final fail
+            pass
+            

whatap_python-0.3.35/whatap/build.py

@@ -0,0 +1,4 @@
+app = 'Python'
+name = 'whatap-python'
+version = '0.3.35'
+release_date = '20240625'

{whatap-python-0.1.120 → whatap_python-0.3.35}/whatap/conf/configuration.py

@@ -5,7 +5,12 @@ Configuration = {
     "trace_auto_normalize_enabled": True,
     "trace_user_enabled": True,
     "trace_user_using_ip": True,
+    "trace_logging_enabled": False,
+    "trace_loguru_enabled": False,
+    "trace_ignore_url_set": [],
+    "trace_ignore_url_prefix": None,
 
+    "debug": False,
     "query_string_enabled": False,
     "query_string_urls": "",
 
@@ -194,4 +199,8 @@ Configuration = {
     "master_agent_port" : 6600,
     "unix_socket_enabled":False,
     "unix_socket":"whatap.sock",
+    "debug_stack_enabled": False,
+    "log_unhandled_exception": 'false',
+    "threadstack_faulthandler": False,
+    "max_send_queue_size": 1000,
 }

{whatap-python-0.1.120 → whatap_python-0.3.35}/whatap/conf/configure.py

@@ -14,13 +14,23 @@ class Configure(object):
     last_config_modified = 0
     observers = []
     POD_NAME = os.environ.get("POD_NAME") if os.environ.get('POD_NAME') else os.environ.get("PODNAME") if os.environ.get("PODNAME") else ''
+    license = None
+    max_send_queue_size = 1000
 
     @classmethod
     def init(cls, display=True):
-        
+        if cls.last_loaded == 0:
+            cls.loadFromEnv()
         cls.last_loaded = time.time()
         return cls.load(display)
 
+    @classmethod
+    def loadFromEnv(cls):
+        for key, value in Configuration.items():
+            setattr(cls, key, value)
+        if not hasattr(cls, "license") and hasattr(cls, "accesskey"):
+            setattr(cls, "license", getattr(cls, "accesskey"))
+
     @classmethod
     def load(cls, display=True):
         home = 'WHATAP_HOME'
@@ -29,8 +39,9 @@ class Configure(object):
             last_modified = os.path.getmtime(whatap_config)
             if cls.last_config_modified >= last_modified:
                 return True
-            for key, value in Configuration.items():
+            for key, value in os.environ.items():
                 setattr(cls, key, value)
+            
             with open(whatap_config, 'r') as f:
                 for line in f:
                     cls.last_config_modified = last_modified
@@ -41,11 +52,15 @@ class Configure(object):
                         key, value = line.split('=')
                         key = key.strip()
                         value = value.strip()
+                        if key and key.endswith("_set"):
+                            value = value.split(",")
                         cls.setProperty(key, value)
                     
                     except Exception as e:
                         print('WHATAP: ', e)
                         continue
+            if not getattr(cls, "license") and getattr(cls, "accesskey"):
+                setattr(cls, "license", getattr(cls, "accesskey"))
             for callback in cls.observers:
                 callback()
         except Exception as e:
@@ -63,6 +78,8 @@ class Configure(object):
     def getProperty(cls, key, value=None):
         if hasattr(cls, key):
             return getattr(cls, key)
+        elif key in os.environ:
+            return os.environ[key]
         else:
             return value
     

{whatap-python-0.1.120 → whatap_python-0.3.35}/whatap/io/data_outputx.py

@@ -256,3 +256,7 @@ class DataOutputX(object):
             buffer_arr = bytearray(self.buffer.getvalue())
             buffer_arr[pos:pos + 4] = b.getvalue()
             self.buffer = io.BytesIO(buffer_arr)
+
+    def size(self):
+        return self.buffer.getbuffer().nbytes
+        

whatap_python-0.3.35/whatap/net/async_sender.py

@@ -0,0 +1,64 @@
+import queue, time, threading
+from . import udp_session
+from whatap.conf.configure import Configure as conf
+from whatap.trace.simple_trace_context import SimpleTraceContext
+
+q = queue.Queue(conf.max_send_queue_size)
+
+from enum import Enum
+class SendType(Enum):
+    DATAS = 1
+    RELAY = 2
+
+def send_packet( packet_type, ctx, datas=[]):
+    ctx = SimpleTraceContext(ctx.id, ctx.start_time, ctx.elapsed, ctx.thread_id)
+    _initThread()
+    global q
+    if q.full():
+        return
+    q.put((SendType.DATAS, (packet_type, ctx, datas)))
+
+def send_relaypack( packbytes):
+    _initThread()
+    global q
+    if q.full():
+        return
+    q.put((SendType.RELAY, packbytes))
+
+def startWhatapThread():
+    def __sendPackets():
+        global q
+        while True:
+            packet_env = q.get()
+            if not packet_env:
+                time.sleep(0.1)
+                continue
+            sendType,params = packet_env
+            if sendType == SendType.DATAS:
+                packet_type, ctx, datas = params
+                udp_session.UdpSession.send_packet(packet_type, ctx, datas)
+            elif sendType == SendType.RELAY:
+                packbytes = params
+                udp_session.UdpSession.send_relaypack(packbytes)    
+    t = threading.Thread(target=__sendPackets)
+    t.setDaemon(True)
+    t.start()
+
+_lock = threading.Lock()
+_initialized = False
+
+def _initThread():
+    global _lock, _initialized
+    if _initialized:
+        return
+    try:
+        if _initialized:
+            return
+        _lock.acquire()
+        if _initialized:
+            return
+        _initialized=True
+        startWhatapThread()
+    finally:
+        _lock.release()
+