wmglobalqueue 2.4.5.1__py3-none-any.whl

Utils/ProcessStats.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+File       : ProcessStats.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: This module provide ability to get detailed information about
+certain UNIX process, referred by PID. It is based on psutil and threading
+python modules. It can be used in any web/data-server to get live status
+about the running server. Here is an example
+
+.. doctest:
+
+    from Utils.procstats import status threadStack
+
+    class SomeServer(object):
+        def __init__(self):
+        self.time0 = time.time()
+        def status(self):
+            sdict = {'server': processStatus()}
+            sdict['server'].update({'uptime': time.time()-self.time0})
+            sdict.update(threadStack())
+            return sdict
+
+The exposed status endpoint will return a json dictionary containing server
+upteima and cpu/mem/threads information.
+"""
+
+# system modules
+import os
+import sys
+import json
+import time
+import argparse
+import threading
+import traceback
+
+try:
+    import psutil
+except ImportError:
+    pass
+
+def _baseProcessStatusFormat(pid=None):
+    if not pid:
+        pid = os.getpid()
+    ttime = time.localtime()
+    tstamp = time.strftime('%d/%b/%Y:%H:%M:%S', ttime)
+    return {'pid': pid, 'timestamp': tstamp, 'time': time.time()}
+
+def processStatus(pid=None):
+    "Return status of the process in a dictionary format"
+
+    pdict = _baseProcessStatusFormat(pid)
+    if 'psutil' not in sys.modules:
+        return pdict
+    proc = psutil.Process(pid)
+    pdict.update(proc.as_dict())
+    return pdict
+
+def processStatusDict(pid=None):
+    "Return status of the process in a dictionary format"
+    pdict = _baseProcessStatusFormat(pid)
+    if 'psutil' not in sys.modules:
+        return pdict
+    proc = psutil.Process(pid)
+    pdict.update({"cpu_times": dict(proc.cpu_times()._asdict())})
+    pdict.update({"cpu_percent": proc.cpu_percent(interval=1.0)})
+    pdict.update({"cpu_num": proc.cpu_num()})
+    pdict.update({"memory_full_info": dict(proc.memory_full_info()._asdict())})
+    pdict.update({"memory_percent": proc.memory_percent()})
+    return pdict
+
+def threadStack():
+    """
+    Return context of all threads in dictionary format where individual
+    thread information stored in its own dicts and all threads are groupped
+    into threads list. Code based on example from StackOverflow:
+    http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
+    """
+    tdict = {}
+    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    threads = []
+    for tid, stack in list(sys._current_frames().items()):
+        tdict = {"thead": id2name.get(tid, ""), "thead_id": tid}
+        stacklist = []
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            sdict = dict(filename=filename, line_number=lineno, name=name, line=line)
+            stacklist.append(sdict)
+        tdict.update({"stack": stacklist})
+        threads.append(tdict)
+    return dict(threads=threads)
+
+def processThreadsInfo(pid):
+    """
+    Provides information about process and its threads using psutils
+    :param pid: process PID, string
+    :return: dictionary statistics about process and threads
+    """
+    try:
+        process = psutil.Process(pid)
+
+        # Get overall process details
+        process_info = {
+            "process_name": process.name(),
+            "pid": process.pid,
+            "status": process.status(),
+            "ppid": process.ppid(),  # Parent Process ID
+            "is_running":process.is_running(), # NOTE: Zombie processes could also be reported as running
+            "cmdline": process.cmdline(),  # Full command-line arguments
+            "cpu_usage_percent": process.cpu_percent(interval=1.0),  # CPU usage in percentage
+            "memory_usage_percent": process.memory_percent(),  # RAM usage in percentage
+            "memory_rss": process.memory_info().rss,  # Resident Set Size (bytes)
+            "num_open_files": len(process.open_files()),  # Number of open file descriptors
+            "num_connections": len(process.connections()),  # Number of active connections
+            "threads": []
+        }
+
+        # Iterate over threads to get per-thread details
+        for thread in process.threads():
+            if str(pid) == str(thread.id):
+                continue
+            thread_id = thread.id
+
+            thread_info = {
+                "thread_id": thread_id,
+                "user_time": thread.user_time,  # CPU time spent in user mode
+                "system_time": thread.system_time,  # CPU time spent in system mode
+                "cpu_usage_percent": None,
+                "memory_usage_bytes": None,
+                "num_open_files": None,
+                "num_connections": None,
+                "is_running":process.is_running(), # NOTE: Zombie processes could also be reported as running
+                "status": "unknown",
+                "name": "thread"
+            }
+            if str(pid) == str(thread_id):
+                thread_info['name'] = "process"
+
+            # Try getting thread status
+            try:
+                thread_status = psutil.Process(thread_id).status()
+                thread_info["status"] = thread_status
+            except psutil.NoSuchProcess:
+                thread_info["status"] = psutil.STATUS_ZOMBIE
+
+            # Try getting per-thread CPU usage
+            try:
+                thread_info["cpu_usage_percent"] = psutil.Process(thread_id).cpu_percent(interval=1.0)
+            except Exception:
+                thread_info["cpu_usage_percent"] = "N/A"
+
+            # Try getting per-thread memory usage
+            try:
+                thread_info["memory_usage_bytes"] = psutil.Process(thread_id).memory_info().rss
+            except Exception:
+                thread_info["memory_usage_bytes"] = "N/A"
+
+            # Try getting per-thread open file descriptors
+            try:
+                thread_info["num_open_files"] = len(psutil.Process(thread_id).open_files())
+            except Exception:
+                thread_info["num_open_files"] = "N/A"
+
+            # Try getting per-thread open connections
+            try:
+                thread_info["num_connections"] = len(psutil.Process(thread_id).connections())
+            except Exception:
+                thread_info["num_connections"] = "N/A"
+
+            # Append to process_info
+            process_info["threads"].append(thread_info)
+
+        return process_info
+    except psutil.NoSuchProcess:
+        return {"error": f"No process found with PID {pid}"}
+    except Exception as e:
+        return {"error": str(e)}
+
+def main():
+    "Main function to use this module as a stand-along script."
+    parser = argparse.ArgumentParser(prog='PROG')
+    parser.add_argument("--pid", action="store", dest="pid", help="process id")
+    opts = parser.parse_args()
+
+    pdict = processStatus(int(opts.pid))
+    pdict.update(threadStack())
+    print(f"Process status for {opts.pid}")
+    print(json.dumps(pdict, indent=4))
+
+    processInfo = processThreadsInfo(int(opts.pid))
+    print(f"Process/threads status for {opts.pid}")
+    print(json.dumps(processInfo, indent=4))
+
+if __name__ == '__main__':
+    main()

Utils/PythonVersion.py

@@ -0,0 +1,17 @@
+"""
+Easily get the version of the python interpreter at runtime
+"""
+
+ # Jenkins CI
+
+import sys
+import pickle
+
+PY3 = sys.version_info[0] == 3
+PY2 = sys.version_info[0] == 2
+
+# We need to keep compatibility between multiple python versions
+# Further details at: https://github.com/dmwm/WMCore/pull/10726
+# For PY2: set highest protocol to 2
+# For PY3: set highest protocol to 4 (compatible with python3.6 and python3.8)
+HIGHEST_PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL if PY2 else 4

Utils/Signals.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : Signals.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: This module provide set of hooks to running program
+The dumpthreads callback function will be called every time when
+running program will receive SIGUSR1 signal. E.g., in unix shell
+just do
+shell# kill -s SIGUSR1 <pid>
+"""
+
+# system modules
+import sys
+import signal
+import threading
+import traceback
+
+def dumpthreads(isignal, iframe):
+    """
+    Dump context of all threads upon given signal
+    http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
+    """
+    print("DAS stack, signal=%s, frame=%s" % (isignal, iframe))
+    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    code = []
+    for tid, stack in list(sys._current_frames().items()):
+        code.append("\n# Thread: %s(%d)" % (id2name.get(tid,""), tid))
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
+            if  line:
+                code.append("  %s" % (line.strip()))
+    print("\n".join(code))
+
+signal.signal(signal.SIGUSR1, dumpthreads)

Utils/TemporaryEnvironment.py

@@ -0,0 +1,27 @@
+#! /usr/bin/env python
+
+import contextlib
+import os
+
+
+@contextlib.contextmanager
+def tmpEnv(**environ):
+    """
+    Temporarily set the process environment variables.
+
+    >>> with tmpEnv(PLUGINS_DIR=u'test/plugins'):
+    ...   "PLUGINS_DIR" in os.environ
+    True
+
+    >>> "PLUGINS_DIR" in os.environ
+    False
+
+    :param environ: Environment variables to set
+    """
+    oldEnviron = dict(os.environ)
+    os.environ.update(environ)
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(oldEnviron)

Utils/Throttled.py

@@ -0,0 +1,227 @@
+# pylint: disable-msg=C0103,C0111,W0212
+"""
+Throttled module defines necessary classes to throttle context.
+Should be used in Web server APIs which wants to throttle clients.
+
+The counter based approach, implemented in _ThrottleCounter class,
+defines throttling on total number of user access to the underlying
+app. But its counter depends on time execution of the API.
+
+The time based approach, implemented in _ThrottleTimeCount class,
+count total number of user accesses within specified time interval
+regardless of time execution of the underlying API(s).
+
+Here is an example how to use this module/classes:
+
+# counter based throttline
+
+from Utils.Throttling import UserThrottle
+thr = UserThrottle(limit=5) # adjust threshold limit here
+@thr.make_throttled()
+def api():
+    # define your api logic here
+
+# time based throttling
+
+from Utils.Throttling import UserThrottleTime
+thr = UserThrottleTime(limit=5) # adjust threshold limit here
+@thr.make_throttled(trange=2) # adjust trange (in sec) here
+def api():
+    # define your api logic here
+
+"""
+
+
+# standard modules
+import time
+from builtins import object
+import logging
+import threading
+
+# cherrypy modules
+import cherrypy
+
+
+class _ThrottleCounter(object):
+    """
+    _ThrottleCounter class defines throttle parameter and
+    enter/exit methods to work with `with` context.
+    """
+
+    def __init__(self, throttle, user, debug=False):
+        self.throttle = throttle
+        self.user = user
+        if debug:
+            self.throttle.logger.setLevel(logging.DEBUG)
+
+    def __enter__(self):
+        "Define enter method for `with` context"
+        ctr = self.throttle._incUser(self.user)
+        msg = "Entering throttled function with counter %d for user %s" \
+                % (ctr, self.user)
+        self.throttle.logger.debug(msg)
+        if ctr > self.throttle.getLimit():
+            msg = "The current number of active operations for this resource"
+            msg += " exceeds the limit of %d for user %s" \
+                   % (self.throttle.getLimit(), self.user)
+            raise cherrypy.HTTPError("429 Too Many Requests", msg)
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        "Define exit method for `with` context"
+        ctr = self.throttle._decUser(self.user)
+        msg = "Exiting throttled function with counter %d for user %s" \
+                % (ctr, self.user)
+        self.throttle.logger.debug(msg)
+
+class _ThrottleTimeCounter(object):
+    """
+    _ThrottleTimeCounter class defines time range throttled mechanism
+    """
+
+    def __init__(self, throttle, user, trange):
+        self.throttle = throttle
+        self.user = user
+        self.trange = trange
+
+    def __enter__(self):
+        "Define enter method for `with` context"
+        ctr = self.throttle._incUser(self.user, self.trange)
+        if ctr > self.throttle.getLimit():
+            self.throttle._decUser(self.user)
+            msg = "The current number of active operations for this resource"
+            msg += " exceeds the limit of %d for user %s in last %s sec" \
+                   % (self.throttle.getLimit(), self.user, self.trange)
+            raise cherrypy.HTTPError("429 Too Many Requests", msg)
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        "Define exit method for `with` context"
+        pass
+
+
+class UserThrottleTime(object):
+    """
+    UserThrottle class defines how to handle throttle time range based mechanism.
+    """
+
+    def __init__(self, limit=3):
+        self.lock = threading.Lock()
+        self.users = {}
+        self.users_time = {}
+        self.limit = limit
+
+    def getLimit(self):
+        "Return throttle limit"
+        return self.limit
+
+    def throttleContext(self, user, trange=60):
+        """
+        Define throttle context via _ThrottleTimeCounter class
+        which will count number of requests given user made in trange
+        :param user: user name
+        :param trange: time range while keep user access history
+        """
+        return _ThrottleTimeCounter(self, user, trange)
+
+    def make_throttled(self, trange=60):
+        "decorator for throttled context"
+        def throttled_decorator(fn):
+            def throttled_wrapped_function(*args, **kw):
+                username = cherrypy.request.user.get('login', 'Unknown') \
+                        if hasattr(cherrypy.request, 'user') else 'Unknown'
+                with self.throttleContext(username, trange):
+                    return fn(*args, **kw)
+            return throttled_wrapped_function
+        return throttled_decorator
+
+    def reset(self, user):
+        "Reset user activities, i.e. access counter and last time of access"
+        self.users[user] = 0
+        self.users_time[user] = time.time()
+
+    def _incUser(self, user, trange=60):
+        "increment user count"
+        with self.lock:
+            self.users.setdefault(user, 0)
+            last_time = self.users_time.setdefault(user, time.time())
+            # increase counter within our trange
+            if abs(time.time()-last_time) < trange:
+                self.users[user] += 1
+            else:
+                self.reset(user)
+        return self.users[user]
+
+    def _decUser(self, user, trange=60):
+        "decrecrement user count"
+        with self.lock:
+            # decrease counter outside of our trange
+            last_time = self.users_time[user]
+            if abs(time.time()-last_time) < trange:
+                # gradually decrease user counter based on his/her elapsed time
+                step = int(trange - abs(time.time()-last_time))
+                self.users[user] -= step
+                if self.users[user] < 0:
+                    self.reset(user)
+            else:
+                self.reset(user)
+
+class UserThrottle(object):
+    """
+    UserThrottle class defines throttle counter based mechanism.
+    """
+
+    def __init__(self, limit=3):
+        self.lock = threading.Lock()
+        self.tls = threading.local()
+        self.users = {}
+        self.limit = limit
+        self.logger = logging.getLogger("WMCore.UserThrottle")
+
+    def getLimit(self):
+        "Return throttle limit"
+        return self.limit
+
+    def throttleContext(self, user, debug=False):
+        "defint throttle context"
+        self.users.setdefault(user, 0)
+        return _ThrottleCounter(self, user, debug)
+
+    def make_throttled(self, debug=False):
+        """
+        decorator for throttled context
+        """
+        def throttled_decorator(fn):
+            """
+            A decorator
+            """
+            def throttled_wrapped_function(*args, **kw):
+                """
+                A wrapped function.
+                """
+                username = cherrypy.request.user.get('login', 'Unknown') \
+                        if hasattr(cherrypy.request, 'user') else 'Unknown'
+                with self.throttleContext(username, debug):
+                    return fn(*args, **kw)
+            return throttled_wrapped_function
+        return throttled_decorator
+
+    def _incUser(self, user):
+        "increment user count"
+        retval = 0
+        with self.lock:
+            retval = self.users[user]
+            if getattr(self.tls, 'count', None) is None:
+                self.tls.count = 0
+            self.tls.count += 1
+            if self.tls.count == 1:
+                self.users[user] = retval + 1
+        return retval
+
+    def _decUser(self, user):
+        "decrecrement user count"
+        retval = 0
+        with self.lock:
+            retval = self.users[user]
+            self.tls.count -= 1
+            if self.tls.count == 0:
+                self.users[user] = retval - 1
+        return retval

Utils/Timers.py

@@ -0,0 +1,130 @@
+#!/bin/env python
+"""
+Utilities related to timing and performance testing
+"""
+
+from builtins import object
+import logging
+import time
+import calendar
+from datetime import tzinfo, timedelta
+
+
+def gmtimeSeconds():
+    """
+    Return GMT time in seconds
+    """
+    return int(time.mktime(time.gmtime()))
+
+
+def encodeTimestamp(secs):
+    """
+    Encode second since epoch to a string GMT timezone representation
+    :param secs: input timestamp value (either int or float) in seconds since epoch
+
+    :return: time string in GMT timezone representation
+    """
+    if not isinstance(secs, (int, float)):
+        raise Exception("Wrong input, should be seconds since epoch either int or float value")
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(secs)))
+
+
+def decodeTimestamp(timeString):
+    """
+    Decode timestamps in provided document
+    :param timeString: timestamp string represention in GMT timezone, see encodeTimestamp
+
+    :return: seconds since ecouch in GMT timezone
+    """
+    if not isinstance(timeString, str):
+        raise Exception("Wrong input, should be time string in GMT timezone representation")
+    return calendar.timegm(time.strptime(timeString, "%Y-%m-%dT%H:%M:%SZ"))
+
+
+def timeFunction(func):
+    """
+    source: https://www.andreas-jung.com/contents/a-python-decorator-for-measuring-the-execution-time-of-methods
+
+    Decorator function to measure how long a method/function takes to run
+    It returns a tuple with:
+      * wall clock time spent
+      * returned result of the function
+      * the function name
+    """
+
+    def wrapper(*arg, **kw):
+        t1 = time.time()
+        res = func(*arg, **kw)
+        t2 = time.time()
+        return round((t2 - t1), 4), res, func.__name__
+
+    return wrapper
+
+
+class CodeTimer(object):
+    """
+    A context manager for timing function calls.
+    Adapted from https://www.blog.pythonlibrary.org/2016/05/24/python-101-an-intro-to-benchmarking-your-code/
+
+    Use like
+
+    with CodeTimer(label='Doing something'):
+        do_something()
+    """
+
+    def __init__(self, label='The function', logger=None):
+        self.start = time.time()
+        self.label = label
+        self.logger = logger or logging.getLogger()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        end = time.time()
+        runtime = round((end - self.start), 3)
+        self.logger.info(f"{self.label} took {runtime} seconds to complete")
+
+
+class LocalTimezone(tzinfo):
+    """
+    A required python 2 class to determine current timezone for formatting rfc3339 timestamps
+    Required for sending alerts to the MONIT AlertManager
+    Can be removed once WMCore starts using python3
+
+    Details of class can be found at: https://docs.python.org/2/library/datetime.html#tzinfo-objects
+    """
+
+    def __init__(self):
+        super(LocalTimezone, self).__init__()
+        self.ZERO = timedelta(0)
+        self.STDOFFSET = timedelta(seconds=-time.timezone)
+        if time.daylight:
+            self.DSTOFFSET = timedelta(seconds=-time.altzone)
+        else:
+            self.DSTOFFSET = self.STDOFFSET
+
+        self.DSTDIFF = self.DSTOFFSET - self.STDOFFSET
+
+    def utcoffset(self, dt):
+        if self._isdst(dt):
+            return self.DSTOFFSET
+        else:
+            return self.STDOFFSET
+
+    def dst(self, dt):
+        if self._isdst(dt):
+            return self.DSTDIFF
+        else:
+            return self.ZERO
+
+    def tzname(self, dt):
+        return time.tzname[self._isdst(dt)]
+
+    def _isdst(self, dt):
+        tt = (dt.year, dt.month, dt.day,
+              dt.hour, dt.minute, dt.second,
+              dt.weekday(), 0, 0)
+        stamp = time.mktime(tt)
+        tt = time.localtime(stamp)
+        return tt.tm_isdst > 0