wmglobalqueue 2.3.10__py3-none-any.whl

Utils/Throttled.py

@@ -0,0 +1,227 @@
+# pylint: disable-msg=C0103,C0111,W0212
+"""
+Throttled module defines necessary classes to throttle context.
+Should be used in Web server APIs which wants to throttle clients.
+
+The counter based approach, implemented in _ThrottleCounter class,
+defines throttling on total number of user access to the underlying
+app. But its counter depends on time execution of the API.
+
+The time based approach, implemented in _ThrottleTimeCount class,
+count total number of user accesses within specified time interval
+regardless of time execution of the underlying API(s).
+
+Here is an example how to use this module/classes:
+
+# counter based throttline
+
+from Utils.Throttling import UserThrottle
+thr = UserThrottle(limit=5) # adjust threshold limit here
+@thr.make_throttled()
+def api():
+    # define your api logic here
+
+# time based throttling
+
+from Utils.Throttling import UserThrottleTime
+thr = UserThrottleTime(limit=5) # adjust threshold limit here
+@thr.make_throttled(trange=2) # adjust trange (in sec) here
+def api():
+    # define your api logic here
+
+"""
+
+
+# standard modules
+import time
+from builtins import object
+import logging
+import threading
+
+# cherrypy modules
+import cherrypy
+
+
+class _ThrottleCounter(object):
+    """
+    _ThrottleCounter class defines throttle parameter and
+    enter/exit methods to work with `with` context.
+    """
+
+    def __init__(self, throttle, user, debug=False):
+        self.throttle = throttle
+        self.user = user
+        if debug:
+            self.throttle.logger.setLevel(logging.DEBUG)
+
+    def __enter__(self):
+        "Define enter method for `with` context"
+        ctr = self.throttle._incUser(self.user)
+        msg = "Entering throttled function with counter %d for user %s" \
+                % (ctr, self.user)
+        self.throttle.logger.debug(msg)
+        if ctr > self.throttle.getLimit():
+            msg = "The current number of active operations for this resource"
+            msg += " exceeds the limit of %d for user %s" \
+                   % (self.throttle.getLimit(), self.user)
+            raise cherrypy.HTTPError("429 Too Many Requests", msg)
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        "Define exit method for `with` context"
+        ctr = self.throttle._decUser(self.user)
+        msg = "Exiting throttled function with counter %d for user %s" \
+                % (ctr, self.user)
+        self.throttle.logger.debug(msg)
+
+class _ThrottleTimeCounter(object):
+    """
+    _ThrottleTimeCounter class defines time range throttled mechanism
+    """
+
+    def __init__(self, throttle, user, trange):
+        self.throttle = throttle
+        self.user = user
+        self.trange = trange
+
+    def __enter__(self):
+        "Define enter method for `with` context"
+        ctr = self.throttle._incUser(self.user, self.trange)
+        if ctr > self.throttle.getLimit():
+            self.throttle._decUser(self.user)
+            msg = "The current number of active operations for this resource"
+            msg += " exceeds the limit of %d for user %s in last %s sec" \
+                   % (self.throttle.getLimit(), self.user, self.trange)
+            raise cherrypy.HTTPError("429 Too Many Requests", msg)
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        "Define exit method for `with` context"
+        pass
+
+
+class UserThrottleTime(object):
+    """
+    UserThrottle class defines how to handle throttle time range based mechanism.
+    """
+
+    def __init__(self, limit=3):
+        self.lock = threading.Lock()
+        self.users = {}
+        self.users_time = {}
+        self.limit = limit
+
+    def getLimit(self):
+        "Return throttle limit"
+        return self.limit
+
+    def throttleContext(self, user, trange=60):
+        """
+        Define throttle context via _ThrottleTimeCounter class
+        which will count number of requests given user made in trange
+        :param user: user name
+        :param trange: time range while keep user access history
+        """
+        return _ThrottleTimeCounter(self, user, trange)
+
+    def make_throttled(self, trange=60):
+        "decorator for throttled context"
+        def throttled_decorator(fn):
+            def throttled_wrapped_function(*args, **kw):
+                username = cherrypy.request.user.get('login', 'Unknown') \
+                        if hasattr(cherrypy.request, 'user') else 'Unknown'
+                with self.throttleContext(username, trange):
+                    return fn(*args, **kw)
+            return throttled_wrapped_function
+        return throttled_decorator
+
+    def reset(self, user):
+        "Reset user activities, i.e. access counter and last time of access"
+        self.users[user] = 0
+        self.users_time[user] = time.time()
+
+    def _incUser(self, user, trange=60):
+        "increment user count"
+        with self.lock:
+            self.users.setdefault(user, 0)
+            last_time = self.users_time.setdefault(user, time.time())
+            # increase counter within our trange
+            if abs(time.time()-last_time) < trange:
+                self.users[user] += 1
+            else:
+                self.reset(user)
+        return self.users[user]
+
+    def _decUser(self, user, trange=60):
+        "decrecrement user count"
+        with self.lock:
+            # decrease counter outside of our trange
+            last_time = self.users_time[user]
+            if abs(time.time()-last_time) < trange:
+                # gradually decrease user counter based on his/her elapsed time
+                step = int(trange - abs(time.time()-last_time))
+                self.users[user] -= step
+                if self.users[user] < 0:
+                    self.reset(user)
+            else:
+                self.reset(user)
+
+class UserThrottle(object):
+    """
+    UserThrottle class defines throttle counter based mechanism.
+    """
+
+    def __init__(self, limit=3):
+        self.lock = threading.Lock()
+        self.tls = threading.local()
+        self.users = {}
+        self.limit = limit
+        self.logger = logging.getLogger("WMCore.UserThrottle")
+
+    def getLimit(self):
+        "Return throttle limit"
+        return self.limit
+
+    def throttleContext(self, user, debug=False):
+        "defint throttle context"
+        self.users.setdefault(user, 0)
+        return _ThrottleCounter(self, user, debug)
+
+    def make_throttled(self, debug=False):
+        """
+        decorator for throttled context
+        """
+        def throttled_decorator(fn):
+            """
+            A decorator
+            """
+            def throttled_wrapped_function(*args, **kw):
+                """
+                A wrapped function.
+                """
+                username = cherrypy.request.user.get('login', 'Unknown') \
+                        if hasattr(cherrypy.request, 'user') else 'Unknown'
+                with self.throttleContext(username, debug):
+                    return fn(*args, **kw)
+            return throttled_wrapped_function
+        return throttled_decorator
+
+    def _incUser(self, user):
+        "increment user count"
+        retval = 0
+        with self.lock:
+            retval = self.users[user]
+            if getattr(self.tls, 'count', None) is None:
+                self.tls.count = 0
+            self.tls.count += 1
+            if self.tls.count == 1:
+                self.users[user] = retval + 1
+        return retval
+
+    def _decUser(self, user):
+        "decrecrement user count"
+        retval = 0
+        with self.lock:
+            retval = self.users[user]
+            self.tls.count -= 1
+            if self.tls.count == 0:
+                self.users[user] = retval - 1
+        return retval

Utils/Timers.py

@@ -0,0 +1,130 @@
+#!/bin/env python
+"""
+Utilities related to timing and performance testing
+"""
+
+from builtins import object
+import logging
+import time
+import calendar
+from datetime import tzinfo, timedelta
+
+
+def gmtimeSeconds():
+    """
+    Return GMT time in seconds
+    """
+    return int(time.mktime(time.gmtime()))
+
+
+def encodeTimestamp(secs):
+    """
+    Encode second since epoch to a string GMT timezone representation
+    :param secs: input timestamp value (either int or float) in seconds since epoch
+
+    :return: time string in GMT timezone representation
+    """
+    if not isinstance(secs, (int, float)):
+        raise Exception("Wrong input, should be seconds since epoch either int or float value")
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(secs)))
+
+
+def decodeTimestamp(timeString):
+    """
+    Decode timestamps in provided document
+    :param timeString: timestamp string represention in GMT timezone, see encodeTimestamp
+
+    :return: seconds since ecouch in GMT timezone
+    """
+    if not isinstance(timeString, str):
+        raise Exception("Wrong input, should be time string in GMT timezone representation")
+    return calendar.timegm(time.strptime(timeString, "%Y-%m-%dT%H:%M:%SZ"))
+
+
+def timeFunction(func):
+    """
+    source: https://www.andreas-jung.com/contents/a-python-decorator-for-measuring-the-execution-time-of-methods
+
+    Decorator function to measure how long a method/function takes to run
+    It returns a tuple with:
+      * wall clock time spent
+      * returned result of the function
+      * the function name
+    """
+
+    def wrapper(*arg, **kw):
+        t1 = time.time()
+        res = func(*arg, **kw)
+        t2 = time.time()
+        return round((t2 - t1), 4), res, func.__name__
+
+    return wrapper
+
+
+class CodeTimer(object):
+    """
+    A context manager for timing function calls.
+    Adapted from https://www.blog.pythonlibrary.org/2016/05/24/python-101-an-intro-to-benchmarking-your-code/
+
+    Use like
+
+    with CodeTimer(label='Doing something'):
+        do_something()
+    """
+
+    def __init__(self, label='The function', logger=None):
+        self.start = time.time()
+        self.label = label
+        self.logger = logger or logging.getLogger()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        end = time.time()
+        runtime = round((end - self.start), 3)
+        self.logger.info(f"{self.label} took {runtime} seconds to complete")
+
+
+class LocalTimezone(tzinfo):
+    """
+    A required python 2 class to determine current timezone for formatting rfc3339 timestamps
+    Required for sending alerts to the MONIT AlertManager
+    Can be removed once WMCore starts using python3
+
+    Details of class can be found at: https://docs.python.org/2/library/datetime.html#tzinfo-objects
+    """
+
+    def __init__(self):
+        super(LocalTimezone, self).__init__()
+        self.ZERO = timedelta(0)
+        self.STDOFFSET = timedelta(seconds=-time.timezone)
+        if time.daylight:
+            self.DSTOFFSET = timedelta(seconds=-time.altzone)
+        else:
+            self.DSTOFFSET = self.STDOFFSET
+
+        self.DSTDIFF = self.DSTOFFSET - self.STDOFFSET
+
+    def utcoffset(self, dt):
+        if self._isdst(dt):
+            return self.DSTOFFSET
+        else:
+            return self.STDOFFSET
+
+    def dst(self, dt):
+        if self._isdst(dt):
+            return self.DSTDIFF
+        else:
+            return self.ZERO
+
+    def tzname(self, dt):
+        return time.tzname[self._isdst(dt)]
+
+    def _isdst(self, dt):
+        tt = (dt.year, dt.month, dt.day,
+              dt.hour, dt.minute, dt.second,
+              dt.weekday(), 0, 0)
+        stamp = time.mktime(tt)
+        tt = time.localtime(stamp)
+        return tt.tm_isdst > 0

Utils/Timestamps.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+"""
+_Timestamps_ module designed to insert proper WM timing into WMCore FJR.
+Usage: python3 Timestamps.py --reportFile=$outputFile --wmJobStart=<sec> --wmJobEnd=<sec>
+where outputFile represents output FJR file, and wmJobStart/wmJobEnd represent
+start and end time of WM job, respectively.
+
+NOTE: this script should reside in Utils area of WMCore repository since it is required
+at run time on condor node where there is no WMCore installation and we only provide WMCore.zip
+archive, and worker node may not have unzip tool to extract this script.
+
+"""
+
+import getopt
+import logging
+import os
+import pickle
+import sys
+
+
+# script options
+options = {"reportFile=": "", "wmJobStart=": '', "wmJobEnd=": ''}
+
+
+def addTimestampMetrics(config, wmJobStart, wmJobEnd):
+    """
+    Adjust timestamp metrics of provied FJR data.
+
+    :param config: input FJR data
+    :param wmJobStart: start time of WM job in seconds
+    :param wmJobEnd: end time of WM job in seconds
+    """
+    wmTiming = config.section_('WMTiming')
+    wmTiming.WMJobStart = wmJobStart
+    wmTiming.WMJobEnd = wmJobEnd
+    wmTiming.WMTotalWallClockTime = wmJobEnd - wmJobStart
+    return config
+
+
+def main():
+    """
+    Main function of the script performs business logic:
+    - parse input arguments
+    - read provided input FJR
+    - adjust timestamp metrics of FJR data
+    - write out FJR
+    """
+    try:
+        opts, _ = getopt.getopt(sys.argv[1:], "", list(options.keys()))
+    except getopt.GetoptError as ex:
+        msg = "Error processing commandline args:\n"
+        msg += str(ex)
+        logging.error(msg)
+        sys.exit(1)
+
+    for opt, arg in opts:
+        if opt == "--reportFile":
+            reportFile = arg
+        if opt == "--wmJobStart":
+            wmJobStart = float(arg)
+        if opt == "--wmJobEnd":
+            wmJobEnd = float(arg)
+
+    # read content of given FJR report file
+    data = {}
+    with open(reportFile, 'rb') as istream:
+        data = pickle.load(istream)
+
+    # adjust FJR data with provided metrics
+    data = addTimestampMetrics(data, wmJobStart, wmJobEnd)
+
+    # write content of FJR back to report file
+    reportOutFile = reportFile + ".new"
+    msg = f"Adding wmJobTime metric to {reportFile}"
+    with open(reportOutFile, 'wb') as ostream:
+        logging.info(msg)
+        pickle.dump(data, ostream)
+
+    # if we successfully wrote reportOutFile we can swap it with input one
+    sizeStatus = os.path.getsize(reportOutFile) >= os.path.getsize(reportFile)
+    if os.path.isfile(reportOutFile) and sizeStatus:
+        os.rename(reportOutFile, reportFile)
+
+
+if __name__ == '__main__':
+    main()

Utils/TokenManager.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#pylint: disable=W0212
+"""
+File       : TokenManager.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: IAM token manager
+             by default it relies on the following environment/parameters:
+             - IAM_TOKEN is either file name or actual token value
+             - https://cms-auth.web.cern.ch/jwk is default CMS IAM provider
+             - https://wlcg.cern.ch/jwt/v1/any is default for audience parameter
+"""
+
+# system modules
+import os
+import ssl
+import time
+import logging
+import traceback
+
+# third part library
+try:
+    import jwt
+except ImportError:
+    traceback.print_exc()
+    jwt = None
+
+from Utils.Utilities import encodeUnicodeToBytes
+
+# prevent "SSL: CERTIFICATE_VERIFY_FAILED" error
+# this will cause pylint warning W0212, therefore we ignore it above
+ssl._create_default_https_context = ssl._create_unverified_context
+
+
+def readToken(name=None):
+    """
+    Read IAM token either from environment or file name
+    :param name: ether file name containing token or environment name which hold the token value.
+    If not provided it will be assumed to read token from IAM_TOKEN environment.
+    :return: token or None
+    """
+    if name and os.path.exists(name):
+        token = None
+        with open(name, 'r', encoding='utf-8') as istream:
+            token = istream.read()
+        return token
+    if name:
+        return os.environ.get(name)
+    return os.environ.get("IAM_TOKEN")
+
+
+def tokenData(token, url="https://cms-auth.web.cern.ch/jwk", audUrl="https://wlcg.cern.ch/jwt/v1/any"):
+    """
+    inspect and extract token data
+    :param token: token string
+    :param url: IAM provider URL
+    :param audUrl: audience string
+    """
+    if not token or not jwt:
+        return {}
+    if isinstance(token, str):
+        token = encodeUnicodeToBytes(token)
+    jwksClient = jwt.PyJWKClient(url)
+    signingKey = jwksClient.get_signing_key_from_jwt(token)
+    key = signingKey.key
+    headers = jwt.get_unverified_header(token)
+    alg = headers.get('alg', 'RS256')
+    data = jwt.decode(
+        token,
+        key,
+        algorithms=[alg],
+        audience=audUrl,
+        options={"verify_exp": True},
+        )
+    return data
+
+
+def isValidToken(token):
+    """
+    check if given token is valid or not
+
+    :param token: token string
+    :return: true or false
+    """
+    tokenDict = {}
+    tokenDict = tokenData(token)
+    exp = tokenDict.get('exp', 0)  # expire, seconds since epoch
+    if not exp or exp < time.time():
+        return False
+    return True
+
+
+class TokenManager():
+    """
+    TokenManager class handles IAM tokens
+    """
+
+    def __init__(self,
+                 name=None,
+                 url="https://cms-auth.web.cern.ch/jwk",
+                 audUrl="https://wlcg.cern.ch/jwt/v1/any",
+                 logger=None):
+        """
+        Token manager reads IAM tokens either from file or env.
+        It caches token along with expiration timestamp.
+        By default the env variable to use is IAM_TOKEN.
+        :param name: string representing either file or env where we should read token from
+        :param url: IAM provider URL
+        :param audUrl: audience string
+        :param logger: logger object or none to use default one
+        """
+        self.name = name
+        self.url = url
+        self.audUrl = audUrl
+        self.expire = 0
+        self.token = None
+        self.logger = logger if logger else logging.getLogger()
+        try:
+            self.token = self.getToken()
+        except Exception as exc:
+            self.logger.exception("Failed to get token. Details: %s", str(exc))
+
+    def getToken(self):
+        """
+        Return valid token and sets its expire timestamp
+        """
+        if not self.token or not isValidToken(self.token):
+            self.token = readToken(self.name)
+        tokenDict = {}
+        try:
+            tokenDict = tokenData(self.token, url=self.url, audUrl=self.audUrl)
+            self.logger.debug(tokenDict)
+        except Exception as exc:
+            self.logger.exception(str(exc))
+            raise
+        self.expire = tokenDict.get('exp', 0)
+        return self.token
+
+    def getLifetime(self):
+        """
+        Return reamaining lifetime of existing token
+        """
+        return self.expire - int(time.time())

Utils/Tracing.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+"""
+Stuff to help with debugging from https://pymotw.com/3/sys/tracing.html
+
+Use like this:
+
+        import sys
+        from Utils.Tracing import trace_calls_and_returns, dont_trace
+
+        sys.settrace(trace_calls_and_returns)
+        # Do the things you want to trace
+        sys.settrace(dont_trace)
+
+"""
+
+
+def trace_calls(frame, event, arg, to_be_traced=None):
+    if event != 'call':
+        return
+    co = frame.f_code
+    func_name = co.co_name
+    if func_name == 'write':
+        return  # Ignore write() calls from printing
+    line_no = frame.f_lineno
+    filename = co.co_filename
+    print('* Call to {} on line {} of {}'.format(
+        func_name, line_no, filename))
+    if func_name in to_be_traced:
+        # Trace into this function
+        return trace_lines
+    return
+
+
+def trace_calls_and_returns(frame, event, arg):
+    co = frame.f_code
+    func_name = co.co_name
+    if func_name == 'write':
+        return  # Ignore write() calls from printing
+    line_no = frame.f_lineno
+    filename = co.co_filename
+    if event == 'call':
+        print('Call to {} on line {} of {}'.format(func_name, line_no, filename))
+        return trace_calls_and_returns
+    elif event == 'return':
+        print(' {} => {} ({})'.format(func_name, arg, filename))
+    return
+
+
+def trace_lines(frame, event, arg):
+    if event != 'line':
+        return
+    co = frame.f_code
+    func_name = co.co_name
+    line_no = frame.f_lineno
+    print('*  {} line {}'.format(func_name, line_no))
+
+
+def dont_trace(frame, event, arg):
+    return