wmglobalqueue 2.3.10__py3-none-any.whl

Utils/IteratorTools.py

@@ -0,0 +1,80 @@
+#! /usr/bin/env python
+
+from builtins import str, map
+import collections.abc
+from itertools import islice, chain, groupby
+
+def grouper(iterable, n):
+    """
+    :param iterable: List of other iterable to slice
+    :type: iterable
+    :param n: Chunk size for resulting lists
+    :type: int
+    :return: iterator of the sliced list
+
+    Source: http://stackoverflow.com/questions/3992735/python-generator-that-groups-another-iterable-into-groups-of-n
+    """
+    iterable = iter(iterable)
+    return iter(lambda: list(islice(iterable, n)), [])
+
+
+def getChunk(arr, step):
+    """
+    Return chunk of entries from given array and step, it is similar in behavior to grouper
+    function but instead of returning new list it provides a generator iterable object.
+    :param arr: input array of data
+    :param step: step to iterate
+    :return: generator, set of slices with number of entries equal to step of iteration
+    """
+    for i in range(0, len(arr), step):
+        yield arr[i:i + step]
+
+
+def flattenList(doubleList):
+    """
+    Make flat a list of lists.
+    """
+    return list(chain.from_iterable(doubleList))
+
+
+def nestedDictUpdate(d, u):
+    """
+    Code from Alex Matelli
+    http://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth
+    """
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            r = nestedDictUpdate(d.get(k, {}), v)
+            d[k] = r
+        else:
+            d[k] = u[k]
+    return d
+
+def convertFromUnicodeToBytes(data):
+    """
+    code fram
+    http://stackoverflow.com/questions/1254454/fastest-way-to-convert-a-dicts-keys-values-from-unicode-to-str
+    """
+    if isinstance(data, str):
+        return data.encode('utf-8')
+    elif isinstance(data, collections.abc.Mapping):
+        return dict(list(map(convertFromUnicodeToBytes, list(data.items()))))
+    elif isinstance(data, collections.abc.Iterable):
+        return type(data)(list(map(convertFromUnicodeToBytes, data)))
+    else:
+        return data
+
+
+def makeListElementsUnique(listObj):
+    """
+    Given a list of lists or a list of tuples, find all duplicate elements
+    and make them unique.
+    :param listObj: an unsorted list of lists or a list of tuples, e.g.:
+        [[1, 1], [1, 5], [1, 1]]; or
+        [(1, 1), (1, 5), (1, 1)]
+    :return: the same list object but with no duplicates
+
+    Source: https://stackoverflow.com/questions/2213923/removing-duplicates-from-a-list-of-lists
+    """
+    listObj.sort()
+    return list(k for k, _ in groupby(listObj))

Utils/MathUtils.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""
+Module containing mathematical and physics utils
+"""
+
+
+from builtins import int, str
+from math import ceil
+
+
+def quantize(inputVal, quanta):
+    """
+    _quantize_
+
+    Quantize the input value following the quanta provided.
+    """
+    if isinstance(inputVal, str):
+        inputVal = float(inputVal)
+    elif not isinstance(inputVal, (int, float)):
+        msg = "Input value has to be either int or float, not %s" % (type(inputVal))
+        raise ValueError(msg)
+
+    if isinstance(quanta, (str, int, float)):
+        quanta = int(float(quanta))
+    elif not isinstance(quanta, int):
+        msg = "Quanta value has to be either int or float, not %s" % (type(quanta))
+        raise ValueError(msg)
+
+    res = int(ceil(inputVal / quanta))
+
+    return res * quanta

Utils/MemoryCache.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Simple in-memory and non-thread safe cache.
+Note that this module does not support home-made object types, since there is
+an explicit data type check when adding a new item to the cache.
+
+It raises a TypeError exception if the cache data type chagens;
+or if the user tries to extend the cache with an incompatible
+data type.
+"""
+
+from copy import copy
+
+from builtins import object
+from time import time
+
+
+class MemoryCacheException(Exception):
+    def __init__(self, message):
+        super(MemoryCacheException, self).__init__(message)
+
+class MemoryCache():
+
+    __slots__ = ["lastUpdate", "expiration", "_cache"]
+
+    def __init__(self, expiration, initialData=None):
+        """
+        Initializes cache object
+
+        :param expiration: expiration time in seconds
+        :param initialData: initial value for the cache
+        """
+        self.lastUpdate = int(time())
+        self.expiration = expiration
+        self._cache = initialData
+
+    def __contains__(self, item):
+        """
+        Check whether item is in the current cache
+        :param item: a simple object (string, integer, etc)
+        :return: True if the object can be found in the cache, False otherwise
+        """
+        return item in self._cache
+
+    def __getitem__(self, keyName):
+        """
+        If the cache is a dictionary, return that item from the cache. Else, raise an exception.
+        :param keyName: the key name from the dictionary
+        """
+        if isinstance(self._cache, dict):
+            return copy(self._cache.get(keyName))
+        else:
+            raise MemoryCacheException("Cannot retrieve an item from a non-dict MemoryCache object: {}".format(self._cache))
+
+    def reset(self):
+        """
+        Resets the cache to its current data type
+        """
+        if isinstance(self._cache, (dict, set)):
+            self._cache.clear()
+        elif isinstance(self._cache, list):
+            del self._cache[:]
+        else:
+            raise MemoryCacheException("The cache needs to be reset manually, data type unknown")
+
+    def isCacheExpired(self):
+        """
+        Evaluate whether the cache has already expired, returning
+        True if it did, otherwise it returns False
+        """
+        return self.lastUpdate + self.expiration < int(time())
+
+    def getCache(self):
+        """
+        Raises an exception if the cache has expired, otherwise returns
+        its data
+        """
+        if self.isCacheExpired():
+            expiredSince = int(time()) - (self.lastUpdate + self.expiration)
+            raise MemoryCacheException("Memory cache expired for %d seconds" % expiredSince)
+        return self._cache
+
+    def setCache(self, inputData):
+        """
+        Refresh the cache with the content provided (refresh its expiration as well)
+        This method enforces the user to not change the cache data type
+        :param inputData: data to store in the cache
+        """
+        if not isinstance(self._cache, type(inputData)):
+            raise TypeError("Current cache data type: %s, while new value is: %s" %
+                            (type(self._cache), type(inputData)))
+        self.reset()
+        self.lastUpdate = int(time())
+        self._cache = inputData
+
+    def addItemToCache(self, inputItem):
+        """
+        Adds new item(s) to the cache, without resetting its expiration.
+        It, of course, only works for data caches of type: list, set or dict.
+        :param inputItem: additional item to be added to the current cached data
+        """
+        if isinstance(self._cache, set) and isinstance(inputItem, (list, set)):
+            # extend another list or set into a set
+            self._cache.update(inputItem)
+        elif isinstance(self._cache, set) and isinstance(inputItem, (int, float, str)):
+            # add a simple object (integer, string, etc) to a set
+            self._cache.add(inputItem)
+        elif isinstance(self._cache, list) and isinstance(inputItem, (list, set)):
+            # extend another list or set into a list
+            self._cache.extend(inputItem)
+        elif isinstance(self._cache, list) and isinstance(inputItem, (int, float, str)):
+            # add a simple object (integer, string, etc) to a list
+            self._cache.append(inputItem)
+        elif isinstance(self._cache, dict) and isinstance(inputItem, dict):
+            self._cache.update(inputItem)
+        else:
+            msg = "Input item type: %s cannot be added to a cache type: %s" % (type(self._cache), type(inputItem))
+            raise TypeError("Cache and input item data type mismatch. %s" % msg)

Utils/Patterns.py

@@ -0,0 +1,24 @@
+"""
+Patterns module provides set of CS patterns
+"""
+import re
+
+class Singleton(type):
+    """Implementation of Singleton class"""
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = \
+                    super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+def getDomainName(urlStr):
+    """
+    Given a URL string, return the domain name.
+    :param urlStr: URL string
+    :return: a string with the domain name (e.g. "cmsweb-prod")
+    """
+    domainPattern = re.compile(r'https?://([^/]+)\.cern\.ch')
+    match = domainPattern.search(urlStr)
+    return match.group(1) if match else ""

Utils/Pipeline.py

@@ -0,0 +1,137 @@
+"""
+File       : Pipeline.py
+Description: Provides 2 basic classes:
+             - Functor:  A class to create function calls from a function object
+                         and arbitrary number of arguments
+             - Pipeline: A class to provide building blocks for creating functional
+                         pipelines for cumulative execution on an arbitrary object
+"""
+
+from builtins import object
+from functools import reduce
+
+
+class Functor(object):
+    """
+    A simple functor class used to construct a function call which later to be
+    applied on an (any type) object.
+    NOTE:
+        It expects a function in the constructor and an (any type) object
+        passed to the run or __call__ methods, which methods once called they
+        construct and return the following function:
+        func(obj, *args, **kwargs)
+    NOTE:
+        All the additional arguments which the function may take must be set in
+        the __init__ method. If any of them are passed during run time an error
+        will be raised.
+
+    :func:
+        The function to which the rest of the constructor arguments are about
+        to be attached and then the newly created function will be returned.
+        - The function needs to take at least one parameter since the object
+        passed to the run/__call__ methods will always be put as a first
+        argument to the function.
+
+    :Example:
+
+    def adder(a, b, *args, **kwargs):
+        if args:
+            print("adder args: %s" % args)
+        if kwargs:
+            print("adder kwargs: %s" % kwargs)
+        res = a + b
+        return res
+
+    >>> x=Functor(adder, 8, 'foo', bar=True)
+    >>> x(2)
+    adder args: foo
+    adder kwargs: {'bar': True}
+    adder res: 10
+    10
+
+    >>> x
+    <Pipeline.Functor instance at 0x7f319bbaeea8>
+
+
+    """
+    def __init__(self, func, *args, **kwargs):
+        """
+        The init method for class Functor
+        """
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def __call__(self, obj):
+        """
+        The call method for class Functor
+        """
+        return self.run(obj)
+
+    def run(self, obj):
+        return self.func(obj, *self.args, **self.kwargs)
+
+
+class Pipeline(object):
+    """
+    A simple Functional Pipeline Class: applies a set of functions to an object,
+    where the output of every previous function is an input to the next one.
+    """
+    # NOTE:
+    #    Similar and inspiring approaches but yet some different implementations
+    #    are discussed in the following two links [1] & [2]. With a quite good
+    #    explanation in [1], which helped a lot. All in all at the bottom always
+    #    sits the reduce function.
+    #    [1]
+    #    https://softwarejourneyman.com/python-function-pipelines.html
+    #    [2]
+    #    https://gitlab.com/mc706/functional-pipeline
+
+    def __init__(self, funcLine=None, name=None):
+        """
+        :funcLine: A list of functions or Functors of function + arguments (see
+                   the Class definition above) that are to be applied sequentially
+                   to the object.
+                   - If any of the elements of 'funcLine' is a function, a direct
+                   function call with the object as an argument is performed.
+                   - If any of the elements of 'funcLine' is a Functor, then the
+                   first argument of the Functor constructor is the function to
+                   be evaluated and the object is passed as a first argument to
+                   the function with all the rest of the arguments passed right
+                   after it eg. the following Functor in the funcLine:
+
+                   Functor(func, 'foo', bar=True)
+
+                   will result in the following function call later when the
+                   pipeline is executed:
+
+                   func(obj, 'foo', bar=True)
+
+        :Example:
+            (using the adder function from above and an object of type int)
+
+        >>> pipe = Pipeline([Functor(adder, 5),
+                             Functor(adder, 6),
+                             Functor(adder, 7, "extraArg"),
+                             Functor(adder, 8, update=True)])
+
+        >>> pipe.run(1)
+        adder res: 6
+        adder res: 12
+        adder args: extraArg
+        adder res: 19
+        adder kwargs: {'update': True}
+        adder res: 27
+        """
+        self.funcLine = funcLine or []
+        self.name = name
+
+    def getPipelineName(self):
+        """
+        __getPipelineName__
+        """
+        name = self.name or "Unnamed Pipeline"
+        return name
+
+    def run(self, obj):
+        return reduce(lambda obj, functor: functor(obj), self.funcLine, obj)

Utils/PortForward.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+"""
+_PortForward_
+
+A decorator for swapping ports in an url
+"""
+
+from builtins import str, bytes
+
+def portForward(port):
+    """
+    Decorator wrapper function for port forwarding of the REST calls of any
+    function to a given port.
+
+    Currently there are three constraints for applying this decorator.
+    1. The function to be decorated must be defined within a class and not being a static method.
+       The reason for that is because we need to be sure the function's signature will
+       always include the class instance as its first argument.
+    2. The url argument must be present as the second one in the positional argument list
+       of the decorated function (right after the class instance argument).
+    3. The url must follow the syntax specifications in RFC 1808:
+       https://tools.ietf.org/html/rfc1808.html
+
+    If all of the above constraints are fulfilled and the url is part of the
+    urlMangleList, then the url is parsed and the port is substituted with the
+    one provided as an argument to the decorator's wrapper function.
+
+    param port: The port to which the REST call should be forwarded.
+    """
+    def portForwardDecorator(callFunc):
+        """
+        The actual decorator
+        """
+
+        def portMangle(callObj, url, *args, **kwargs):
+            """
+            Function used to check if the url coming with the current argument list
+            is to be forwarded and if so change the port to the one provided as an
+            argument to the decorator wrapper.
+
+            :param classObj: This is the class object (slef from within the class)
+                             which is always to be present in the signature of a
+                             public method. We will never use this argument, but
+                             we need it there for not breaking the positional
+                             argument order
+            :param url:      This is the actual url to be (eventually) forwarded
+            :param *args:    The positional argument list coming from the original function
+            :param *kwargs:  The keywords argument list coming from the original function
+            """
+            forwarded = False
+            try:
+                if isinstance(url, str):
+                    urlToMangle = 'https://cmsweb'
+                    if url.startswith(urlToMangle):
+                        newUrl = url.replace('.cern.ch/', '.cern.ch:%d/' % port, 1)
+                        forwarded = True
+                elif isinstance(url, bytes):
+                    urlToMangle = b'https://cmsweb'
+                    if url.startswith(urlToMangle):
+                        newUrl = url.replace(b'.cern.ch/', b'.cern.ch:%d/' % port, 1)
+                        forwarded = True
+
+            except Exception:
+                pass
+            if forwarded:
+                return callFunc(callObj, newUrl, *args, **kwargs)
+            else:
+                return callFunc(callObj, url, *args, **kwargs)
+        return portMangle
+    return portForwardDecorator
+
+
+class PortForward():
+    """
+    A class with a call method implementing a simple way to use the functionality
+    provided by the protForward decorator as a pure functional call:
+    EXAMPLE:
+        from Utils.PortForward import PortForward
+
+        portForwarder = PortForward(8443)
+        url = 'https://cmsweb-testbed.cern.ch/couchdb'
+        url = portForwarder(url)
+    """
+    def __init__(self, port):
+        """
+        The init method for the PortForward call class. This one is supposed
+        to simply provide an initial class instance with a logger.
+        """
+        self.port = port
+
+    def __call__(self, url):
+        """
+        The call method for the PortForward class
+        """
+        def dummyCall(self, url):
+            return url
+        return portForward(self.port)(dummyCall)(self, url)

Utils/ProcessStats.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+File       : ProcessStats.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: This module provide ability to get detailed information about
+certain UNIX process, referred by PID. It is based on psutil and threading
+python modules. It can be used in any web/data-server to get live status
+about the running server. Here is an example
+
+.. doctest:
+
+    from Utils.procstats import status threadStack
+
+    class SomeServer(object):
+        def __init__(self):
+        self.time0 = time.time()
+        def status(self):
+            sdict = {'server': processStatus()}
+            sdict['server'].update({'uptime': time.time()-self.time0})
+            sdict.update(threadStack())
+            return sdict
+
+The exposed status endpoint will return a json dictionary containing server
+upteima and cpu/mem/threads information.
+"""
+
+# system modules
+import os
+import sys
+import json
+import time
+import argparse
+import threading
+import traceback
+
+try:
+    import psutil
+except ImportError:
+    pass
+
+def _baseProcessStatusFormat(pid=None):
+    if not pid:
+        pid = os.getpid()
+    ttime = time.localtime()
+    tstamp = time.strftime('%d/%b/%Y:%H:%M:%S', ttime)
+    return {'pid': pid, 'timestamp': tstamp, 'time': time.time()}
+
+def processStatus(pid=None):
+    "Return status of the process in a dictionary format"
+
+    pdict = _baseProcessStatusFormat(pid)
+    if 'psutil' not in sys.modules:
+        return pdict
+    proc = psutil.Process(pid)
+    pdict.update(proc.as_dict())
+    return pdict
+
+def processStatusDict(pid=None):
+    "Return status of the process in a dictionary format"
+    pdict = _baseProcessStatusFormat(pid)
+    if 'psutil' not in sys.modules:
+        return pdict
+    proc = psutil.Process(pid)
+    pdict.update({"cpu_times": dict(proc.cpu_times()._asdict())})
+    pdict.update({"cpu_percent": proc.cpu_percent(interval=1.0)})
+    pdict.update({"cpu_num": proc.cpu_num()})
+    pdict.update({"memory_full_info": dict(proc.memory_full_info()._asdict())})
+    pdict.update({"memory_percent": proc.memory_percent()})
+    return pdict
+
+def threadStack():
+    """
+    Return context of all threads in dictionary format where individual
+    thread information stored in its own dicts and all threads are groupped
+    into threads list. Code based on example from StackOverflow:
+    http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
+    """
+    tdict = {}
+    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    threads = []
+    for tid, stack in list(sys._current_frames().items()):
+        tdict = {"thead": id2name.get(tid, ""), "thead_id": tid}
+        stacklist = []
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            sdict = dict(filename=filename, line_number=lineno, name=name, line=line)
+            stacklist.append(sdict)
+        tdict.update({"stack": stacklist})
+        threads.append(tdict)
+    return dict(threads=threads)
+
+def main():
+    "Main function to use this module as a stand-along script."
+    parser = argparse.ArgumentParser(prog='PROG')
+    parser.add_argument("--pid", action="store", dest="pid", help="process id")
+    opts = parser.parse_args()
+
+    pdict = processStatus(int(opts.pid))
+    pdict.update(threadStack())
+    print(json.dumps(pdict))
+
+if __name__ == '__main__':
+    main()

Utils/PythonVersion.py

@@ -0,0 +1,17 @@
+"""
+Easily get the version of the python interpreter at runtime
+"""
+
+ # Jenkins CI
+
+import sys
+import pickle
+
+PY3 = sys.version_info[0] == 3
+PY2 = sys.version_info[0] == 2
+
+# We need to keep compatibility between multiple python versions
+# Further details at: https://github.com/dmwm/WMCore/pull/10726
+# For PY2: set highest protocol to 2
+# For PY3: set highest protocol to 4 (compatible with python3.6 and python3.8)
+HIGHEST_PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL if PY2 else 4

Utils/Signals.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : Signals.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: This module provide set of hooks to running program
+The dumpthreads callback function will be called every time when
+running program will receive SIGUSR1 signal. E.g., in unix shell
+just do
+shell# kill -s SIGUSR1 <pid>
+"""
+
+# system modules
+import sys
+import signal
+import threading
+import traceback
+
+def dumpthreads(isignal, iframe):
+    """
+    Dump context of all threads upon given signal
+    http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
+    """
+    print("DAS stack, signal=%s, frame=%s" % (isignal, iframe))
+    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    code = []
+    for tid, stack in list(sys._current_frames().items()):
+        code.append("\n# Thread: %s(%d)" % (id2name.get(tid,""), tid))
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
+            if  line:
+                code.append("  %s" % (line.strip()))
+    print("\n".join(code))
+
+signal.signal(signal.SIGUSR1, dumpthreads)

Utils/TemporaryEnvironment.py

@@ -0,0 +1,27 @@
+#! /usr/bin/env python
+
+import contextlib
+import os
+
+
+@contextlib.contextmanager
+def tmpEnv(**environ):
+    """
+    Temporarily set the process environment variables.
+
+    >>> with tmpEnv(PLUGINS_DIR=u'test/plugins'):
+    ...   "PLUGINS_DIR" in os.environ
+    True
+
+    >>> "PLUGINS_DIR" in os.environ
+    False
+
+    :param environ: Environment variables to set
+    """
+    oldEnviron = dict(os.environ)
+    os.environ.update(environ)
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(oldEnviron)