wmglobalqueue 2.4.5.1__py3-none-any.whl

Utils/IteratorTools.py

@@ -0,0 +1,80 @@
+#! /usr/bin/env python
+
+from builtins import str, map
+import collections.abc
+from itertools import islice, chain, groupby
+
+def grouper(iterable, n):
+    """
+    :param iterable: List of other iterable to slice
+    :type: iterable
+    :param n: Chunk size for resulting lists
+    :type: int
+    :return: iterator of the sliced list
+
+    Source: http://stackoverflow.com/questions/3992735/python-generator-that-groups-another-iterable-into-groups-of-n
+    """
+    iterable = iter(iterable)
+    return iter(lambda: list(islice(iterable, n)), [])
+
+
+def getChunk(arr, step):
+    """
+    Return chunk of entries from given array and step, it is similar in behavior to grouper
+    function but instead of returning new list it provides a generator iterable object.
+    :param arr: input array of data
+    :param step: step to iterate
+    :return: generator, set of slices with number of entries equal to step of iteration
+    """
+    for i in range(0, len(arr), step):
+        yield arr[i:i + step]
+
+
+def flattenList(doubleList):
+    """
+    Make flat a list of lists.
+    """
+    return list(chain.from_iterable(doubleList))
+
+
+def nestedDictUpdate(d, u):
+    """
+    Code from Alex Matelli
+    http://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth
+    """
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            r = nestedDictUpdate(d.get(k, {}), v)
+            d[k] = r
+        else:
+            d[k] = u[k]
+    return d
+
+def convertFromUnicodeToBytes(data):
+    """
+    code fram
+    http://stackoverflow.com/questions/1254454/fastest-way-to-convert-a-dicts-keys-values-from-unicode-to-str
+    """
+    if isinstance(data, str):
+        return data.encode('utf-8')
+    elif isinstance(data, collections.abc.Mapping):
+        return dict(list(map(convertFromUnicodeToBytes, list(data.items()))))
+    elif isinstance(data, collections.abc.Iterable):
+        return type(data)(list(map(convertFromUnicodeToBytes, data)))
+    else:
+        return data
+
+
+def makeListElementsUnique(listObj):
+    """
+    Given a list of lists or a list of tuples, find all duplicate elements
+    and make them unique.
+    :param listObj: an unsorted list of lists or a list of tuples, e.g.:
+        [[1, 1], [1, 5], [1, 1]]; or
+        [(1, 1), (1, 5), (1, 1)]
+    :return: the same list object but with no duplicates
+
+    Source: https://stackoverflow.com/questions/2213923/removing-duplicates-from-a-list-of-lists
+    """
+    listObj.sort()
+    return list(k for k, _ in groupby(listObj))

Utils/MathUtils.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""
+Module containing mathematical and physics utils
+"""
+
+
+from builtins import int, str
+from math import ceil
+
+
+def quantize(inputVal, quanta):
+    """
+    _quantize_
+
+    Quantize the input value following the quanta provided.
+    """
+    if isinstance(inputVal, str):
+        inputVal = float(inputVal)
+    elif not isinstance(inputVal, (int, float)):
+        msg = "Input value has to be either int or float, not %s" % (type(inputVal))
+        raise ValueError(msg)
+
+    if isinstance(quanta, (str, int, float)):
+        quanta = int(float(quanta))
+    elif not isinstance(quanta, int):
+        msg = "Quanta value has to be either int or float, not %s" % (type(quanta))
+        raise ValueError(msg)
+
+    res = int(ceil(inputVal / quanta))
+
+    return res * quanta

Utils/MemoryCache.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Simple in-memory and non-thread safe cache.
+Note that this module does not support home-made object types, since there is
+an explicit data type check when adding a new item to the cache.
+
+It raises a TypeError exception if the cache data type chagens;
+or if the user tries to extend the cache with an incompatible
+data type.
+"""
+
+from copy import copy
+
+from builtins import object
+from time import time
+
+
+class MemoryCacheException(Exception):
+    def __init__(self, message):
+        super(MemoryCacheException, self).__init__(message)
+
+class MemoryCache():
+
+    __slots__ = ["lastUpdate", "expiration", "_cache"]
+
+    def __init__(self, expiration, initialData=None):
+        """
+        Initializes cache object
+
+        :param expiration: expiration time in seconds
+        :param initialData: initial value for the cache
+        """
+        self.lastUpdate = int(time())
+        self.expiration = expiration
+        self._cache = initialData
+
+    def __contains__(self, item):
+        """
+        Check whether item is in the current cache
+        :param item: a simple object (string, integer, etc)
+        :return: True if the object can be found in the cache, False otherwise
+        """
+        return item in self._cache
+
+    def __getitem__(self, keyName):
+        """
+        If the cache is a dictionary, return that item from the cache. Else, raise an exception.
+        :param keyName: the key name from the dictionary
+        """
+        if isinstance(self._cache, dict):
+            return copy(self._cache.get(keyName))
+        else:
+            raise MemoryCacheException("Cannot retrieve an item from a non-dict MemoryCache object: {}".format(self._cache))
+
+    def reset(self):
+        """
+        Resets the cache to its current data type
+        """
+        if isinstance(self._cache, (dict, set)):
+            self._cache.clear()
+        elif isinstance(self._cache, list):
+            del self._cache[:]
+        else:
+            raise MemoryCacheException("The cache needs to be reset manually, data type unknown")
+
+    def isCacheExpired(self):
+        """
+        Evaluate whether the cache has already expired, returning
+        True if it did, otherwise it returns False
+        """
+        return self.lastUpdate + self.expiration < int(time())
+
+    def getCache(self):
+        """
+        Raises an exception if the cache has expired, otherwise returns
+        its data
+        """
+        if self.isCacheExpired():
+            expiredSince = int(time()) - (self.lastUpdate + self.expiration)
+            raise MemoryCacheException("Memory cache expired for %d seconds" % expiredSince)
+        return self._cache
+
+    def setCache(self, inputData):
+        """
+        Refresh the cache with the content provided (refresh its expiration as well)
+        This method enforces the user to not change the cache data type
+        :param inputData: data to store in the cache
+        """
+        if not isinstance(self._cache, type(inputData)):
+            raise TypeError("Current cache data type: %s, while new value is: %s" %
+                            (type(self._cache), type(inputData)))
+        self.reset()
+        self.lastUpdate = int(time())
+        self._cache = inputData
+
+    def addItemToCache(self, inputItem):
+        """
+        Adds new item(s) to the cache, without resetting its expiration.
+        It, of course, only works for data caches of type: list, set or dict.
+        :param inputItem: additional item to be added to the current cached data
+        """
+        if isinstance(self._cache, set) and isinstance(inputItem, (list, set)):
+            # extend another list or set into a set
+            self._cache.update(inputItem)
+        elif isinstance(self._cache, set) and isinstance(inputItem, (int, float, str)):
+            # add a simple object (integer, string, etc) to a set
+            self._cache.add(inputItem)
+        elif isinstance(self._cache, list) and isinstance(inputItem, (list, set)):
+            # extend another list or set into a list
+            self._cache.extend(inputItem)
+        elif isinstance(self._cache, list) and isinstance(inputItem, (int, float, str)):
+            # add a simple object (integer, string, etc) to a list
+            self._cache.append(inputItem)
+        elif isinstance(self._cache, dict) and isinstance(inputItem, dict):
+            self._cache.update(inputItem)
+        else:
+            msg = "Input item type: %s cannot be added to a cache type: %s" % (type(self._cache), type(inputItem))
+            raise TypeError("Cache and input item data type mismatch. %s" % msg)

Utils/Patterns.py

@@ -0,0 +1,24 @@
+"""
+Patterns module provides set of CS patterns
+"""
+import re
+
+class Singleton(type):
+    """Implementation of Singleton class"""
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = \
+                    super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+def getDomainName(urlStr):
+    """
+    Given a URL string, return the domain name.
+    :param urlStr: URL string
+    :return: a string with the domain name (e.g. "cmsweb-prod")
+    """
+    domainPattern = re.compile(r'https?://([^/]+)\.cern\.ch')
+    match = domainPattern.search(urlStr)
+    return match.group(1) if match else ""

Utils/Pipeline.py

@@ -0,0 +1,137 @@
+"""
+File       : Pipeline.py
+Description: Provides 2 basic classes:
+             - Functor:  A class to create function calls from a function object
+                         and arbitrary number of arguments
+             - Pipeline: A class to provide building blocks for creating functional
+                         pipelines for cumulative execution on an arbitrary object
+"""
+
+from builtins import object
+from functools import reduce
+
+
+class Functor(object):
+    """
+    A simple functor class used to construct a function call which later to be
+    applied on an (any type) object.
+    NOTE:
+        It expects a function in the constructor and an (any type) object
+        passed to the run or __call__ methods, which methods once called they
+        construct and return the following function:
+        func(obj, *args, **kwargs)
+    NOTE:
+        All the additional arguments which the function may take must be set in
+        the __init__ method. If any of them are passed during run time an error
+        will be raised.
+
+    :func:
+        The function to which the rest of the constructor arguments are about
+        to be attached and then the newly created function will be returned.
+        - The function needs to take at least one parameter since the object
+        passed to the run/__call__ methods will always be put as a first
+        argument to the function.
+
+    :Example:
+
+    def adder(a, b, *args, **kwargs):
+        if args:
+            print("adder args: %s" % args)
+        if kwargs:
+            print("adder kwargs: %s" % kwargs)
+        res = a + b
+        return res
+
+    >>> x=Functor(adder, 8, 'foo', bar=True)
+    >>> x(2)
+    adder args: foo
+    adder kwargs: {'bar': True}
+    adder res: 10
+    10
+
+    >>> x
+    <Pipeline.Functor instance at 0x7f319bbaeea8>
+
+
+    """
+    def __init__(self, func, *args, **kwargs):
+        """
+        The init method for class Functor
+        """
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def __call__(self, obj):
+        """
+        The call method for class Functor
+        """
+        return self.run(obj)
+
+    def run(self, obj):
+        return self.func(obj, *self.args, **self.kwargs)
+
+
+class Pipeline(object):
+    """
+    A simple Functional Pipeline Class: applies a set of functions to an object,
+    where the output of every previous function is an input to the next one.
+    """
+    # NOTE:
+    #    Similar and inspiring approaches but yet some different implementations
+    #    are discussed in the following two links [1] & [2]. With a quite good
+    #    explanation in [1], which helped a lot. All in all at the bottom always
+    #    sits the reduce function.
+    #    [1]
+    #    https://softwarejourneyman.com/python-function-pipelines.html
+    #    [2]
+    #    https://gitlab.com/mc706/functional-pipeline
+
+    def __init__(self, funcLine=None, name=None):
+        """
+        :funcLine: A list of functions or Functors of function + arguments (see
+                   the Class definition above) that are to be applied sequentially
+                   to the object.
+                   - If any of the elements of 'funcLine' is a function, a direct
+                   function call with the object as an argument is performed.
+                   - If any of the elements of 'funcLine' is a Functor, then the
+                   first argument of the Functor constructor is the function to
+                   be evaluated and the object is passed as a first argument to
+                   the function with all the rest of the arguments passed right
+                   after it eg. the following Functor in the funcLine:
+
+                   Functor(func, 'foo', bar=True)
+
+                   will result in the following function call later when the
+                   pipeline is executed:
+
+                   func(obj, 'foo', bar=True)
+
+        :Example:
+            (using the adder function from above and an object of type int)
+
+        >>> pipe = Pipeline([Functor(adder, 5),
+                             Functor(adder, 6),
+                             Functor(adder, 7, "extraArg"),
+                             Functor(adder, 8, update=True)])
+
+        >>> pipe.run(1)
+        adder res: 6
+        adder res: 12
+        adder args: extraArg
+        adder res: 19
+        adder kwargs: {'update': True}
+        adder res: 27
+        """
+        self.funcLine = funcLine or []
+        self.name = name
+
+    def getPipelineName(self):
+        """
+        __getPipelineName__
+        """
+        name = self.name or "Unnamed Pipeline"
+        return name
+
+    def run(self, obj):
+        return reduce(lambda obj, functor: functor(obj), self.funcLine, obj)

Utils/PortForward.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+"""
+_PortForward_
+
+A decorator for swapping ports in an url
+"""
+
+from builtins import str, bytes
+
+def portForward(port):
+    """
+    Decorator wrapper function for port forwarding of the REST calls of any
+    function to a given port.
+
+    Currently there are three constraints for applying this decorator.
+    1. The function to be decorated must be defined within a class and not being a static method.
+       The reason for that is because we need to be sure the function's signature will
+       always include the class instance as its first argument.
+    2. The url argument must be present as the second one in the positional argument list
+       of the decorated function (right after the class instance argument).
+    3. The url must follow the syntax specifications in RFC 1808:
+       https://tools.ietf.org/html/rfc1808.html
+
+    If all of the above constraints are fulfilled and the url is part of the
+    urlMangleList, then the url is parsed and the port is substituted with the
+    one provided as an argument to the decorator's wrapper function.
+
+    param port: The port to which the REST call should be forwarded.
+    """
+    def portForwardDecorator(callFunc):
+        """
+        The actual decorator
+        """
+
+        def portMangle(callObj, url, *args, **kwargs):
+            """
+            Function used to check if the url coming with the current argument list
+            is to be forwarded and if so change the port to the one provided as an
+            argument to the decorator wrapper.
+
+            :param classObj: This is the class object (slef from within the class)
+                             which is always to be present in the signature of a
+                             public method. We will never use this argument, but
+                             we need it there for not breaking the positional
+                             argument order
+            :param url:      This is the actual url to be (eventually) forwarded
+            :param *args:    The positional argument list coming from the original function
+            :param *kwargs:  The keywords argument list coming from the original function
+            """
+            forwarded = False
+            try:
+                if isinstance(url, str):
+                    urlToMangle = 'https://cmsweb'
+                    if url.startswith(urlToMangle):
+                        newUrl = url.replace('.cern.ch/', '.cern.ch:%d/' % port, 1)
+                        forwarded = True
+                elif isinstance(url, bytes):
+                    urlToMangle = b'https://cmsweb'
+                    if url.startswith(urlToMangle):
+                        newUrl = url.replace(b'.cern.ch/', b'.cern.ch:%d/' % port, 1)
+                        forwarded = True
+
+            except Exception:
+                pass
+            if forwarded:
+                return callFunc(callObj, newUrl, *args, **kwargs)
+            else:
+                return callFunc(callObj, url, *args, **kwargs)
+        return portMangle
+    return portForwardDecorator
+
+
+class PortForward():
+    """
+    A class with a call method implementing a simple way to use the functionality
+    provided by the protForward decorator as a pure functional call:
+    EXAMPLE:
+        from Utils.PortForward import PortForward
+
+        portForwarder = PortForward(8443)
+        url = 'https://cmsweb-testbed.cern.ch/couchdb'
+        url = portForwarder(url)
+    """
+    def __init__(self, port):
+        """
+        The init method for the PortForward call class. This one is supposed
+        to simply provide an initial class instance with a logger.
+        """
+        self.port = port
+
+    def __call__(self, url):
+        """
+        The call method for the PortForward class
+        """
+        def dummyCall(self, url):
+            return url
+        return portForward(self.port)(dummyCall)(self, url)

Utils/ProcFS.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+File       : ProcFS.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: This module provides function which rely on Linux proc fs
+"""
+# system modules
+import os
+
+# 3rd party modules
+try:
+    import psutil
+    usePsutil = True
+except ImportError:
+    usePsutil = False
+
+
+def processStatus(pid, method=None):
+    """
+    Return status of the given process PID along with its threads using psutil.
+    :param pid: Process ID to inspect
+    :param method: define which method to use procfs or psutil
+    :return: list of dictionaries of `{process, pid, status}` data-structure
+    """
+    if method == 'procfs' or usePsutil == False:
+        return processStatusViaProcFS(pid)
+    return processStatusViaPsutil(pid)
+
+
+def processStatusViaPsutil(pid):
+    """
+    Return status of the given process PID along with its threads using psutil.
+    :param pid: Process ID to inspect
+    :return: list of dictionaries with {process, pid, status, type}
+    """
+    statusList = []
+
+    try:
+        proc = psutil.Process(int(pid))
+        processName = proc.name()
+        processStatus = proc.status()
+        statusList.append({
+            "process": processName,
+            "pid": str(pid),
+            "status": processStatus,
+            "type": "process"
+        })
+
+        # Iterate over threads
+        threads = proc.threads()
+        for thread in threads:
+            if str(pid) == str(thread.id):
+                continue
+            threadId = thread.id
+            # psutil doesn't provide thread name or detailed state, so we approximate
+            statusList.append({
+                "process": f"{processName}-thread",
+                "pid": str(threadId),
+                "status": "running",  # threads are assumed running if present
+                "type": "thread"
+            })
+
+    except psutil.NoSuchProcess:
+        return [{"error": f"Process {pid} not found"}]
+    except Exception as e:
+        return [{"error": str(e)}]
+
+    return statusList
+
+
+def processStatusViaProcFS(pid):
+    """
+    Return status of given process PID along with its threads via /proc FS look-up (available on all Linux OSes)
+    :return: list of dictionaries of `{process, pid, status}` data-structure
+    """
+    statusList = []
+    procPath = f"/proc/{pid}"
+
+    if not os.path.exists(procPath):
+        return [{"error": f"Process {pid} not found"}]
+
+    try:
+        with open(f"{procPath}/status", encoding='utf-8') as f:
+            processName = ""
+            processState = ""
+            for line in f:
+                if line.startswith("Name:"):
+                    processName = line.split(":")[1].strip()
+                elif line.startswith("State:"):
+                    processState = line.split(":")[1].strip()
+            statusList.append({"process": processName, "pid": str(pid), "status": processState, "type": "process"})
+
+        taskPath = f"{procPath}/task"
+        if os.path.exists(taskPath):
+            for threadId in os.listdir(taskPath):
+                if str(threadId) == str(pid):
+                    continue
+                with open(f"{taskPath}/{threadId}/status", encoding='utf-8') as f:
+                    threadName = ""
+                    threadState = ""
+                    for line in f:
+                        if line.startswith("Name:"):
+                            threadName = line.split(":")[1].strip()
+                        elif line.startswith("State:"):
+                            threadState = line.split(":")[1].strip()
+                    statusList.append({"process": threadName, "pid": threadId, "status": threadState, "type": "thread"})
+
+    except Exception as e:
+        return [{"error": str(e)}]
+
+    return statusList