wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/Algorithms/MathAlgos.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env
+
+"""
+_MathAlgos_
+
+Simple mathematical tools and tricks that might prove to
+be useful.
+"""
+from __future__ import print_function, division
+from builtins import str, range
+
+import math
+import decimal
+import logging
+
+from WMCore.WMException import WMException
+
+
+class MathAlgoException(WMException):
+    """
+    Some simple math algo exceptions
+
+    """
+    pass
+
+
+def getAverageStdDev(numList):
+    """
+    _getAverageStdDev_
+
+    Given a list, calculate both the average and the
+    standard deviation.
+    """
+
+    total   = 0.0
+    average = 0.0
+    stdBase = 0.0
+
+    # Assemble the average
+    skipped = 0
+    for value in numList:
+        try:
+            if math.isnan(value) or math.isinf(value):
+                skipped += 1
+                continue
+            else:
+                total += value
+        except TypeError:
+            msg =  "Attempted to take average of non-numerical values.\n"
+            msg += "Expected int or float, got %s: %s" % (value.__class__, value)
+            logging.error(msg)
+            logging.debug("FullList: %s", numList)
+            raise MathAlgoException(msg)
+
+    length = len(numList) - skipped
+    if length < 1:
+        return average, total
+
+    average = total / length
+
+    for value in numList:
+        tmpValue = value - average
+        stdBase += (tmpValue * tmpValue)
+
+    stdDev = math.sqrt(stdBase / length)
+
+    if math.isnan(average) or math.isinf(average):
+        average = 0.0
+    if math.isnan(stdDev) or math.isinf(average) or not decimal.Decimal(str(stdDev)).is_finite():
+        stdDev = 0.0
+    if not isinstance(stdDev, (int, float)):
+        stdDev = 0.0
+
+    return average, stdDev
+
+
+def createHistogram(numList, nBins, limit):
+    """
+    _createHistogram_
+
+    Create a histogram proxy (a list of bins) for a
+    given list of numbers
+    """
+
+    average, stdDev = getAverageStdDev(numList = numList)
+
+    underflow  = []
+    overflow   = []
+    histEvents = []
+    histogram  = []
+    for value in numList:
+        if math.fabs(average - value) <= limit * stdDev:
+            # Then we counted this event
+            histEvents.append(value)
+        elif average < value:
+            overflow.append(value)
+        elif average > value:
+            underflow.append(value)
+
+    if len(underflow) > 0:
+        binAvg, binStdDev = getAverageStdDev(numList=underflow)
+        histogram.append({'type': 'underflow',
+                          'average': binAvg,
+                          'stdDev': binStdDev,
+                          'nEvents': len(underflow)})
+    if len(overflow) > 0:
+        binAvg, binStdDev = getAverageStdDev(numList=overflow)
+        histogram.append({'type': 'overflow',
+                          'average': binAvg,
+                          'stdDev': binStdDev,
+                          'nEvents': len(overflow)})
+    if len(histEvents) < 1:
+        # Nothing to do?
+        return histogram
+
+    histEvents.sort()
+    upperBound = max(histEvents)
+    lowerBound = min(histEvents)
+    if lowerBound == upperBound:
+        # This is a problem
+        logging.debug("Only one value in the histogram!")
+        nBins = 1
+        upperBound = upperBound + 1
+        lowerBound = lowerBound - 1
+    binSize = (upperBound - lowerBound)/nBins
+    binSize = floorTruncate(binSize)
+
+    for x in range(nBins):
+        lowerEdge = floorTruncate(lowerBound + (x * binSize))
+        histogram.append({'type': 'standard',
+                          'lowerEdge': lowerEdge,
+                          'upperEdge': lowerEdge + binSize,
+                          'average': 0.0,
+                          'stdDev': 0.0,
+                          'nEvents': 0})
+
+    for bin_ in histogram:
+        if bin_['type'] != 'standard':
+            continue
+        binList = []
+        for value in histEvents:
+            if value >= bin_['lowerEdge'] and value <= bin_['upperEdge']:
+                # Then we're in the bin
+                binList.append(value)
+            elif value > bin_['upperEdge']:
+                # Because this is a sorted list we are now out of the bin range
+                # Calculate our values and break
+                break
+            else:
+                continue
+
+        # If we get here, it's because we're out of values in the bin
+        # Time to do some math
+        if len(binList) < 1:
+            # Nothing to do here, leave defaults
+            continue
+
+        binAvg, binStdDev = getAverageStdDev(numList=binList)
+        bin_['average'] = binAvg
+        bin_['stdDev']  = binStdDev
+        bin_['nEvents'] = len(binList)
+
+    return histogram
+
+
+def floorTruncate(value, precision=3):
+    """
+    _floorTruncate_
+
+    Truncate a value to a set number of decimal points
+
+    Always truncates to a LOWER value, this is so that using it for
+    histogram binning creates values beneath the histogram lower edge.
+    """
+    prec = math.pow(10, precision)
+
+    return math.floor(value * prec)/prec
+
+
+def sortDictionaryListByKey(dictList, key, reverse=False):
+    """
+    _sortDictionaryListByKey_
+
+    Given a list of dictionaries and a key with a numerical
+    value, sort that dictionary in order of that key's value.
+
+    NOTE: If the key does not exist, this will not raise an exception
+    This is because this is used for sorting of performance histograms
+    And not all histograms have the same value
+    """
+
+    return sorted(dictList, key=lambda k: float(k.get(key, 0.0)), reverse=reverse)
+
+
+def getLargestValues(dictList, key, n=1):
+    """
+    _getLargestValues_
+
+    Take a list of dictionaries, sort them by the value of a
+    particular key, and return the n largest entries.
+
+    Key must be a numerical key.
+    """
+
+    sortedList = sortDictionaryListByKey(dictList=dictList, key=key, reverse=True)
+
+    return sortedList[:n]
+
+
+def validateNumericInput(value):
+    """
+    _validateNumericInput_
+
+    Check that the value is actually an usable number
+    """
+    value = float(value)
+    try:
+        if math.isnan(value) or math.isinf(value):
+            return False
+    except TypeError:
+        return False
+
+    return True
+
+
+def calculateRunningAverageAndQValue(newPoint, n, oldM, oldQ):
+    """
+    _calculateRunningAverageAndQValue_
+
+    Use the algorithm described in:
+    Donald E. Knuth (1998). The Art of Computer Programming, volume 2: Seminumerical Algorithms, 3rd ed.., p. 232. Boston: Addison-Wesley.
+
+    To calculate an average and standard deviation while getting data, the standard deviation
+    can be obtained from the so-called Q value with the following equation:
+
+    sigma = sqrt(Q/n)
+
+    This is also contained in the function calculateStdDevFromQ in this module. The average is equal to M.
+    """
+
+    if not validateNumericInput(newPoint): raise MathAlgoException("Provided a non-valid newPoint")
+    if not validateNumericInput(n): raise MathAlgoException("Provided a non-valid n")
+
+    if n == 1:
+        M = newPoint
+        Q = 0.0
+    else:
+        if not validateNumericInput(oldM): raise MathAlgoException("Provided a non-valid oldM")
+        if not validateNumericInput(oldQ): raise MathAlgoException("Provided a non-valid oldQ")
+        M = oldM + (newPoint - oldM) / n
+        Q = oldQ + ((n - 1) * (newPoint - oldM) * (newPoint - oldM) / n)
+
+    return M, Q
+
+
+def calculateStdDevFromQ(Q, n):
+    """
+    _calculateStdDevFromQ_
+
+    If Q is the sum of the squared differences of some points to their average,
+    then the standard deviation is given by:
+
+    sigma = sqrt(Q/n)
+
+    This function calculates that formula
+    """
+    if not validateNumericInput(Q): raise MathAlgoException("Provided a non-valid Q")
+    if not validateNumericInput(n): raise MathAlgoException("Provided a non-valid n")
+
+    sigma = math.sqrt(Q / n)
+
+    if not validateNumericInput(sigma): return 0.0
+
+    return sigma

WMCore/Algorithms/MiscAlgos.py

@@ -0,0 +1,67 @@
+#! /usr/bin/env python
+
+"""
+_MiscAlgos_
+
+Useful little tools that can be handy anywhere.
+"""
+import logging
+
+def sortListByKey(data, key):
+    """
+    Return list of dictionaries as a
+    dictionary of lists of dictionaries
+    keyed by one original key
+
+    """
+    final = {}
+
+    for entry in data:
+        value = entry.get(key, None)
+        if value == None:
+            # Empty dict value?
+            # This is an error, but we can't handle it here
+            logging.error("Found entry with no key in sortListByKey: %s", entry)
+            logging.error("Skipping")
+            continue
+        if isinstance(value, set):
+            try:
+                v     = value
+                value = v.pop()
+                v.add(value)
+            except KeyError:
+                # Set was empty?
+                # This is peculiar, we can't handle this.
+                logging.error("Found list entry with empty key set in sortListByKey: %s", entry)
+                logging.error("Skipping")
+                continue
+        if value not in final:
+            final[value] = []
+        final[value].append(entry)
+
+    return final
+
+
+
+## {{{ http://code.activestate.com/recipes/576644/ (r1)
+def dict_diff(first, second):
+    """ Return a dict of keys that differ with another config object.  If a value is
+        not found in one fo the configs, it will be represented by KEYNOTFOUND.
+        @param first:   Fist dictionary to diff.
+        @param second:  Second dicationary to diff.
+        @return diff:   Dict of Key => (first.val, second.val)
+    """
+    KEYNOTFOUND = '<KEYNOTFOUND>'
+    diff = {}
+    # Check all keys in first dict
+    for key in first:
+        if (key not in second):
+            diff[key] = (first[key], KEYNOTFOUND)
+        elif (first[key] != second[key]):
+            diff[key] = (first[key], second[key])
+    # Check all keys in second dict to find missing
+    for key in second:
+        if (key not in first):
+            diff[key] = (KEYNOTFOUND, second[key])
+    return diff
+## end of http://code.activestate.com/recipes/576644/ }}}

WMCore/Algorithms/ParseXMLFile.py

@@ -0,0 +1,115 @@
+#!/bin/env python
+
+"""
+_ParseXMLFile_
+
+This holds the methods used to take an xmlFilename and return a tree structure.
+Used for the expat xml parsers
+
+"""
+
+from builtins import next, str, object
+from future.utils import viewitems
+
+import xml.parsers.expat
+
+class Node(object):
+    """
+    _Node_
+
+    Really simple DOM like container to simplify parsing the XML file
+    and formatting the character data without all the whitespace guff
+
+    """
+    def __init__(self, name, attrs):
+        self.name = str(name)
+        self.attrs = {}
+        self.text = None
+        for k, v in viewitems(attrs):
+            self.attrs.__setitem__(str(k), str(v))
+        self.children = []
+
+    def __str__(self):
+
+        result = " %s %s \"%s\"\n" % (self.name, self.attrs, self.text)
+        for child in self.children:
+            result += str(child)
+        return result
+
+
+
+def coroutine(func):
+    """
+    _coroutine_
+
+    Decorator method used to prime coroutines
+
+    """
+    def start(*args,**kwargs):
+        cr = func(*args,**kwargs)
+        next(cr)
+        return cr
+    return start
+
+
+def xmlFileToNode(reportFile):
+    """
+    _xmlFileToNode_
+
+    Use expat and the build coroutine to parse the XML file and build
+    a node structure
+
+    """
+    node = Node("JobReports", {})
+    expat_parse(open(reportFile, 'rb'),
+                build(node))
+    return node
+
+
+def expat_parse(f, target):
+    """
+    _expat_parse_
+
+    Expat based XML parsing that feeds a node building coroutine
+
+    """
+    parser = xml.parsers.expat.ParserCreate()
+    #parser.buffer_size = 65536
+    parser.buffer_text = True
+
+    # a leftover from the py2py3 migration - TO BE REMOVED
+    # parser.returns_unicode = False
+    parser.StartElementHandler = \
+       lambda name,attrs: target.send(('start',(name,attrs)))
+    parser.EndElementHandler = \
+       lambda name: target.send(('end',name))
+    parser.CharacterDataHandler = \
+       lambda data: target.send(('text',data))
+    parser.ParseFile(f)
+
+
+@coroutine
+def build(topNode):
+    """
+    _build_
+
+    Node structure builder that is fed from the expat_parse method
+
+    """
+    nodeStack = [topNode]
+    charCache = []
+    while True:
+        event, value = (yield)
+        if event == "start":
+            charCache = []
+            newnode = Node(value[0], value[1])
+            nodeStack[-1].children.append(newnode)
+            nodeStack.append(newnode)
+
+        elif event == "text":
+            charCache.append(value)
+
+        else: # end
+            nodeStack[-1].text = str(''.join(charCache)).strip()
+            nodeStack.pop()
+            charCache = []

WMCore/Algorithms/Permissions.py

@@ -0,0 +1,27 @@
+'''
+A bunch of functions that check the permissions on a file are what they should
+be, or more restrictive.
+'''
+
+
+
+from builtins import oct
+import os
+import stat
+
+def check_permissions(filehandle, permission, pass_stronger = False):
+    info = os.stat(filehandle)
+    filepermission = oct(info[stat.ST_MODE] & 0o777)
+    if pass_stronger:
+        assert filepermission <= permission, "file's permissions are too weak"
+    else:
+        assert filepermission == permission, "file does not have the correct permissions"
+
+def owner_readonly(file_):
+    check_permissions(file_, oct(0o400))
+
+def owner_readwrite(file_):
+    check_permissions(file_, oct(0o600))
+
+def owner_readwriteexec(file_):
+    check_permissions(file_, oct(0o700))

WMCore/Algorithms/Singleton.py

@@ -0,0 +1,58 @@
+#! /usr/bin/env python
+
+"""
+A singleton pattern implemented in python. Adapted from ActiveState Code
+Recipe 52558: The Singleton Pattern implemented with Python
+http://code.activestate.com/recipes/52558/
+"""
+
+
+
+
+from builtins import object
+class Singleton(object):
+    """
+    A python singleton
+    """
+
+
+    class SingletonImplementation(object):
+        """
+        Implementation of the singleton interface
+        """
+
+        def singletonId(self):
+            """
+            Test method, return singleton id
+            """
+            return id(self)
+
+
+    # Storage for the instance reference
+    __instance = None
+
+    def __init__(self):
+        """
+        Create singleton instance
+        """
+        # Check whether we already have an instance
+        if Singleton.__instance is None:
+            # Create and remember instance
+            Singleton.__instance = Singleton.SingletonImplementation()
+
+        # Store instance reference as the only member in the handle
+        self.__dict__['_Singleton__instance'] = Singleton.__instance
+
+
+    def __getattr__(self, attr):
+        """
+        Delegate access to implementation
+        """
+        return getattr(self.__instance, attr)
+
+
+    def __setattr__(self, attr, value):
+        """
+        Delegate access to implementation
+        """
+        return setattr(self.__instance, attr, value)

WMCore/Algorithms/SubprocessAlgos.py

@@ -0,0 +1,129 @@
+#!/bin/env python
+
+"""
+_SubprocessAlgos_
+
+Little tricks you can do on the command line with Subprocess
+i.e., stand-ins for Linux command line functions
+
+"""
+
+from builtins import str
+import os
+import re
+import signal
+import logging
+import subprocess
+
+from WMCore.Algorithms.Alarm import Alarm, alarmHandler
+from WMCore.WMException      import WMException
+from Utils.Utilities import decodeBytesToUnicode
+from Utils.PythonVersion import PY3
+
+class SubprocessAlgoException(WMException):
+    """
+    _SubprocessAlgoException_
+
+    Clever exception that does nothing.  Cleverly.
+    """
+    pass
+
+
+def findPIDs(name, user = os.getpid()):
+    """
+    Finds the PIDs for a process with name name being used by a user with a certain uid
+
+    """
+
+    pids = []
+
+    ps = subprocess.Popen(['ps', '-u', user, 'w'], stdout=subprocess.PIPE).communicate()[0]
+    processes = ps.split('\n')
+
+    for line in processes:
+        if len(line.split()) < 5:
+            continue
+        if re.match(name, line.split()[4]):
+            #Then we have matching process
+            pids.append(line.split()[0])
+
+    return pids
+
+
+def killProcessByName(name, user = os.getpid(), sig = None):
+    """
+    Kills all processes of a certain type (name)
+
+    """
+
+    pids = findPIDs(name = name, user = user)
+
+    if len(pids) == 0:
+        #We have no processes to kill of this type
+        return pids
+
+    command = ['kill']
+    if sig:
+        command.append('-%i' % sig)
+    for pid in pids:
+        command.append(pid)
+
+    subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]
+
+
+    return pids
+
+
+def tailNLinesFromFile(file_, n):
+    """
+    Loads the last N lines from a file
+
+    """
+
+    if not os.path.isfile(file_):
+        return None
+
+    command = ['tail', '-n', str(n), file_]
+
+    output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]
+
+    return output.split('\n')
+
+
+
+def runCommand(cmd, shell = True, timeout = None):
+    """
+    Run generic command
+
+    This is NOT secure and hence NOT recommended
+    It does however have the timeout functions built into it
+    timeout must be an int
+    Note, setting timeout = 0 does nothing!
+    """
+
+    if timeout:
+        if not isinstance(timeout, int):
+            timeout = None
+            logging.error("SubprocessAlgo.runCommand expected int timeout, got %s", timeout)
+        else:
+            signal.signal(signal.SIGALRM, alarmHandler)
+            signal.alarm(timeout)
+    try:
+        pipe = subprocess.Popen(cmd, stdout = subprocess.PIPE,
+                                stderr = subprocess.PIPE, shell = shell)
+        stdout, stderr = pipe.communicate()
+        if PY3:
+            stdout = decodeBytesToUnicode(stdout)
+            stderr = decodeBytesToUnicode(stderr)
+        returnCode     = pipe.returncode
+    except Alarm:
+        msg =  "Alarm sounded while running command after %s seconds.\n" % timeout
+        msg += "Command: %s\n" % cmd
+        msg += "Raising exception"
+        logging.error(msg)
+        raise SubprocessAlgoException(msg)
+
+    if timeout:
+        signal.alarm(0)
+
+    return stdout, stderr, returnCode

WMCore/Algorithms/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+"""
+__init__
+
+"""
+
+__all__ = []