wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/DataStructs/Mask.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+"""
+_Mask_
+
+Basic Mask object that can be used to restrict the run/lumi/event range in a
+job in two ways:
+
+- Inclusive: The stuff within the mask is processed
+- Exclusive: The stuff outside of the mask is processed
+
+"""
+
+from builtins import range
+from WMCore.DataStructs.Run import Run
+
+
+class Mask(dict):
+    """
+    _Mask_
+    """
+
+    def __init__(self, **kwargs):
+        dict.__init__(self, **kwargs)
+        self.inclusive = True
+        self.setdefault("inclusivemask", True)
+        self.setdefault("FirstEvent", None)
+        self.setdefault("LastEvent", None)
+        self.setdefault("FirstLumi", None)
+        self.setdefault("LastLumi", None)
+        self.setdefault("FirstRun", None)
+        self.setdefault("LastRun", None)
+        self.setdefault("runAndLumis", {})
+
+    def setMaxAndSkipEvents(self, maxEvents, skipEvents):
+        """
+        _setMaxAndSkipEvents_
+
+        Set FirstEvent & LastEvent fields as max & skip events
+
+        """
+
+        self['FirstEvent'] = skipEvents
+        if maxEvents is not None:
+            self['LastEvent'] = skipEvents + maxEvents
+
+        return
+
+    def setMaxAndSkipLumis(self, maxLumis, skipLumi):
+        """
+        _setMaxAndSkipLumis
+
+        Set the Maximum number of lumi sections and the starting point
+
+        """
+
+        self['FirstLumi'] = skipLumi
+        self['LastLumi'] = skipLumi + maxLumis
+
+        return
+
+    def setMaxAndSkipRuns(self, maxRuns, skipRun):
+        """
+        _setMaxAndSkipRuns
+
+        Set the Maximum number of runss and the starting point
+
+        """
+
+        self['FirstRun'] = skipRun
+        self['LastRun'] = skipRun + maxRuns
+
+        return
+
+    def getMaxEvents(self):
+        """
+        _getMaxEvents_
+
+        return maxevents setting
+
+        """
+        if self['LastEvent'] is None or self['FirstEvent'] is None:
+            return None
+        return self['LastEvent'] - self['FirstEvent'] + 1
+
+    def getMax(self, keyType=None):
+        """
+        _getMax_
+
+        returns the maximum number of runs/events/etc of the type of the type string
+        """
+        if 'First%s' % (keyType) not in self:
+            return None
+        if self['First%s' % (keyType)] is None or self['Last%s' % (keyType)] is None:
+            return None
+        return self['Last%s' % (keyType)] - self['First%s' % (keyType)] + 1
+
+    def addRun(self, run):
+        """
+        _addRun_
+
+        Add a run object
+        """
+        run.lumis.sort()
+        firstLumi = run.lumis[0]
+        lastLumi = run.lumis[0]
+        for lumi in run.lumis:
+            if lumi <= lastLumi + 1:
+                lastLumi = lumi
+            else:
+                self.addRunAndLumis(run.run, lumis=[firstLumi, lastLumi])
+                firstLumi = lumi
+                lastLumi = lumi
+        self.addRunAndLumis(run.run, lumis=[firstLumi, lastLumi])
+        return
+
+    def addRunWithLumiRanges(self, run, lumiList):
+        """
+        _addRunWithLumiRanges_
+
+        Add to runAndLumis with call signature
+        addRunWithLumiRanges(run=run, lumiList = [[start1,end1], [start2, end2], ...]
+        """
+        self['runAndLumis'][run] = lumiList
+        return
+
+    def addRunAndLumis(self, run, lumis=None):
+        """
+        _addRunAndLumis_
+
+        Add runs and lumis directly
+        TODO: The name of this function is a little misleading. If you pass a list of lumis
+              it ignores the content of the list and adds a range based on the max/min in
+              the list. Missing lumis in the list are ignored.
+
+        NOTE: If the new run/lumi range overlaps with the pre-existing lumi ranges in the
+              mask, no attempt is made to merge these together.  This can result in a mask
+              with duplicate lumis.
+        """
+        lumis = lumis or []
+        if not isinstance(lumis, list):
+            lumis = list(lumis)
+
+        if run not in self['runAndLumis']:
+            self['runAndLumis'][run] = []
+
+        self['runAndLumis'][run].append([min(lumis), max(lumis)])
+
+        return
+
+    def getRunAndLumis(self):
+        """
+        _getRunAndLumis_
+
+        Return list of active runs and lumis
+        """
+
+        return self['runAndLumis']
+
+    def runLumiInMask(self, run, lumi):
+        """
+        _runLumiInMask_
+
+        See if a particular runLumi is in the mask
+        """
+
+        if self['runAndLumis'] == {}:
+            # Empty dictionary
+            # ALWAYS TRUE
+            return True
+
+        if run not in self['runAndLumis']:
+            return False
+
+        for pair in self['runAndLumis'][run]:
+            # Go through each max and min pair
+            if pair[0] <= lumi and pair[1] >= lumi:
+                # Then the lumi is bracketed
+                return True
+
+        return False
+
+    def filterRunLumisByMask(self, runs):
+        """
+        _filterRunLumisByMask_
+
+        Pass a Mask a list of run objects, get back a list of
+        run objects that correspond to the actual mask allowed values
+        """
+        if self['runAndLumis'] == {}:
+            # Empty dictionary
+            # ALWAYS TRUE
+            return runs
+
+        runDict = {}
+        for r in runs:
+            if r.run in runDict:
+                runDict[r.run].extendLumis(r.lumis)
+            else:
+                runDict[r.run] = r
+
+        maskRuns = set(self["runAndLumis"].keys())
+        passedRuns = set([r.run for r in runs])
+        filteredRuns = maskRuns.intersection(passedRuns)
+
+        newRuns = set()
+        for runNumber in filteredRuns:
+            maskLumis = set()
+            for pair in self["runAndLumis"][runNumber]:
+                if pair[0] == pair[1]:
+                    maskLumis.add(pair[0])
+                else:
+                    maskLumis = maskLumis.union(list(range(pair[0], pair[1] + 1)))
+
+            filteredLumis = set(runDict[runNumber].lumis).intersection(maskLumis)
+            if len(filteredLumis) > 0:
+                filteredLumiEvents = [(lumi, runDict[runNumber].getEventsByLumi(lumi)) for lumi in filteredLumis]
+                newRuns.add(Run(runNumber, *filteredLumiEvents))
+
+        return newRuns

WMCore/DataStructs/MathStructs/ContinuousSummaryHistogram.py

@@ -0,0 +1,197 @@
+"""
+_ContinuousSummaryHistogram_
+
+Continuous histogram module, to be used by the TaskArchiver
+to store histograms in the summary where the data is a continuous number.
+
+Created on Nov 20, 2012
+
+@author: dballest
+"""
+from __future__ import division
+import math
+
+from WMCore.DataStructs.MathStructs.SummaryHistogram import SummaryHistogram
+from WMCore.Algorithms.MathAlgos import validateNumericInput
+from WMCore.Algorithms.MathAlgos import calculateRunningAverageAndQValue, calculateStdDevFromQ
+
+class ContinuousSummaryHistogram(SummaryHistogram):
+    """
+    _ContinuousSummaryHistogram_
+
+    A histogram where there are continuous points
+    with certain frequency, it follows
+    that there is only one value in Y and
+    that the average and standard deviation are
+    not calculated on the frequency values but the X values.
+    """
+
+    def __init__(self, title, xLabel, yLabel = None,
+                 roundingDecimals = 2, nBins = None,
+                 dropOutliers = False, sigmaLimit = 5,
+                 storeHistogram = True):
+        """
+        __init__
+
+        Initialize a more complex histogram structure, containing different
+        data to calculate online average and standard deviations. This data is also
+        stored in the JSON to allow rebuilding and adding histograms.
+
+        All histograms are binned when requested, the resolution can be specified
+        through nBins, otherwise the value used is the one recommended in:
+        Wand, M.P. (1997), "Data-Based Choice of Histogram Bin Width," The American Statistician, 51, 59-64.
+
+        If specified, outlier farther than sigmaLimit standard deviations from the
+        mean will not be included in the binned histogram.
+        """
+        # Initialize the parent object
+        SummaryHistogram.__init__(self, title, xLabel)
+
+        # Indicate this is a discrete histogram
+        self.continuous = True
+
+        # Add data only used in the continuous version
+        self.yLabel            = yLabel
+        self.nPoints           = 0
+        self.QValue            = None
+        self.average           = None
+
+        # Configuration parameters for the continuous histograms
+        self.roundingDecimals = roundingDecimals
+        self.fixedNBins       = nBins
+        self.dropOutliers     = dropOutliers
+        self.sigmaLimit       = sigmaLimit
+        self.binned           = False
+        self.storeHistogram   = storeHistogram
+
+        # Override initialization of some attributes
+        self.average = 0.0
+        self.stdDev  = 0.0
+
+        return
+
+    def addPoint(self, xValue, yLabel = None):
+        """
+        _addPoint_
+
+        Add a point from a continuous set (only-numbers allowed currently) to the histogram data,
+        calculate the running average and standard deviation.
+        If no y-label had been specified before, one must be supplied
+        otherwise the given y-label must be either None or equal
+        to the stored value.
+        """
+        if self.binned:
+            # Points can't be added to binned histograms!
+            raise Exception("Points can't be added to binned histograms")
+
+        if self.yLabel is None and yLabel is None:
+            raise Exception("Some y-label must be stored for the histogram")
+        elif self.yLabel is None:
+            self.yLabel = yLabel
+        elif yLabel is not None and self.yLabel != yLabel:
+            raise Exception("Only one y-label is allowed on continuous histograms")
+
+        if not validateNumericInput(xValue):
+            # Do nothing if it is not a number
+            return
+
+        xValue = float(xValue)
+        xValue = round(xValue, self.roundingDecimals)
+
+        if self.storeHistogram:
+            if xValue not in self.data:
+                self.data[xValue] = 0
+            self.data[xValue] += 1
+
+        self.nPoints += 1
+
+        (self.average, self.QValue) = calculateRunningAverageAndQValue(xValue, self.nPoints, self.average, self.QValue)
+
+        return
+
+    def __add__(self, other):
+        #TODO: For HG1302, support multiple agents properly in the workload summary
+        raise NotImplementedError
+
+    def toJSON(self):
+        """
+        _toJSON_
+
+        Bin the histogram if any, calculate the standard deviation. Store
+        the internal data needed for reconstruction of the histogram
+        from JSON and call superclass toJSON method.
+        """
+        if self.nPoints:
+            self.stdDev = calculateStdDevFromQ(self.QValue, self.nPoints)
+        if not self.binned and self.storeHistogram:
+            self.binHistogram()
+        self.jsonInternal = {}
+        self.jsonInternal['yLabel'] = self.yLabel
+        self.jsonInternal['QValue'] = self.QValue
+        self.jsonInternal['nPoints'] = self.nPoints
+        return SummaryHistogram.toJSON(self)
+
+    def binHistogram(self):
+        """
+        _binHistogram_
+
+        Histograms of continuous data must be binned,
+        this takes care of that using given or optimal parameters.
+        Note that this modifies the data object,
+        and points can't be added to the histogram after this.
+        """
+
+        if not self.nPoints:
+            return
+
+        self.binned = True
+
+        # Number of bins can be specified or calculated based on number of points
+        nBins = self.fixedNBins
+        if nBins is None:
+            nBins = int(math.floor((5.0 / 3.0) * math.pow(self.nPoints, 1.0 / 3.0)))
+
+        # Define min and max
+        if not self.dropOutliers:
+            upperLimit = max(self.data.keys())
+            lowerLimit = min(self.data.keys())
+        else:
+            stdDev = calculateStdDevFromQ(self.QValue, self.nPoints)
+            upperLimit = self.average + (stdDev * self.sigmaLimit)
+            lowerLimit = self.average - (stdDev * self.sigmaLimit)
+
+        # Incremental delta
+        delta = abs(float(upperLimit - lowerLimit)) / nBins
+
+        # Build the bins, it's a list of tuples for now
+        bins = []
+
+        a = lowerLimit
+        b = lowerLimit + delta
+        while len(bins) < nBins:
+            bins.append((a, b))
+            a += delta
+            b += delta
+        # Go through data and populate the binned histogram
+        binnedHisto = {}
+        currentBin = 0
+        currentPoint = 0
+        sortedData = sorted(self.data.keys())
+        while currentPoint < len(sortedData):
+            point = sortedData[currentPoint]
+            encodedTuple = "%s,%s" % (bins[currentBin][0], bins[currentBin][1])
+            if encodedTuple not in binnedHisto:
+                binnedHisto[encodedTuple] = 0
+            if point > upperLimit or point < lowerLimit:
+                currentPoint += 1
+            elif currentBin == len(bins) - 1:
+                binnedHisto[encodedTuple] += self.data[point]
+                currentPoint += 1
+            elif point >= bins[currentBin][0] and point < bins[currentBin][1]:
+                binnedHisto[encodedTuple] += self.data[point]
+                currentPoint += 1
+            else:
+                currentBin += 1
+
+        self.data = binnedHisto
+        return

WMCore/DataStructs/MathStructs/DiscreteSummaryHistogram.py

@@ -0,0 +1,92 @@
+"""
+_DiscreteSummaryHistogram_
+
+Discrete histogram module, to be used by the TaskArchiver
+to store histograms in the summary where the data is categorical.
+
+Created on Nov 20, 2012
+
+@author: dballest
+"""
+
+from WMCore.DataStructs.MathStructs.SummaryHistogram import SummaryHistogram
+from WMCore.Algorithms.MathAlgos import getAverageStdDev
+
+class DiscreteSummaryHistogram(SummaryHistogram):
+    """
+    _DiscreteSummaryHistogram_
+
+    A histogram where the data is organized by
+    a finite number of categories, it can have
+    many values for each category.
+    """
+
+
+    def __init__(self, title, xLabel):
+        """
+        __init__
+
+        Initialize a simpler histogram that only stores
+        the histogram. Everything else is calculated when the JSON is requested.
+        """
+        # Initialize the parent object
+        SummaryHistogram.__init__(self, title, xLabel)
+
+        # Indicate this is a discrete histogram
+        self.continuous = False
+
+        # Add data only used in the discrete version
+        self.yLabels    = set()
+
+        # Override initialization of some attributes
+        self.average = {}
+        self.stdDev  = {}
+
+        return
+
+    def addPoint(self, xValue, yLabel):
+        """
+        _addPoint_
+
+        Add point to discrete histogram,
+        x value is a category and therefore not rounded.
+        There can be many yLabel and standard deviations are
+        not calculated online. Histograms are always stored.
+        """
+        if xValue not in self.data:
+            # Record the category
+            self.data[xValue] = {}
+            for label in self.yLabels:
+                self.data[xValue][label] = 0
+
+        if yLabel not in self.yLabels:
+            # Record the label
+            self.yLabels.add(yLabel)
+            self.average[yLabel] = 0.0
+            self.stdDev[yLabel]  = 0.0
+            for category in self.data:
+                self.data[category][yLabel] = 0
+
+        self.data[xValue][yLabel] += 1
+
+        return
+
+    def __add__(self, other):
+        #TODO: For HG1302, support multiple agents properly in the workload summary
+        raise NotImplementedError
+
+    def toJSON(self):
+        """
+        _toJSON_
+
+        Calculate average and standard deviation, store it
+        and call the parent class toJSON method
+        """
+
+        for yLabel in self.yLabels:
+            numList = []
+            for xValue in self.data:
+                numList.append(self.data[xValue][yLabel])
+            (self.average[yLabel], self.stdDev[yLabel]) = getAverageStdDev(numList)
+
+        return SummaryHistogram.toJSON(self)

WMCore/DataStructs/MathStructs/SummaryHistogram.py

@@ -0,0 +1,117 @@
+"""
+_SummaryHistogram_
+
+Histogram module, to be used by the TaskArchiver
+to store histograms in the summary.
+
+Created on Nov 16, 2012
+
+@author: dballest
+"""
+from builtins import str
+from WMCore.DataStructs.WMObject import WMObject
+
+class SummaryHistogram(WMObject):
+    """
+    _SummaryHistogram_
+
+    Histogram object, provides familiar CRUD methods
+    which take care of most of the statistical
+    calculations when adding points, this object
+    can also be converted into a dictionary
+    for JSON documents. It knows how to combine
+    with other histograms and create itself from
+    a dictionary provided it has matching structure.
+
+    This is an interface, the real work is done
+    by the ContinuousSummaryHistogram and
+    DiscreteSummaryHistogram objects
+    """
+
+    def __init__(self, title = None, xLabel = None):
+        """
+        __init__
+        Initialize the elements in the object.
+        """
+
+        # Meta-information about the histogram, it can be changed at any point
+        self.title      = title
+        self.xLabel     = xLabel
+
+        # These shouldn't be touched from anything outside the SummaryHistogram object and children classes
+        self.continuous         = None
+        self.jsonInternal       = None
+        self.data               = {}
+        self.average            = None
+        self.stdDev             = None
+
+        return
+
+    def setTitle(self, newTitle):
+        """
+        _setTitle_
+
+        Set the title
+        """
+        self.title = newTitle
+        return
+
+    def setHorizontalLabel(self, xLabel):
+        """
+        _setHorizontalLabel_
+
+        Set the label on the x axis
+        """
+        self.xLabel = xLabel
+        return
+
+    def addPoint(self, xValue, yLabel):
+        """
+        _addPoint_
+
+        Add a point to the histogram data, a histogram
+        can have many types of y values for the same x if
+        x is continuous otherwise it is only one yLabel.
+        They should be in a similar scale for best results.
+        """
+        raise NotImplementedError("SummaryHistogram objects can't be used, use either the continuous or discrete implementation")
+
+    def toJSON(self):
+        """
+        _toJSON_
+
+        Return a dictionary which is compatible
+        with a JSON object
+        """
+        if self.continuous is None:
+            raise TypeError("toJSON can't be called on a bare SummaryHistogram object")
+
+        # Get what the children classes did
+        jsonDict = {}
+        jsonDict['internalData'] = self.jsonInternal or {}
+
+        # Add the common things
+        jsonDict['title']      = self.title
+        jsonDict['xLabel']     = self.xLabel
+        jsonDict['continuous'] = self.continuous
+        jsonDict['data']       = self.data
+        jsonDict['stdDev']     = self.stdDev
+        jsonDict['average']    = self.average
+
+        return jsonDict
+
+    def __add__(self, other):
+        """
+        __add__
+
+        Add two histograms, combine statistics.
+        """
+        raise NotImplementedError("SummaryHistogram objects can't be used, use either the continuous or discrete implementation")
+
+    def __str__(self):
+        """
+        __str__
+
+        Return the str object of the JSON
+        """
+        return str(self.toJSON())

WMCore/DataStructs/Pickleable.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+_Pickleable_
+
+An object that can be persisted using pickle
+
+"""
+__all__ = []
+
+
+from WMCore.DataStructs.WMObject import WMObject
+
+class Pickleable(WMObject):
+    def save(self):
+        """
+        Pickle the object
+        """
+        pass
+
+    def load(self):
+        """
+        Load the object from a pickle file
+        """
+        pass