wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/DataStructs/Run.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env python
+
+"""
+_Run_
+
+container representing a run, and its constituent lumi sections and event counts
+"""
+
+from __future__ import print_function
+from future.utils import viewitems, listitems
+
+import hashlib
+from builtins import str as newstr, bytes as newbytes
+from Utils.PythonVersion import PY3
+from Utils.Utilities import encodeUnicodeToBytesConditional, encodeUnicodeToBytes
+from WMCore.DataStructs.WMObject import WMObject
+
+
+class Run(WMObject):
+    """
+    _Run_
+
+    Run container, is a list of lumi sections with associate event counts
+
+    TODO 
+    - use the decorator `from functools import total_ordering` after
+      dropping support for python 2.6
+    - then, drop __ne__, __le__, __gt__, __ge__
+    """
+
+    def __init__(self, runNumber=None, *newLumis):
+        WMObject.__init__(self)
+        self.run = runNumber
+        self.eventsPerLumi = {}
+        self.extendLumis(newLumis)
+
+    def __str__(self):
+        return "Run%s:%s" % (self.run, self.eventsPerLumi)
+
+    def __eq__(self, rhs):
+        """
+        Check equality of run numbers and then underlying lumi/event dicts
+        """
+        if not isinstance(rhs, Run):
+            return False
+        if self.run != rhs.run:
+            return False
+        return self.eventsPerLumi == rhs.eventsPerLumi
+
+    def __ne__(self, rhs):
+        return not self.__eq__(rhs)
+
+    def __lt__(self, rhs):
+        """
+        Compare on run # first, then by lumis as a list, then by events in each lumi
+        """
+        # check run number
+        if self.run != rhs.run:
+            return self.run < rhs.run
+        # if same run number, check list of lumi sections
+        if sorted(self.eventsPerLumi.keys()) != sorted(rhs.eventsPerLumi.keys()):
+            return sorted(self.eventsPerLumi.keys()) < sorted(rhs.eventsPerLumi.keys())
+        # if same list of lumis, check events in each lumi section
+        for lumiNumber in sorted(self.eventsPerLumi):
+            if self.eventsPerLumi[lumiNumber] == rhs.eventsPerLumi.get(lumiNumber):
+                continue
+            else: 
+                return self.eventsPerLumi[lumiNumber] < rhs.eventsPerLumi.get(lumiNumber)
+        # if same run number, same lumi sections list, 
+        # same events in each lumi section:
+        # then the runs are equal and __lt__ should return false
+        return False
+
+    def __le__(self, other):
+        return self.__lt__(other) or self.__eq__(other)
+
+    def __gt__(self, other):
+        return not self.__le__(other)
+
+    def __ge__(self, other):
+        return not self.__lt__(other)
+
+    def extend(self, items):
+        """
+        Redirect to the function that already does this
+        """
+        self.extendLumis(items)
+        return
+
+    def __add__(self, rhs):
+        """
+        Combine two runs
+        """
+        if self.run != rhs.run:
+            msg = "Adding together two different runs"
+            msg += "Run %s does not equal Run %s" % (self.run, rhs.run)
+            raise RuntimeError(msg)
+
+        for lumi, events in viewitems(rhs.eventsPerLumi):
+            if lumi not in self.eventsPerLumi or not self.eventsPerLumi[lumi]:  # Either doesn't exist, 0, or None
+                self.eventsPerLumi[lumi] = events
+            else:
+                self.eventsPerLumi[lumi] += events
+        return self
+
+    def __iter__(self):
+        return self.eventsPerLumi.__iter__()
+
+    def __next__(self):
+        """
+        __next__ no longer needed
+        """
+        raise NotImplementedError
+
+    def __len__(self):
+        """
+        Number of lumis
+        """
+        return self.eventsPerLumi.__len__()
+
+    def __getitem__(self, key):
+        """
+        Get the nth lumi from the list (no event count)
+        """
+        return sorted(self.eventsPerLumi.keys()).__getitem__(key)
+
+    def __setitem__(self, key, lumi):
+        """
+        Replace the nth lumi from the list (no event count)
+        """
+        try:
+            oldLumi = sorted(self.eventsPerLumi.keys())[key]  # Extract the lumi from the sorted list
+            del self.eventsPerLumi[oldLumi]  # Delete it and add the new one
+        except IndexError:
+            pass
+        self.appendLumi(lumi)
+
+    def __delitem__(self, key):
+        try:
+            oldLumi = sorted(self.eventsPerLumi.keys())[key]  # Extract the lumi from the sorted list
+            del self.eventsPerLumi[oldLumi]  # Delete it
+        except IndexError:
+            pass
+
+    def __hash__(self):
+        """
+        Calculate the value of the hash
+
+        NOTE: Python2 maxint is:
+        > python -c 'import sys; print(sys.maxint)'
+        9223372036854775807
+        so we cannot use the full range of the hexadecimal hash code because
+        it could cause an integer overflow. This is the maximum slice/value we
+        can safely use:
+        > int(15 * "f", base=16)
+        1152921504606846975
+        """
+        if isinstance(self.run, (newstr, newbytes)):
+            value = encodeUnicodeToBytesConditional(self.run, condition=PY3)
+        else:
+            value = encodeUnicodeToBytesConditional(str(self.run), condition=PY3)
+        hashValue = hashlib.sha1(value)
+        # Generate immutable sorted list of lumis
+        frozenEvents = str(sorted(listitems(self.eventsPerLumi), key=lambda x: x[0]))
+        hashValue.update(encodeUnicodeToBytes(frozenEvents))
+        return int(hashValue.hexdigest()[:15], 16)
+
+    @property
+    def lumis(self):
+        """
+        Property that makes existing uses of myRun.lumis function by returning a list
+        """
+        return sorted(self.eventsPerLumi.keys())
+
+    @lumis.setter
+    def lumis(self, lumiList):
+        """
+        Setter to allow for replacement of the lumis with a list or list of tuples
+        """
+        self.eventsPerLumi = {}  # Remove existing dictionary
+        for lumi in lumiList:
+            if isinstance(lumi, (list, tuple)):
+                self.eventsPerLumi[lumi[0]] = lumi[1]
+            else:
+                self.eventsPerLumi[lumi] = None
+
+    def extendLumis(self, lumiList):
+        """
+        Method to replace myRun.lumis.extend() which does not work with the property
+        """
+        for lumi in lumiList:
+            if not isinstance(lumi, (list, tuple)):  # comma separated lumi numbers
+                self.eventsPerLumi[lumi] = None
+            else:
+                if isinstance(lumi, list) and not isinstance(lumi[0], tuple):  # then it's a plain list
+                    for l in lumi:
+                        self.eventsPerLumi[l] = None
+                else:
+                    if isinstance(lumi, tuple):  # it's an unpacked list of tuples
+                        lumi = [(lumi)]
+                    # it's a list/tuple of tuples
+                    for tp in lumi:
+                        if tp[0] in self.eventsPerLumi and self.eventsPerLumi[tp[0]]:
+                            self.eventsPerLumi[tp[0]] += tp[1]  # Already exists, add events
+                        else:  # Doesn't exist or is 0 or None
+                            self.eventsPerLumi[tp[0]] = tp[1]
+
+    def appendLumi(self, lumi):
+        """
+        Method to replace myRun.lumis.append() which does not work with the property
+        """
+        if isinstance(lumi, (list, tuple)) and self.eventsPerLumi[lumi[0]]:  # Already exists, add events
+            self.eventsPerLumi[lumi[0]] += lumi[1]
+        elif isinstance(lumi, (list, tuple)):  # Doesn't exist or is 0 or None
+            self.eventsPerLumi[lumi[0]] = lumi[1]
+        else:  # Just given lumis, not events
+            if lumi not in self.eventsPerLumi:  # Don't overwrite existing events
+                self.eventsPerLumi[lumi] = None
+
+    def getEventsByLumi(self, lumi):
+        """
+        getter to select event counts by given lumi
+        """
+        return self.eventsPerLumi.get(lumi)
+
+    def json(self):
+        """
+        _json_
+
+        Convert to JSON friendly format.  Include some information for the
+        thunker so that we can convert back.
+        """
+        return {"Run": self.run, "Lumis": self.eventsPerLumi,
+                "thunker_encoded_json": True, "type": "WMCore.DataStructs.Run.Run"}
+
+    def __to_json__(self, thunker=None):
+        """
+        __to_json__
+
+        This is the standard way we JSONize other objects.
+        Included here so we have a uniform method.
+        """
+        return self.json()
+
+    def __from_json__(self, jsondata, thunker):
+        """
+        __from_json__
+
+        Convert JSON data back into a Run object with integer lumi numbers
+        """
+        self.run = jsondata["Run"]
+        self.eventsPerLumi = {}
+        for lumi, events in viewitems(jsondata["Lumis"]):
+            self.eventsPerLumi[int(lumi)] = events  # Make the keys integers again
+
+        return self

WMCore/DataStructs/Subscription.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+"""
+_Subscription_
+
+workflow + fileset = subscription
+"""
+
+from builtins import range
+import copy
+
+from WMCore.DataStructs.Pickleable import Pickleable
+from WMCore.DataStructs.Fileset import Fileset
+
+class Subscription(Pickleable, dict):
+    def __init__(self, fileset = None, workflow = None,
+                 split_algo = "FileBased", type = "Processing"):
+        if fileset == None:
+            fileset = Fileset()
+
+        self.setdefault('fileset', fileset)
+        self.setdefault('workflow', workflow)
+        self.setdefault('type', type)
+
+        self.setdefault('split_algo', split_algo)
+
+        self.available = Fileset(name=fileset.name,
+                                 files = fileset.getFiles())
+
+        self.acquired = Fileset(name='acquired')
+        self.completed = Fileset(name='completed')
+        self.failed = Fileset(name='failed')
+
+    def name(self):
+        return self.getWorkflow().name.replace(' ', '') + '_' + \
+                    self.getFileset().name.replace(' ', '')
+
+    def getWorkflow(self):
+        return self["workflow"]
+
+    def workflowName(self):
+        if self["workflow"] == None:
+            return "Unknown"
+        return self["workflow"].name
+
+    def workflowType(self):
+        if self["workflow"] == None:
+            return "Unknown"
+        return self["workflow"].wfType
+
+    def taskName(self):
+        if self['workflow'] == None:
+            return "Unknown"
+        return self['workflow'].task
+
+    def owner(self):
+        if self['workflow'] == None:
+            return 'Unknown'
+        return self['workflow'].owner
+
+    def getFileset(self):
+        return self['fileset']
+
+    def acquireFiles(self, files = [], size=1):
+        """
+        Return the files acquired
+        """
+        self.acquired.commit()
+        self.available.commit()
+        self.failed.commit()
+        self.completed.commit()
+        retval = []
+        if len(files):
+            for i in files:
+                # Check each set, instead of elif, just in case something has
+                # got out of synch
+                if i in self.available.files:
+                    self.available.files.remove(i)
+                if i in self.failed.files:
+                    self.failed.files.remove(i)
+                if i in self.completed.files:
+                    self.completed.files.remove(i)
+                self.acquired.addFile(i)
+        else:
+            if len(self.available.files) < size or size == 0:
+                size = len(self.available.files)
+            for i in range(size):
+                self.acquired.addFile(self.available.files.pop())
+
+        return self.acquired.listNewFiles()
+
+    def completeFiles(self, files):
+        """
+        Return the number of files complete
+        """
+        self.acquired.commit()
+        self.available.commit()
+        self.failed.commit()
+        self.completed.commit()
+        for i in files:
+            # Check each set, instead of elif, just in case something has
+            # got out of synch
+            if i in self.available.files:
+                self.available.files.remove(i)
+            if i in self.failed.files:
+                self.failed.files.remove(i)
+            if i in self.acquired.files:
+                self.acquired.files.remove(i)
+            self.completed.addFile(i)
+
+    def failFiles(self, files):
+        """
+        Return the number of files failed
+        """
+        self.acquired.commit()
+        self.available.commit()
+        self.failed.commit()
+        self.completed.commit()
+        for i in files:
+            # Check each set, instead of elif, just in case something has
+            # got out of synch
+            if i in self.available.files:
+                self.available.files.remove(i)
+            if i in self.completed.files:
+                self.completed.files.remove(i)
+            if i in self.acquired.files:
+                self.acquired.files.remove(i)
+            self.failed.addFile(i)
+
+    def filesOfStatus(self, status=None, doingJobSplitting = False):
+        """
+        _filesOfStatus_
+
+        Return a Set of File objects that are associated with the subscription
+        and have a particular status.
+        """
+        status = status.title()
+        if status == 'Available':
+            return self.available.getFiles(type='set') - \
+            (self.acquiredFiles() | self.completedFiles() | self.failedFiles())
+        elif status == 'Acquired':
+            return self.acquired.getFiles(type='set')
+        elif status == 'Completed':
+            return self.completed.getFiles(type='set')
+        elif status == 'Failed':
+            return self.failed.getFiles(type='set')
+
+    def availableFiles(self, limit = None, doingJobSplitting = False):
+        """
+        _availableFiles_
+
+        Return a Set of files that are available for processing
+        (e.g. not already in use)
+        """
+        if limit:
+            return list(self.filesOfStatus(status = "Available", doingJobSplitting = doingJobSplitting))[:limit]
+        else:
+            return self.filesOfStatus(status = "Available", doingJobSplitting = doingJobSplitting)
+
+    def acquiredFiles(self):
+        """
+        Set of files marked as acquired.
+        """
+        return self.filesOfStatus(status = "Acquired")
+
+    def completedFiles(self):
+        """
+        Set of files marked as completed.
+        """
+        return self.filesOfStatus(status = "Completed")
+
+    def failedFiles(self):
+        """
+        Set of files marked as failed.
+        """
+        return self.filesOfStatus(status = "Failed")

WMCore/DataStructs/WMObject.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""
+_WMObject_
+
+Helper class that other objects should inherit from
+
+"""
+from builtins import object
+__all__ = []
+
+
+
+
+class WMObject(object):
+    """
+    Helper class that other objects should inherit from
+    """
+    def __init__(self, config = {}):
+        #Config is a WMCore.Configuration
+        self.config = config
+
+    def makelist(self, thelist):
+        """
+        Simple method to ensure thelist is a list
+        """
+        if isinstance(thelist, set):
+            thelist = list(thelist)
+        elif not isinstance(thelist, list):
+            thelist = [thelist]
+        return thelist
+
+    def makeset(self, theset):
+        """
+        Simple method to ensure theset is a set
+        """
+        if not isinstance(theset, set):
+            theset = set(self.makelist(theset))
+        return theset
+
+    def flatten(self, list):
+        """
+        If a list has only one element return just that element, otherwise
+        return the original list
+        """
+        if len(list) == 1:
+            return list[0]
+        return list

WMCore/DataStructs/WorkUnit.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+"""
+_WorkUnit_
+
+Data object that contains details for a single work unit
+
+"""
+
+from __future__ import absolute_import, division, print_function
+
+from future.utils import listitems
+
+import sys
+import hashlib
+import time
+from functools import total_ordering
+
+from Utils.Utilities import encodeUnicodeToBytes
+from WMCore.DataStructs.WMObject import WMObject
+
+
+@total_ordering
+class WorkUnit(WMObject, dict):
+    """
+    _WorkUnit_
+    Data object that contains details for a single work unit
+    corresponding to tables workunit and frl_workunit_assoc
+    """
+
+    fieldsToCopy = ['taskid', 'retry_count', 'last_unit_count', 'last_submit_time', 'status', 'firstevent',
+                    'lastevent', 'fileid']
+    fieldsForInfo = fieldsToCopy + ['run_lumi']
+
+    def __init__(self, taskID=None, retryCount=0, lastUnitCount=None, lastSubmitTime=int(time.time()),
+                 status=0, firstEvent=1, lastEvent=sys.maxsize, fileid=None, runLumi=None):
+        super(WorkUnit, self).__init__(self)
+        self.setdefault('taskid', taskID)
+        self.setdefault('retry_count', retryCount)
+        self.setdefault('last_unit_count', lastUnitCount)
+        self.setdefault('last_submit_time', lastSubmitTime)
+        self.setdefault('status', status)
+
+        self.setdefault('firstevent', firstEvent)
+        self.setdefault('lastevent', lastEvent)
+        self.setdefault('fileid', fileid)
+        self.setdefault('run_lumi', runLumi)
+
+    def __lt__(self, rhs):
+        """
+        Compare work units in task id, run, lumi, first event, last event
+        """
+
+        if self['taskid'] != rhs['taskid']:
+            return self['taskid'] < rhs['taskid']
+        if self['run_lumi'].run != rhs['run_lumi'].run:
+            return self['run_lumi'].run < rhs['run_lumi'].run
+        if self['run_lumi'].lumis != rhs['run_lumi'].lumis:
+            return self['run_lumi'].lumis < rhs['run_lumi'].lumis
+        if self['first_event'] != rhs['first_event']:
+            return self['first_event'] < rhs['first_event']
+        return self['last_event'] < rhs['last_event']
+
+    def __eq__(self, rhs):
+        """
+        Work unit is equal if it has the same task, run, and lumi
+        """
+
+        return (self['taskid'] == rhs['taskid'] and self['run_lumi'].run == self['run_lumi'].run and
+                self['run_lumi'].lumis == self['run_lumi'].lumis and self['firstevent'] == rhs['firstevent'] and
+                self['lastevent'] == rhs['lastevent'])
+
+    def __hash__(self):
+        """
+        Hash function for this dict.
+        """
+        # Generate an immutable sorted string representing this object
+        # NOTE: the run object needs to be hashed
+        immutableSelf = []
+        for keyName in sorted(self):
+            if keyName == "run_lumi":
+                immutableSelf.append((keyName, hash(self[keyName])))
+            else:
+                immutableSelf.append((keyName, self[keyName]))
+        hashValue = hashlib.sha1(encodeUnicodeToBytes(str(immutableSelf)))
+        return int(hashValue.hexdigest()[:15], 16)
+
+    def json(self, thunker=None):
+        """
+        _json_
+
+        Serialize the object.  Only copy select fields and construct one new field.
+        """
+
+        jsonDict = {k: self[k] for k in WorkUnit.fieldsToCopy}
+        jsonDict["run_lumi"] = {"run_number": self['run_lumi'].run, "lumis": self['run_lumi'].lumis}
+
+        return jsonDict
+
+    def __to_json__(self, thunker=None):
+        """
+        __to_json__
+
+        This is the standard way we jsonize other objects.
+        Included here so we have a uniform method.
+        """
+        return self.json(thunker)
+
+    def getInfo(self):
+        """
+        Returns: tuple of parameters for the work unit
+        """
+        return tuple(self[x] for x in WorkUnit.fieldsForInfo)

WMCore/DataStructs/Workflow.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+"""
+_Workflow_
+
+A class that describes some work to be undertaken on some files
+"""
+
+from builtins import str
+from WMCore.DataStructs.Pickleable import Pickleable
+
+class Workflow(Pickleable):
+    def __init__(self, spec = None, owner = "unknown", dn = "unknown",
+                 group = "unknown", owner_vogroup = "unknown",
+                 owner_vorole = "unknown", name = None, task = None,
+                 wfType = None, priority = None):
+        self.spec = spec
+        self.name = name
+        # person making the request
+        self.owner = owner
+        self.dn = dn
+        self.vogroup = owner_vogroup
+        self.vorole = owner_vorole
+        self.group = group
+        # task is the name of the task within the Workload
+        self.task = task
+        self.wfType = wfType
+        self.outputMap = {}
+        self.priority = priority or 0
+
+    def addOutput(self, outputIdentifier, outputFileset,
+                  mergedOutputFileset = None):
+        """
+        _addOutput_
+
+        Associate an output of this workflow with a particular fileset.
+        """
+        mappingDict = {"output_fileset": outputFileset,
+                       "merged_output_fileset": mergedOutputFileset}
+
+        if outputIdentifier in self.outputMap:
+            self.outputMap[outputIdentifier].append(mappingDict)
+        else:
+            self.outputMap[outputIdentifier] = [mappingDict]
+
+        return
+
+    def __str__(self):
+        """
+        __str__
+
+        Print out some useful info just because
+        this does not inherit from dict.
+        """
+
+        d = {'id': self.id, 'spec': self.spec, 'name': self.name,
+             'owner': self.owner, 'task': self.task,
+             'vogroup': self.vogroup, 'vorole': self.vorole,
+             'group': self.group, 'wfType': self.wfType}
+
+        return str(d)

WMCore/DataStructs/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""
+_DataStructs_
+
+A set of base objects to be used/inherited elsewhere
+
+"""
+__all__ = []