wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/WorkQueue/DataStructs/ACDCBlock.py

@@ -0,0 +1,47 @@
+"""
+Set of rules and functions
+Handling resubmission Block construction
+"""
+
+from builtins import object
+
+ACDC_PREFIX = "acdc"
+
+class ACDCBlock(object):
+
+    @staticmethod
+    def name(wmspecName, taskName, offset, numOfFile):
+        taskName = taskName.replace('/', ':')
+        return "/%s/%s/%s/%s/%s" % (ACDC_PREFIX, wmspecName, taskName, offset,
+                                     numOfFile)
+
+    @staticmethod
+    def checkBlockName(blockName):
+        if blockName.split('/')[1] == ACDC_PREFIX:
+            return True
+        else:
+            return False
+
+    @staticmethod
+    def splitBlockName(blockName):
+        """ return False if the blockName is not acdc Block
+            return original elements of block name in order as a dict
+            {'SpecName': blockSplit[1],
+             'TaskName': blockSplit[2],
+             'Offset': int(blockSplit[3]),
+             'NumOfFiles': int(blockSplit[4])},
+             if it is acdc Block
+            raise ValueError if the block is has wrong format
+        """
+        blockSplit = blockName.split('/')
+        if blockSplit[1] != ACDC_PREFIX:
+            return False
+        elif len(blockSplit) != 6:
+            msg = """blockName should contain prefix, wmspec name, task name,
+                     offset and number of files %s""" % blockName
+            raise ValueError(msg)
+        else:
+            return {'SpecName': str(blockSplit[2]),
+                    'TaskName': str(blockSplit[3]).replace(':', '/'),
+                    'Offset': int(blockSplit[4]),
+                    'NumOfFiles': int(blockSplit[5])}

WMCore/WorkQueue/DataStructs/Block.py

@@ -0,0 +1,48 @@
+"""
+_Block_
+
+A dictionary based object meant to represent a row in the active_block/block table.
+"""
+
+
+
+
+class Block(dict):
+    """
+    _Block_
+
+    A dictionary based object meant to represent subset of dbs block.
+    Which will just need for workQueue update.
+    It contains the following keys:
+      Name
+      Size
+      NumEvent
+      NumFiles
+    """
+    def __init__(self, **args):
+        """
+        ___init___
+
+        Initialize all attributes.
+        """
+        dict.__init__(self)
+
+        self.setdefault("Name", None)
+        self.setdefault("Size", None)
+        self.setdefault("NumEvents", None)
+        self.setdefault("NumFiles", None)
+        self.update(args)
+
+    @staticmethod
+    def getBlock(blockInfo):
+        """
+        convert to the Block structure from db column format
+        """
+        block = Block()
+
+        block["Name"] = blockInfo[0]["name"]
+        block["NumFiles"] = blockInfo[0]["num_files"]
+        block["NumEvents"] = blockInfo[0]["num_events"]
+        block["Size"] = blockInfo[0]["block_size"]
+
+        return block

WMCore/WorkQueue/DataStructs/CouchWorkQueueElement.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+CouchWorkQueueElement.py
+
+Created by Dave Evans on 2010-10-12.
+Copyright (c) 2010 Fermilab. All rights reserved.
+"""
+
+from future.utils import viewitems
+
+import unittest
+import time
+import logging
+
+from WMCore.Database.CMSCouch import Document
+from WMCore.WorkQueue.DataStructs.WorkQueueElement import WorkQueueElement
+
+
+
+class CouchWorkQueueElement(WorkQueueElement):
+    """
+    _CouchWorkQueueElement_
+
+    """
+    def __init__(self, couchDB, id = None, elementParams = None):
+        elementParams = elementParams or {}
+        WorkQueueElement.__init__(self, **elementParams)
+        if id:
+            self._id = id
+        self._document = Document(id = id)
+        self._couch = couchDB
+
+    rev = property(
+        lambda x: str(x._document[u'_rev']) if u'_rev' in x._document else x._document.__getitem__('_rev'),
+        lambda x, newid: x._document.__setitem__('_rev', newid))
+    timestamp = property(
+        lambda x: str(x._document[u'timestamp']) if u'timestamp' in x._document else x._document.__getitem__('timestamp')
+        )
+    updatetime = property(
+        lambda x: str(x._document[u'updatetime']) if u'updatetime' in x._document else 0
+        )
+
+
+    @classmethod
+    def fromDocument(cls, couchDB, doc):
+        """Create element from couch document"""
+        elementParams = doc.pop('WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement')
+        elementParams["CreationTime"] = doc.pop('timestamp')
+        element = CouchWorkQueueElement(couchDB = couchDB,
+                                        id = doc['_id'],
+                                        elementParams = elementParams)
+        element._document['_rev'] = doc.pop('_rev')
+        element._document['timestamp'] = elementParams["CreationTime"]
+        element._document['updatetime'] = doc.pop('updatetime')
+        return element
+
+    def save(self):
+        """
+        _save
+        """
+        self.populateDocument()
+        self._couch.queue(self._document)
+
+    def load(self):
+        """
+        _load_
+
+        Load the document representing this WQE
+        """
+        document = self._couch.document(self._document['_id'])
+        self.update(document.pop('WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'))
+        self._document['_rev'] = document.pop('_rev')
+        self._document['timestamp'] = document.pop('timestamp', None)
+        self._document['updatetime'] = document.pop('updatetime', None)
+        return self
+
+    def delete(self):
+        """Delete element"""
+        self.populateDocument()
+        self._document.delete()
+        self._couch.queue(self._document)
+
+    def populateDocument(self):
+        """Certain attributed shouldn't be stored"""
+        self._document.update(self.__to_json__(None))
+        now = time.time()
+        self._document['updatetime'] = now
+        self._document.setdefault('timestamp', now)
+        if not self._document.get('_id') and self.id:
+            self._document['_id'] = self.id
+        attrs = ['WMSpec', 'Task']
+        for attr in attrs:
+            self._document['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'].pop(attr, None)
+
+
+
+def fixElementConflicts(*elements):
+    """Take conflicting elements, fix conflicts and return
+    First element returned will contain the merged results,
+    the others will be losing revisions
+    """
+    ordered_states = ['Available', 'Negotiating', 'Acquired', 'Running',
+                      'Done', 'Failed', 'CancelRequested', 'Canceled']
+    allowed_keys = ['Status', 'EventsWritten', 'FilesProcessed',
+                    'PercentComplete', 'PercentSuccess', 'Inputs', 'NumOfFilesAdded',
+                    'SubscriptionId', 'Priority', 'SiteWhitelist', 'SiteBlacklist']
+    merged_value = None
+    updated = set()
+    for ele in elements:
+        if not merged_value: # 1st will contain merged result and become winner
+            merged_value = ele
+            continue
+
+        # print differences
+        from WMCore.Algorithms.MiscAlgos import dict_diff
+        logging.info("Conflict between %s revs %s & %s: %s",
+                         ele.id, merged_value.rev, ele.rev,
+                         "; ".join("%s=%s" % (x,y) for x,y in viewitems(dict_diff(merged_value, ele)))
+                    )
+        for key in merged_value:
+            if merged_value[key] == ele.get(key):
+                continue
+            # we need to merge: Take elements from both that seem most advanced, e.g. status & progress stats
+            if key not in allowed_keys:
+                msg = 'Unable to merge conflicting element %s: field "%s" value 1 "%s" value2 "%s"'
+                raise RuntimeError(msg % (ele.id, key, merged_value.get(key), ele.get(key)))
+            elif key == 'Status':
+                if ordered_states.index(ele[key]) > ordered_states.index(merged_value[key]):
+                    merged_value[key] = ele[key]
+            elif key == 'Inputs':
+                for item in merged_value[key]:
+                    # take larger locations list
+                    if merged_value[key][item] < ele[key].get(item, []):
+                        merged_value[key][item] = ele[key][item]
+            elif ele[key] is not None and ele[key] > merged_value[key]:
+                merged_value[key] = ele[key]
+            updated.add(key)
+        # once losing element has been merged - queue for deletion
+        ele._document.delete()
+
+    msg = 'Resolving conflict for wf "%s", id "%s": Remove rev(s): %s: Updates: (%s)'
+    logging.info(msg, str(merged_value['RequestName']),
+                 str(merged_value.id),
+                 ", ".join([x._document['_rev'] for x in elements[1:]]),
+                 "; ".join("%s=%s" % (x, merged_value[x]) for x in updated)
+                 )
+    return elements

WMCore/WorkQueue/DataStructs/WorkQueueElement.py

@@ -0,0 +1,274 @@
+"""
+WorkQueueElement
+
+A dictionary based object meant to represent a WorkQueue element
+"""
+
+from future.utils import viewitems, viewvalues
+import time
+from hashlib import md5
+
+from Utils.Utilities import encodeUnicodeToBytesConditional
+from Utils.PythonVersion import PY3
+
+STATES = ('Available', 'Negotiating', 'Acquired', 'Running',
+          'Done', 'Failed', 'CancelRequested', 'Canceled')
+
+
+def possibleSites(element):
+    """
+    _possibleSites_
+
+    Checks the site and data restrictions and return a list of possible sites
+    to work on this element.
+    """
+    # check if the whole document was provide
+    elem = element.get('WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement', element)
+
+    if elem['NoInputUpdate'] and elem['NoPileupUpdate']:
+        return elem['SiteWhitelist']
+
+    commonSites = set(elem['SiteWhitelist']) - set(elem['SiteBlacklist'])
+
+    if elem['Inputs'] and elem['NoInputUpdate'] is False:
+        commonSites = commonSites.intersection(set([y for x in viewvalues(elem['Inputs']) for y in x]))
+    if elem['ParentFlag'] and elem['NoInputUpdate'] is False:
+        commonSites = commonSites.intersection(set([y for x in viewvalues(elem['ParentData']) for y in x]))
+    if elem['PileupData'] and elem['NoPileupUpdate'] is False:
+        commonSites = commonSites.intersection(set([y for x in viewvalues(elem['PileupData']) for y in x]))
+
+    return list(commonSites)
+
+
+class WorkQueueElement(dict):
+    """Class to represent a WorkQueue element"""
+
+    def __init__(self, **kwargs):
+        dict.__init__(self)
+
+        if 'Status' in kwargs and kwargs['Status'] not in STATES:
+            msg = 'Invalid Status: %s' % kwargs['Status']
+            raise ValueError(msg)
+
+        self.update(kwargs)
+
+        self._id = None
+
+        # XXX If adding or modifying any new parameter which affects the
+        # XXX workflow or data run over, the id function must be updated
+
+        self.setdefault('Inputs', {})
+        self.setdefault('ProcessedInputs', [])
+        self.setdefault('RejectedInputs', [])
+        # Some workflows require additional datasets for PileUp
+        # Track their locations
+        self.setdefault('PileupData', {})
+        # both ParentData and ParentFlag is needed in case there Dataset split,
+        # even though ParentFlag is True it will have empty ParentData
+        self.setdefault('ParentData', {})
+        self.setdefault('ParentFlag', False)
+        # 0 jobs are valid where we need to accept all blocks (only dqm etc subscriptions will run)
+        self.setdefault('Jobs', 0)
+        self.setdefault('WMSpec', None)
+        self.setdefault('SiteWhitelist', [])
+        self.setdefault('SiteBlacklist', [])
+        self.setdefault('Dbs', None)
+        self.setdefault('Task', None)
+        self.setdefault('ParentQueueId', None)
+        self.setdefault('Priority', 0)
+        self.setdefault('SubscriptionId', None)
+        self.setdefault('Status', None)
+        self.setdefault('EventsWritten', 0)
+        self.setdefault('FilesProcessed', 0)
+        self.setdefault('PercentComplete', 0)
+        self.setdefault('PercentSuccess', 0)
+        self.setdefault('RequestName', None)
+        self.setdefault('TaskName', None)
+        self.setdefault('TeamName', None)
+        self.setdefault('StartPolicy', {})
+        self.setdefault('EndPolicy', {})
+        self.setdefault('ACDC', {})
+        self.setdefault('ChildQueueUrl', None)
+        self.setdefault('ParentQueueUrl', None)
+        self.setdefault('WMBSUrl', None)
+        self.setdefault('NumberOfLumis', 0)
+        self.setdefault('NumberOfEvents', 0)
+        self.setdefault('NumberOfFiles', 0)
+        # Number of files added to WMBS including parent files for this element. used only for monitoring purpose
+        self.setdefault('NumOfFilesAdded', 0)
+        # Mask used to constrain MC run/lumi ranges
+        self.setdefault('Mask', None)
+        # is new data being added to the workflow (e.g., growing input datasets)
+        self.setdefault('OpenForNewData', False)
+        # EPOCH time for when the last block was found and added to the workflow
+        self.setdefault('TimestampFoundNewData', int(time.time()))
+        # Trust initial input dataset location only or not
+        self.setdefault('NoInputUpdate', False)
+        # Trust initial pileup dataset location only or not
+        self.setdefault('NoPileupUpdate', False)
+        # EPOCH time with the creation time for this wq element, needed for sorting
+        self.setdefault('CreationTime', int(time.time()))
+        # set to true when updated from a WorkQueueElementResult
+        self.modified = False
+
+    def __to_json__(self, thunker):
+        """Strip unthunkable"""
+        # result = WorkQueueElement(thunker_encoded_json = True,
+        result = dict(thunker_encoded_json=True,
+                      type='WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement')
+        result['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'] = {}
+        result['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'].update(self)
+        # Do we need this 'Subscription' or can we not store this at all?
+        result['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'].pop('Subscription', None)
+        result['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'].pop('WMSpec', None)
+        #        if self.get('Id'):
+        #            result['_id'] = result['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'].pop('Id')
+        #        if self.get('_rev'):
+        #            result['_rev'] = result['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'].pop('_rev')
+        # result['Mask'] = thunker._thunk(result['Mask'])
+        return result
+
+    @property
+    def id(self):
+        """Generate id for element
+
+        id is deterministic and can be used to identify duplicate elements.
+        Calculation only includes fields which affect the workflow and input data.
+        Result is an md5 hash of a ';' separated list of:
+        workflow name, task name, list of inputs, mask, ACDC info, Dbs instance.
+
+        Parent file info not accounted.
+
+        Example:
+        >>> WorkQueueElement(RequestName = 'a', TaskName = 'b').id
+        '9ef03a6ad8f16d74fb5ba44df92bf1ef'
+
+        Warning: Any change to this function may prevent identical existing and
+        new elements from appearing equivalent, thus in the case of expanding
+        work subscriptions work duplication can occur. Care must be taken
+        if any modification is made.
+        """
+        if self._id:
+            return self._id
+        # Assume md5 is good enough for now
+        myhash = md5()
+        spacer = ';'  # character not present in any field
+        myhash.update(encodeUnicodeToBytesConditional(self['RequestName'] + spacer, condition=PY3))
+        # Task will be None in global inbox
+        myhash.update(encodeUnicodeToBytesConditional(repr(self['TaskName']) + spacer, condition=PY3))
+        myhash.update(encodeUnicodeToBytesConditional(",".join(sorted(self['Inputs'].keys())) + spacer, condition=PY3))
+        # Check repr is reproducible - should be
+        if self['Mask']:
+            myhash.update(encodeUnicodeToBytesConditional(",".join(["%s=%s" % (x, y) for x, y in viewitems(self['Mask'])]) + spacer, condition=PY3))
+        else:
+            myhash.update(encodeUnicodeToBytesConditional("None" + spacer, condition=PY3))
+        # Check ACDC is deterministic and all params relevant
+        myhash.update(encodeUnicodeToBytesConditional(",".join(["%s=%s" % (x, y) for x, y in viewitems(self['ACDC'])]) + spacer, condition=PY3))
+        myhash.update(encodeUnicodeToBytesConditional(repr(self['Dbs']) + spacer, condition=PY3))
+        self._id = myhash.hexdigest()
+        return self._id
+
+    @id.setter
+    def id(self, value):
+        """Set id - use to override built-in id calculation"""
+        self._id = value
+
+    def __from_json__(self, jsondata, thunker):
+        """"""
+        self.update(jsondata)
+        return self
+
+    #        self.update(jsondata['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'])
+    #        self.pop('type', None)
+    #        self.pop('thunker_encoded_json', None)
+    ##        self['Id'] = jsondata['_id']
+    #        self['_rev'] = jsondata['_rev'] # what to do here???
+    #        return self
+
+    def inEndState(self):
+        """Have we finished processing"""
+        return self.isComplete() or self.isFailed() or self.isCanceled()
+
+    def isComplete(self):
+        return self['Status'] == 'Done'
+
+    def isFailed(self):
+        return self['Status'] == 'Failed'
+
+    def isRunning(self):
+        return self['Status'] == 'Running'
+
+    def isAcquired(self):
+        return self['Status'] == 'Acquired'
+
+    def isAvailable(self):
+        return self['Status'] == 'Available'
+
+    def isCanceled(self):
+        return self['Status'] == 'Canceled'
+
+    def isCancelRequested(self):
+        return self['Status'] == 'CancelRequested'
+
+    def updateFromSubscription(self, wmbsStatus):
+        """Get subscription status"""
+        mapping = {'EventsWritten': 'events_written',
+                   'FilesProcessed': 'files_processed',
+                   'PercentComplete': 'percent_complete',
+                   'PercentSuccess': 'percent_success'}
+        for ourkey, wmbskey in viewitems(mapping):
+            if wmbskey in wmbsStatus and self[ourkey] != wmbsStatus[wmbskey]:
+                self['Modified'] = True
+                self[ourkey] = wmbsStatus[wmbskey]
+
+    def updateWithResult(self, progressReport):
+        """Take a progress report and update ourself
+           Return True if progress updated"""
+        progressValues = ('Status', 'EventsWritten', 'FilesProcessed',
+                          'PercentComplete', 'PercentSuccess')
+        modified = False
+        for val in progressValues:
+            # ignore if values the same
+            if self[val] == progressReport[val]:
+                continue
+            # ignore new state if it is lower than the current state
+            if val == 'Status' and self[val] and STATES.index(progressReport[val]) <= STATES.index(self[val]):
+                continue
+
+            self[val] = progressReport[val]
+            modified = True
+        self.modified = modified
+
+    def statusMetrics(self):
+        """Returns the status & performance metrics"""
+        return dict(Status=self['Status'],
+                    PercentComplete=self['PercentComplete'],
+                    PercentSuccess=self['PercentSuccess'])
+
+    def passesSiteRestriction(self, site):
+        """
+        _passesSiteRestriction_
+
+        Given a site name, checks whether it passes the site and data restrictions.
+        """
+        # Site list restrictions
+        if site in self['SiteBlacklist'] or site not in self['SiteWhitelist']:
+            return False
+
+        # input data restrictions (TrustSitelists flag)
+        if self['NoInputUpdate'] is False:
+            for locations in viewvalues(self['Inputs']):
+                if site not in locations:
+                    return False
+            if self['ParentFlag']:
+                for locations in viewvalues(self['ParentData']):
+                    if site not in locations:
+                        return False
+
+        # pileup data restrictions (TrustPUSitelists flag)
+        if self['NoPileupUpdate'] is False:
+            for locations in viewvalues(self['PileupData']):
+                if site not in locations:
+                    return False
+
+        return True

WMCore/WorkQueue/DataStructs/WorkQueueElementResult.py

@@ -0,0 +1,152 @@
+"""
+WorkQueueElementResult
+
+A dictionary based object meant to represent a WorkQueue block
+"""
+from __future__ import division
+
+#Can we re-use WorkQueueElement for this?
+
+
+
+
+class WorkQueueElementResult(dict):
+    """Class to hold the status of a related group of WorkQueueElements"""
+    def __init__(self, **kwargs):
+        dict.__init__(self)
+        self.update(kwargs)
+
+        self.setdefault('WMSpec', None)
+        self.setdefault('Elements', [])
+        self.setdefault('Status', self.status())
+        if self['Elements']:
+            self.setdefault('EventsWritten',
+                            sum([x['EventsWritten'] for x in self['Elements']]))
+            self.setdefault('FilesProcessed',
+                            sum([x['FilesProcessed'] for x in self['Elements']]))
+            self.setdefault('Jobs', sum([x['Jobs'] for x in self['Elements']]))
+            self.setdefault('PercentComplete',
+                            int(sum([x['PercentComplete'] for x in self['Elements']],
+                                0.0) / len(self['Elements'])))
+            self.setdefault('PercentSuccess',
+                            int(sum([x['PercentSuccess'] for x in self['Elements']],
+                                0.0) / len(self['Elements'])))
+            self.setdefault('RequestName', self['Elements'][0]['RequestName'])
+            self.setdefault('TeamName', self['Elements'][0]['TeamName'])
+            self.setdefault('Priority', self['Elements'][0]['Priority'])
+            self.setdefault('ParentQueueId', self['Elements'][0]['ParentQueueId'])
+        elif self.get('ParentQueueElement'):
+            self.setdefault('EventsWritten', self['ParentQueueElement']['EventsWritten'])
+            self.setdefault('FilesProcessed', self['ParentQueueElement']['FilesProcessed'])
+            self.setdefault('Jobs', self['ParentQueueElement']['Jobs'])
+            self.setdefault('PercentComplete', self['ParentQueueElement']['PercentComplete'])
+            self.setdefault('PercentSuccess', self['ParentQueueElement']['PercentSuccess'])
+            self.setdefault('RequestName', self['ParentQueueElement']['RequestName'])
+            self.setdefault('TeamName', self['ParentQueueElement']['TeamName'])
+            self.setdefault('Priority', self['ParentQueueElement']['Priority'])
+            self.setdefault('ParentQueueId', self['ParentQueueElement'].id)
+        else:
+            raise RuntimeError("Can create WQEResult: No elements or parent provided")
+
+        # some cross checks
+        for i in self['Elements']:
+            assert(i['RequestName'] == self['RequestName'])
+            assert(i['TeamName'] == self['TeamName'])
+
+    def fractionComplete(self):
+        """Return fraction successful"""
+        return len(self.completeItems()) / float(len(self['Elements']))
+
+    def completeItems(self):
+        """Return complete items"""
+        return [x for x in self['Elements'] if x.isComplete()]
+
+    def failedItems(self):
+        """Return failed items"""
+        return [x for x in self['Elements'] if x.isFailed()]
+
+    def runningItems(self):
+        """Return acquired items"""
+        return [x for x in self['Elements'] if x.isRunning()]
+
+    def acquiredItems(self):
+        """Return available items"""
+        return [x for x in self['Elements'] if x.isAcquired()]
+
+    def availableItems(self):
+        """Return available items"""
+        return [x for x in self['Elements'] if x.isAvailable()]
+
+    def canceledItems(self):
+        """Return canceled items"""
+        return [x for x in self['Elements'] if x.isCanceled()]
+
+    def cancelRequestedItems(self):
+        """Return CancelRequested Items"""
+        return [x for x in self['Elements'] if x.isCancelRequested()]
+
+    def status(self):
+        """Compute status of elements"""
+        if not self['Elements'] and self.get('ParentQueueElement'):
+            return self['ParentQueueElement']['Status']
+        elif not self['Elements']:
+            return None
+
+        if not self.inEndState():
+            if self.cancelRequestedItems():
+                return 'CancelRequested'
+            elif self.runningItems():
+                return 'Running'
+            elif self.acquiredItems() or self.availableItems():
+                # available in local queue is acquired to parent
+                return 'Acquired'
+        else:
+            if self.canceledItems():
+                return "Canceled"
+            elif self.failedItems():
+                return "Failed"
+
+        # if all elements have same status take that
+        status = self['Elements'][0]['Status']
+        for element in self['Elements']:
+            if element['Status'] != status:
+                msg = "Unable to compute overall status of elements: %s"
+                raise RuntimeError(msg % str(self['Elements']))
+        return status
+
+    def inEndState(self):
+        """Are all requests complete (either success or failure)"""
+        if 'Status' in self:
+            return self['Status'] in ('Done', 'Failed', 'Canceled')
+        for element in self['Elements']:
+            if not element.inEndState():
+                return False
+        return True
+
+    def statusMetrics(self):
+        """Returns the status & performance metrics"""
+        keys = ['Status', 'PercentComplete', 'PercentSuccess']
+        return self.fromkeys(keys)
+
+    def formatForWire(self):
+        """Format used to send data over network
+        """
+        to_remove = ['Elements', 'WMSpec']
+        result = dict(self)
+        for item in to_remove:
+            result.pop(item)
+        return result
+
+    def getMaxJobElement(self):
+        maxJobElement = self['Elements'][0]
+        for x in self['Elements']:
+            if x['Jobs'] > maxJobElement['Jobs']:
+                maxJobElement = x
+        return maxJobElement
+
+    def getMinJobElement(self):
+        minJobElement = self['Elements'][0]
+        for x in self['Elements']:
+            if x['Jobs'] < minJobElement['Jobs']:
+                minJobElement = x
+        return minJobElement