wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/DataStructs/File.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+"""
+_File_
+
+Data object that contains details for a single file
+
+"""
+from builtins import str, bytes
+__all__ = []
+
+from WMCore.DataStructs.Run import Run
+from WMCore.DataStructs.WMObject import WMObject
+
+
+class File(WMObject, dict):
+    """
+    _File_
+    Data object that contains details for a single file
+
+    TODO 
+    - use the decorator `from functools import total_ordering` after
+      dropping support for python 2.6
+    - then, drop __ne__, __le__, __gt__, __ge__
+    """
+
+    def __init__(self, lfn="", size=0, events=0, checksums=None,
+                 parents=None, locations=None, merged=False):
+        dict.__init__(self)
+        checksums = checksums or {}
+        self.setdefault("lfn", lfn)
+        self.setdefault("size", size)
+        self.setdefault("events", events)
+        self.setdefault("checksums", checksums)
+        self.setdefault('runs', set())
+        self.setdefault('merged', merged)
+        self.setdefault('last_event', 0)
+        self.setdefault('first_event', 0)
+        if locations is None:
+            self.setdefault("locations", set())
+        else:
+            self.setdefault("locations", locations)
+
+        if parents is None:
+            self.setdefault("parents", set())
+        else:
+            self.setdefault("parents", parents)
+
+    def addRun(self, run):
+        """
+        _addRun_
+
+        run should be an instance of WMCore.DataStructs.Run
+
+        Add a run container to this file, tweak the run and lumi
+        keys to be max run and max lumi for backwards compat.
+
+
+        """
+
+        if not isinstance(run, Run):
+            msg = "addRun argument must be of type WMCore.DataStructs.Run"
+            raise RuntimeError(msg)
+
+        addFlag = False
+        for runMember in self['runs']:
+            if runMember.run == run.run:
+                # this rely on Run object overwrite __add__ to update self
+                runMember + run
+                addFlag = True
+
+        if not addFlag:
+            self['runs'].add(run)
+        return
+
+    def load(self):
+        """
+        A DataStructs file has nothing to load from, other implementations will
+        over-ride this method.
+        """
+        if self['id']:
+            self['lfn'] = '/store/testing/%s' % self['id']
+
+    def save(self):
+        """
+        A DataStructs file has nothing to save to, other implementations will
+        over-ride this method.
+        """
+        pass
+
+    def setLocation(self, pnn):
+        # Make sure we don't add None, [], "" as file location
+        if pnn:
+            self['locations'] = self['locations'] | set(self.makelist(pnn))
+
+    def __eq__(self, rhs):
+        """
+        File is equal if it has the same name
+        """
+        eq = False
+        if isinstance(rhs, type(self)):
+            eq = self['lfn'] == rhs['lfn']
+        elif isinstance(rhs, (str, bytes)):
+            eq = self['lfn'] == rhs
+        return eq
+
+    def __ne__(self, rhs):
+        return not self.__eq__(rhs)
+
+    def __hash__(self):
+        thisHash = self['lfn'].__hash__()
+        return thisHash
+
+    def __lt__(self, rhs):
+        """
+        Sort files based on lexicographical ordering of the value connected
+        to the 'lfn' key
+        """
+        eq = False
+        if isinstance(rhs, type(self)):
+            eq = self['lfn'] < rhs['lfn']
+        elif isinstance(rhs, (str, bytes)):
+            eq = self['lfn'] < rhs
+        return eq
+
+    def __le__(self, other):
+        return self.__lt__(other) or self.__eq__(other)
+
+    def __gt__(self, other):
+        return not self.__le__(other)
+
+    def __ge__(self, other):
+        return not self.__lt__(other)
+
+    def json(self, thunker=None):
+        """
+        _json_
+
+        Serialize the file object.  This will convert all Sets() to lists and
+        weed out the internal data structures that don't need to be shared.
+        """
+        fileDict = {"last_event": self["last_event"],
+                    "first_event": self["first_event"],
+                    "lfn": self["lfn"],
+                    "locations": list(self["locations"]),
+                    "id": self.get("id", None),
+                    "checksums": self["checksums"],
+                    "events": self["events"],
+                    "merged": self["merged"],
+                    "size": self["size"],
+                    "runs": [],
+                    "parents": []}
+
+        for parent in self["parents"]:
+            if isinstance(parent, (str, bytes)):
+                # Then for some reason, we're passing strings
+                # Done specifically for ErrorHandler
+                fileDict['parents'].append(parent)
+            elif thunker is None:
+                continue
+            else:
+                fileDict["parents"].append(thunker._thunk(parent))
+
+        for run in self["runs"]:
+            runDict = {"run_number": run.run,
+                       "lumis": run.lumis}
+            fileDict["runs"].append(runDict)
+
+        return fileDict
+
+    def __to_json__(self, thunker=None):
+        """
+        __to_json__
+
+        This is the standard way we jsonize other objects.
+        Included here so we have a uniform method.
+        """
+        return self.json(thunker)

WMCore/DataStructs/Fileset.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+"""
+_Fileset_
+
+Data object that contains a set of files
+
+"""
+
+from __future__ import print_function
+from builtins import str, map
+
+from operator import itemgetter
+from WMCore.DataStructs.WMObject import WMObject
+__all__ = []
+
+
+class Fileset(WMObject):
+    """
+    _Fileset_
+    Data object that contains a set of files
+    """
+    def __init__(self, name=None, files=None):
+        """
+        Assume input files are new
+        """
+        self.name = name
+        self.files = set()
+
+        if files is None:
+            self.newfiles = set()
+        else:
+            self.newfiles = files
+
+        # assume that the fileset is open at first
+        self.open = True
+
+        # assume that the lastUpdate of fileset is 0 at first
+        self.lastUpdate = 0
+
+    def setLastUpdate(self, timeUpdate):
+        """
+        _setLastUpdate_
+
+        Change the last update time of this fileset.  The lastUpdate parameter is a int
+        representing the last time where the fileset was modifed.
+        """
+        self.lastUpdate = timeUpdate
+
+    def addFile(self, file):
+        """
+        Add a (set of) file(s) to the fileset
+        If the file is already in self.files update that entry
+            e.g. to handle updated location
+        If the file is already in self.newfiles update that entry
+            e.g. to handle updated location
+        Else add the file to self.newfiles
+        """
+        file = self.makeset(file)
+        new = file - self.getFiles(type='set')
+        self.newfiles = self.makeset(self.newfiles) | new
+
+        updated = self.makeset(file) & self.getFiles(type='set')
+        "updated contains the original location information for updated files"
+
+        self.files = self.files.union(updated)
+
+    def getFiles(self, type='list'):
+        if type == 'list':
+            """
+            List all files in the fileset - returns a set of file objects
+            sorted by lfn.
+            """
+            files = list(self.getFiles(type='set'))
+
+            try:
+                files.sort(key=itemgetter('lfn'))
+            except Exception as e:
+                print('Problem with listFiles for fileset:', self.name)
+                print(files.pop())
+                raise e
+            return files
+        elif type == 'set':
+            return self.makeset(self.files) | self.makeset(self.newfiles)
+        elif type == 'lfn':
+            """
+            All the lfn's for files in the filesets
+            """
+            def getLFN(file):
+                return file["lfn"]
+            files = list(map(getLFN, self.getFiles(type='list')))
+            return files
+        elif type == 'id':
+            """
+            All the id's for files in the filesets
+            """
+            def getID(file):
+                return file["id"]
+
+            files = list(map(getID, self.getFiles(type='list')))
+            return files
+
+    def listNewFiles(self):
+        """
+        List all files in the fileset that are new - e.g. not in the DB - returns a set
+        """
+        return self.newfiles
+
+    def commit(self):
+        """
+        Add contents of self.newfiles to self, empty self.newfiles
+        """
+        self.files = self.makeset(self.files) | self.makeset(self.newfiles)
+        self.newfiles = set()
+
+    def __len__(self):
+        return len(self.getFiles(type='set'))
+
+    def __iter__(self):
+        for file in self.getFiles():
+            yield file
+
+    def markOpen(self, isOpen):
+        """
+        _markOpen_
+
+        Change the open status of this fileset.  The isOpen parameter is a bool
+        representing whether or not the fileset is open.
+        """
+        self.open = isOpen
+
+    def __str__(self):
+        """
+        __str__
+
+        Write out something useful
+        """
+
+        st = {'name': self.name, 'files': self.files}
+
+        return str(st)

WMCore/DataStructs/Job.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+"""
+_Job_
+
+Data object that describes a job
+
+"""
+
+from builtins import map, range
+
+__all__ = []
+
+
+from WMCore.DataStructs.Mask import Mask
+from WMCore.DataStructs.WMObject import WMObject
+from WMCore.Configuration import ConfigSection
+
+import time
+
+
+class Job(WMObject, dict):
+    def __init__(self, name = None, files = None):
+        """
+        A job has a jobgroup which gives it its subscription and workflow.
+        inputFiles is a list containing files associated to a job
+        last_update is the time the job last changed
+        """
+        dict.__init__(self)
+        self.baggage = ConfigSection("baggage")
+        if files == None:
+            self["input_files"] = []
+        else:
+            self["input_files"] = files
+
+        self["id"] = None
+        self["jobgroup"] = None
+        self["name"] = name
+        self["state"] = 'new'
+        self["state_time"] = int(time.time())
+        self["outcome"] = "failure"
+        self["retry_count"] = 0
+        self["location"] = None
+        self["mask"] = Mask()
+        self["task"] = None
+        self["fwjr"] = None
+        self["fwjr_path"] = None
+        self["workflow"] = None
+        self["owner"] = None
+        self["estimatedJobTime"] = None
+        self["estimatedMemoryUsage"] = None
+        self["estimatedDiskUsage"] = None
+        return
+
+    #  //
+    # // Use property to define getter/setter API for task
+    #//  makes job.task and job.tast = "value" possible
+    task = property(lambda self: self['task'],
+                    lambda self, x: self.__setitem__('task',x))
+
+    def save(self):
+        """
+        _save_
+
+        Bogus method to make this look more like the WMBS JOB.
+        """
+        return
+
+    def getFiles(self, type = "list"):
+        """
+        _getFiles_
+
+        Retrieve information about the input files for the job.  The type
+        parameter can be set to one of the following:
+          list - A list of File objects will be returned
+          set - A set of File objects will be returned
+          lfn - A list of LFNs will be returned
+          id - A list if File IDs will be returned
+        """
+        if type == "list":
+            return self["input_files"]
+        elif type == "set":
+            return self.makeset(self["input_files"])
+        elif type == "lfn":
+            def getLFN(file):
+                return file["lfn"]
+
+            lfns = list(map(getLFN, self["input_files"]))
+            return lfns
+        elif type == "id":
+            def getID(file):
+                return file["id"]
+
+            ids = list(map(getID, self["input_files"]))
+            return ids
+
+    def addFile(self, file):
+        """
+        _addFile_
+
+        Add a file or list of files to the job's input.
+        """
+        if isinstance(file, list):
+            self["input_files"].extend(file)
+        else:
+            self["input_files"].append(file)
+
+        return
+
+    def changeState(self, newState):
+        """
+        _changeState_
+
+        Change the state of the job.
+        """
+        self["state"] = newState
+        self["state_time"] = int(time.time())
+
+        return
+
+    def changeOutcome(self, jobOutcome):
+        """
+        _changeOutcome_
+
+        Change the final outcome of the job, can be either success or fail.
+        """
+        self["outcome"] = jobOutcome
+        return
+
+    def addResourceEstimates(self, jobTime = None, memory = None, disk = None):
+        """
+        _addResourceEstimates_
+
+        Add to the current resource estimates, if None then initialize them
+        to the given value. Each value can be set independently.
+        """
+        if not self["estimatedJobTime"]:
+            self["estimatedJobTime"] = jobTime
+        elif jobTime:
+            self["estimatedJobTime"] += jobTime
+
+        if not self["estimatedMemoryUsage"]:
+            self["estimatedMemoryUsage"] = memory
+        elif memory:
+            self["estimatedMemoryUsage"] += memory
+
+        if not self["estimatedDiskUsage"]:
+            self["estimatedDiskUsage"] = disk
+        elif disk:
+            self["estimatedDiskUsage"] += disk
+
+        return
+
+    def getBaggage(self):
+        """
+        _getBaggage_
+
+        Get a reference to the embedded ConfigSection that is
+        used to store per job arguments needed to be propagated at
+        submission time
+
+        """
+        return self.baggage
+
+
+    def addBaggageParameter(self, attrName, value):
+        """
+        _addBaggageParameter_
+
+        Add an attribute as process.pset1.pset2.param = value
+        Value should be the appropriate python type
+
+        """
+        currentPSet = self.baggage
+        paramList = attrName.split(".")
+        for i in range(0, len(paramList)):
+            param = paramList.pop(0)
+            if len(paramList) > 0:
+                if not hasattr(currentPSet, param):
+                    currentPSet.section_(param)
+                currentPSet = getattr(currentPSet, param)
+            else:
+                setattr(currentPSet, param, value)

WMCore/DataStructs/JobGroup.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+"""
+_JobGroup_
+
+Definition of JobGroup:
+    Set of jobs running on same input file for same Workflow
+    Set of jobs for a single subscription
+    Required for certain job splitting Algo's (.g. event split to make complete
+    lumi)
+    Subscription:JobGroup == 1:N
+    JobGroup:Jobs = 1:N
+    JobGroup:InFile = 1:1
+    JobGroup:MergedOutFile = N:1
+    JobGroup at least one Lumi section
+
+A JobGroup is a set of jobs and a Fileset that contains their output.
+
+JobGroup knows the Subscription and passes the Workflow to Jobs in the group.
+
+Jobs know their status (active, failed, complete) and know the files they run
+on but don't know the group. They do know their subscription and corresponding
+workflow. This means Jobs can update their state in the database without
+talking to the group, and WMBS JobGroups can calculate status from the database
+instead of the in memory objects.
+
+The group has a status call which goes through the jobs and updates the db for
+state changes and then returns the status of the group (active, failed,
+complete).
+
+WMAgent deals with groups and calls group.status periodically
+"""
+from __future__ import print_function
+
+import datetime
+
+from WMCore.DataStructs.Fileset import Fileset
+from WMCore.DataStructs.WMObject import WMObject
+
+
+class JobGroup(WMObject):
+    """
+    JobGroups are sets of jobs running on files who's output needs to be merged
+    together.
+    """
+
+    def __init__(self, subscription=None, jobs=None):
+        self.jobs = []
+        self.newjobs = []
+        self.id = 0
+
+        if isinstance(jobs, list):
+            self.newjobs = jobs
+        elif jobs is not None:
+            self.newjobs = [jobs]
+
+        self.subscription = subscription
+        self.output = Fileset()
+        self.last_update = datetime.datetime.now()
+
+    def add(self, job):
+        """
+        _add_
+
+        Add a Job or list of jobs to the JobGroup.
+        """
+        jobList = self.makelist(job)
+        self.newjobs.extend(jobList)
+        return
+
+    def commit(self):
+        """
+        _commit_
+
+        Move any jobs in the newjobs dict to the job dict.  Empty the newjobs
+        dict.
+        """
+        self.jobs.extend(self.newjobs)
+        self.newjobs = []
+
+    def commitBulk(self):
+        """
+        Dummy method for consistency with WMBS implementation
+        """
+        self.commit()
+
+    def addOutput(self, file):
+        """
+        _addOutput_
+
+        Add a File to the JobGroup's output fileset.  The File is committed
+        to the Fileset immediately.
+        """
+        self.output.addFile(file)
+        self.output.commit()
+
+    def getJobs(self, type="list"):
+        """
+        _getJobs_
+
+        Retrieve all of the jobs in the JobGroup.  The output will either be
+        returned as a list of Job objects (when type is "list") or a list of
+        Job IDs (when type is "id").
+        """
+        if type == "list":
+            return self.jobs
+        elif type == "id":
+            jobIDs = []
+
+            for job in self.jobs:
+                jobIDs.append(job["id"])
+
+            return jobIDs
+        else:
+            print("Unknown type: %s" % type)
+
+        return
+
+    def getOutput(self, type="list"):
+        """
+        _getOutput_
+
+        Retrieve all of the files that are in the JobGroup's output fileset.
+        Type can be one of the following: list, set, lfn, id.
+        """
+        return self.output.getFiles(type=type)
+
+    def getLength(self, obj):
+        """
+        This just gets a length for either dict or list objects
+        """
+        if isinstance(obj, (dict, list)):
+            return len(obj)
+        else:
+            return 0
+
+    def __len__(self):
+        """
+        Allows use of len() on JobGroup
+        """
+        return self.getLength(self.jobs) + self.getLength(self.newjobs)
+
+        # return len(self.jobs.keys()) + len(self.newjobs.keys())

WMCore/DataStructs/JobPackage.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""
+_JobPackage_
+
+Data structure for storing and retreiving multiple job objects.
+"""
+import pickle
+from Utils.PythonVersion import HIGHEST_PICKLE_PROTOCOL
+from WMCore.DataStructs.WMObject import WMObject
+
+
+class JobPackage(WMObject, dict):
+    """
+    _JobPackage_
+    """
+
+    def __init__(self, directory=None):
+        """
+        __init__
+
+        Allow you to set a directory where the
+        package will be stored.
+        This is then saved for use by the
+        JobSubmitter
+        """
+        dict.__init__(self)
+        self.setdefault('directory', directory)
+
+    def save(self, fileName):
+        """
+        _save_
+
+        Pickle this object and save it to disk.
+        """
+        with open(fileName, 'wb') as fileHandle:
+            pickle.dump(self, fileHandle, protocol=HIGHEST_PICKLE_PROTOCOL)
+        return
+
+    def load(self, fileName):
+        """
+        _load_
+
+        Load a pickled JobPackage object.
+        """
+        self.clear()
+        with open(fileName, 'rb') as fileHandle:
+            loadedJobPackage = pickle.load(fileHandle)
+        self.update(loadedJobPackage)
+        return