wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/WMSpec/Steps/BuildTools.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+"""
+_BuildTools_
+
+Utils to assist in the build process
+
+
+"""
+
+from builtins import object
+from future.utils import viewvalues
+
+import os
+import logging
+from WMCore.WMSpec.ConfigSectionTree import nodeName
+
+
+class File(object):
+    """
+    _File_
+
+    Class representing a file added to a directory.
+
+    Contains a source for the file and a name within
+    the directory
+
+    """
+    def __init__(self, directory, name, source):
+        self.directory = directory
+        self.name = name
+        self.source = source
+
+
+
+
+    def path(self):
+        """
+        _path_
+
+        Get name of this file within directory structure
+
+        """
+        return "%s/%s" % (self.directory.path(), self.name)
+
+    def fetch(self, targetDir):
+        """
+        _fetch_
+
+        Get the source and put it in the target dir.
+
+        Note: for now this uses cp, could use other
+        things based on source type, eg http:// etc etc
+
+        """
+        command = "/bin/cp -rf %s %s/%s" % (self.source,
+                                        targetDir,
+                                        self.name)
+        logging.info("fetch:%s" % command)
+        os.system(command)
+        return
+
+
+
+
+class Directory(object):
+    """
+    _Directory_
+
+    structure representing a dir, to which files can be attached
+
+
+    """
+    def __init__(self, name):
+        self.name = name
+        self.parent = None
+        self.children = {}
+        self.files = {}
+        self.physicalPath = None
+
+
+    def addDirectory(self, name):
+        """
+        _addDirectory_
+
+        Add a new child Directory to this.
+        Return reference to new Directory instance
+
+        """
+        if name in self.children:
+            return self.children[name]
+
+        self.children[name] = Directory(name)
+        self.children[name].parent = self
+        return self.children[name]
+
+
+    def addFile(self, source, targetName = None):
+        """
+        _addFile_
+
+        Add a file to this directory.
+        The file will be pulled in from the source specified.
+        targetName is the optional name of the file in this
+        directory. If not specified, the basename of the file
+        will be used
+
+        """
+        target = targetName
+        if target == None:
+            target = os.path.basename(source)
+
+        if target in self.files:
+            msg = "File %s already exists in directory %s" % (
+                self.name, target)
+            raise RuntimeError(msg)
+
+        newFile = File(self, target, source)
+        self.files[target] = newFile
+        return
+
+
+    def path(self):
+        """
+        _path_
+
+        Get name of this dir within directory structure
+
+        """
+        if self.parent == None:
+            return self.name
+        return "%s/%s" % (self.parent.path(), self.name)
+
+
+    def create(self, targetDir):
+        """
+        _create_
+
+        Make this directory in the targetDirectory provided,
+        pull in all files and then recursively create any
+        children
+
+        """
+        newDir = "%s/%s" % (targetDir, self.name)
+        logging.info("create(%s)" % newDir)
+        if not os.path.exists(newDir):
+            os.makedirs(newDir)
+        for f in viewvalues(self.files):
+            f.fetch(newDir)
+
+        for child in viewvalues(self.children):
+            child.create(newDir)
+        return
+
+    def __str__(self):
+        result = "%s\n" % self.path()
+        for f in viewvalues(self.files):
+            result += "%s ==> %s\n" % (f.path(), f.source)
+        for d in viewvalues(self.children):
+            result += str(d)
+
+        return result
+
+
+
+
+
+
+
+
+def processDir( cfgSect, parent):
+    """
+    _processDir_
+
+    Process a ConfigSection based directory
+
+    """
+    for setting in cfgSect._internal_settings:
+        value = getattr(cfgSect, setting)
+        if not isinstance(value, dict): continue
+        parent.addFile(value['Source'], value['Target'])
+
+    for subdir in cfgSect._internal_children:
+        newsubdir = parent.addDirectory(subdir)
+        processDir(getattr(cfgSect, subdir), newsubdir)
+    return
+
+
+def makeDirectory(step):
+    """
+    _makeDirectory_
+
+    Create a Directory & file structure from the step provided
+
+    """
+
+    dirs = getattr(step.build.directories, nodeName(step))
+    topDir = Directory(nodeName(step))
+
+    processDir(dirs, topDir)
+
+    return topDir

WMCore/WMSpec/Steps/Builder.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+"""
+_Builder_
+
+Interface definition for Step Builder implementations
+
+"""
+from __future__ import print_function
+
+from builtins import object
+import logging
+
+import WMCore.WMSpec.Steps.BuildTools as BuildTools
+from WMCore.WMSpec.ConfigSectionTree import nodeName
+
+taskSpaceInit = "#!/usr/bin/env python\n"
+taskSpaceInit += "# StepSpace Init module for Step\n"
+taskSpaceInit += "# Autogenerated by WMCore.WMSpec.Steps.Builder\n"
+taskSpaceInit += "\n__all__ = []\n\n"
+taskSpaceInit += "from WMCore.WMRuntime.Bootstrap import establishStepSpace\n\n"
+taskSpaceInit += "def _Locator():\n"
+taskSpaceInit += "    pass\n\n"
+taskSpaceInit += "args = {}\n"
+
+
+class Builder(object):
+    """
+    _Builder_
+
+    base interface for any WMStep Builder
+
+    """
+
+    def __init__(self):
+        self.taskName = None
+        self.stepName = None
+        self.stepDir = None
+        self.taskSpaceInitMod = None
+
+    def __call__(self, step, taskName, workingDirectory, **args):
+        """
+        _operator(step)_
+
+
+        """
+        self.stepName = nodeName(step)
+        self.taskName = taskName
+        self.coreBuild(step, workingDirectory)
+        self.build(step, workingDirectory, **args)
+
+    def coreBuild(self, step, workingDirectory):
+        """
+        _coreBuild_
+
+        Build standard stuff that is common to all jobs
+
+        """
+        logging.info("%s.coreBuild invoked", self.__class__.__name__)
+        dirs = BuildTools.makeDirectory(step)
+        dirs.create(workingDirectory)
+        self.stepDir = "%s/%s" % (workingDirectory, self.stepName)
+        #
+        # Install the basic __init__.py module into the step
+        # directory
+
+        self.taskSpaceInitMod = "%s/__init__.py" % self.stepDir
+
+        with open(self.taskSpaceInitMod, 'w') as handle:
+            handle.write(taskSpaceInit)
+            handle.write("""args["TaskName"] = "%s"\n""" % self.taskName)
+            handle.write("""args["StepName"] = "%s"\n""" % self.stepName)
+            handle.write("""args["Locator"] = _Locator\n""")
+            handle.write("""stepSpace = establishStepSpace(**args)\n""")
+
+    def build(self, step, workingDirectory, **args):
+        """
+        _build_
+
+        Build the step into the working area provided
+        args is for all the things we havent thought of yet
+
+        """
+        msg = "WMSpec.Steps.Builder.build method not overridden in "
+        msg += "implementation: %s\n" % self.__class__.__name__
+        raise NotImplementedError(msg)
+
+    def installWorkingArea(self, step, workingArea, **args):
+        """
+        _installWorkingArea_
+
+        Install working directory information into the step in a standard
+        way.
+
+        """
+        step.section_("builder")
+        step.builder.workingDir = workingArea
+        return

WMCore/WMSpec/Steps/Diagnostic.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+"""
+_Diagnostic_
+
+
+Interface and base class for step specific diagnostic handlers
+
+
+"""
+from builtins import object
+
+from WMCore.FwkJobReport.Report import FwkJobReportException
+
+
+class DiagnosticHandler(object):
+    """
+    _DiagnosticHandler_
+
+    Interface definition for handlers for a specific error condition
+
+
+    """
+    def __call__(self, errorCode, executorInstance, **args):
+        """
+        _operator(errCode, executor)_
+
+        Override to act on a particular error, use the executorInstance
+        to access things like the step, logfiles, and report.
+
+        Args will be used to provide extra information such as Exception
+        instances etc
+
+        """
+        msg = "DiagnosticHandler.__call__ not "
+        msg += "implemented for class %s" % self.__class__.__name__
+        raise NotImplementedError(msg)
+
+    def parse(self, executorInstance, jobRepXml):
+        """
+        Add an error to report if parsing the xml fails.
+        """
+        try:
+            executorInstance.report.parse(jobRepXml, executorInstance.stepName)
+        except FwkJobReportException as ex:
+            # Job report is bad, the parse already puts a 50115 in the file
+            msg = "Error reading XML job report file, possibly corrupt XML File:\n"
+            msg += "Details: %s" % str(ex)
+            executorInstance.report.addError(executorInstance.stepName, 50115, "BadFWJRXML", msg)
+            raise
+
+
+class DefaultDiagnosticHandler(DiagnosticHandler):
+    """
+    _DefaultDiagnosticHandler_
+
+    Catch-all that just adds information to the report
+
+    """
+    def __call__(self, errorCode, executorInstance, **args):
+        pass
+
+
+
+
+
+class Diagnostic(object):
+    """
+    _Diagnostic_
+
+    Base class for a Diagnostic implementation specific to a step type
+    Also works as a bare minimum Diagnostic if overriding is not needed
+
+    """
+    def __init__(self):
+        self.handlers = {}
+        self.defaultHandler = DefaultDiagnosticHandler()
+
+
+
+    def __call__(self, errCode, executor, **args):
+        """
+        _operator(errCode, executor, args)_
+
+        Invoke the diagnostic to produce an error report
+
+        """
+        handler = self.handlers.get(errCode, self.defaultHandler)
+        handler(errCode, executor, **args)
+        executor.saveReport()

WMCore/WMSpec/Steps/Emulator.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python2.5
+"""
+_Emulator_
+
+Define the Interface for a Step Emulator
+
+
+"""
+
+
+from builtins import object
+class Emulator(object):
+    """
+    _Emulator_
+
+    Base class for an Emulator implementation for a given step
+
+    """
+    def initialise(self, executorInstance):
+        """
+        _initialise_
+
+        Post ctor initialisation, shortcut to some of the standard variables provided by the executor
+
+        """
+        self.executor = executorInstance
+        self.job = self.executor.job
+        self.report = self.executor.report
+        self.task = self.executor.task
+        self.step = self.executor.step
+        self.stepSpace = self.executor.stepSpace
+        self.stepName = self.executor.stepName
+
+
+    def pre(self):
+        """
+        _pre_
+
+        Override pre step to emulate
+        """
+        return None
+
+
+    def execute(self):
+        """
+        _emulate_
+
+        Emulate the response to the step provided
+
+        """
+        msg = "WMStep.Steps.Emulator.emulate not implemented for "
+        msg += "class %s" % self.__class__.__name__
+        raise NotImplementedError(msg)
+
+    def post(self):
+        """
+        _post_
+
+        Override to emulate post execution step
+
+        """
+        return None

WMCore/WMSpec/Steps/ExecuteMaster.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+# pylint: disable=E1101
+# E1101: Allow imports from currentThread
+"""
+_ExecuteMaster_
+
+Overseer object that traverses a task and invokes the type based executor
+for each step
+
+"""
+
+from builtins import object
+
+import logging
+import os
+import threading
+import traceback
+
+import WMCore.WMSpec.Steps.StepFactory as StepFactory
+from WMCore.WMException import WMException
+from WMCore.WMSpec.Steps.WMExecutionFailure import WMExecutionFailure
+from WMCore.WMSpec.WMStep import WMStepHelper
+
+
+class ExecuteMaster(object):
+    """
+    _ExecuteMaster_
+
+    Traverse the given task and invoke the execute framework
+    If an emulator is provided, then invoke the appropriate emulator
+    instead of the executor
+
+    """
+
+    def __init__(self):
+        pass
+
+    def __call__(self, task, wmbsJob):
+        """
+        _operator(task)_
+
+        Load and run executors for all steps in Task, if an emulator is
+        available for that step, use it instead.
+
+        """
+
+        myThread = threading.currentThread
+
+        try:
+            myThread.watchdogMonitor.setupMonitors(task, wmbsJob)
+            myThread.watchdogMonitor.notifyJobStart(task)
+        except WMException:
+            self.toTaskDirectory()
+            raise
+        except Exception as ex:
+            msg = "Encountered unhandled exception while starting monitors:\n"
+            msg += str(ex) + '\n'
+            msg += str(traceback.format_exc()) + '\n'
+            logging.error(msg)
+            self.toTaskDirectory()
+            raise WMExecutionFailure(msg)
+
+        failureUpstream = False
+        for step in task.steps().nodeIterator():
+            try:
+                helper = WMStepHelper(step)
+                stepType = helper.stepType()
+                stepName = helper.name()
+                if failureUpstream:
+                    # for chained steps, don't execute further steps if a
+                    # failure has already happened
+                    helper.addOverride("previousCmsRunFailure", True)
+
+                executor = StepFactory.getStepExecutor(stepType)
+                result = self.doExecution(executor, step, wmbsJob)
+                logging.info("StepName: %s, StepType: %s, with result: %r", stepName, stepType, result)
+                if result:  # can be either None, or the step exit code
+                    failureUpstream = True
+            except WMException as ex:
+                msg = "Encountered error while running ExecuteMaster:\n"
+                msg += str(ex) + "\n"
+                logging.error(msg)
+                self.toTaskDirectory()
+                break
+            except Exception as ex:
+                msg = "Encountered error while running ExecuteMaster:\n"
+                msg += str(ex) + "\n"
+                msg += str(traceback.format_exc()) + "\n"
+                self.toTaskDirectory()
+                logging.error(msg)
+                break
+
+        try:
+            myThread.watchdogMonitor.notifyJobEnd(task)
+        except WMException:
+            self.toTaskDirectory()
+        except Exception as ex:
+            msg = "Encountered unhandled exception while ending the job:\n"
+            msg += str(ex) + '\n'
+            msg += str(traceback.format_exc()) + '\n'
+            logging.error(msg)
+            self.toTaskDirectory()
+
+        return
+
+    def doExecution(self, executor, step, job):
+        """
+        _doExecution_
+
+        Invoke the Executor for the step provided
+
+        TODO: Add Monitoring thread & setup
+        TODO: Exception Handling
+        TODO: pre/post outcome can change the next execution task, need to
+              ensure that this happens
+
+
+        """
+        myThread = threading.currentThread
+        # Tell the watchdog that we're starting the step
+        myThread.watchdogMonitor.notifyStepStart(step)
+
+        self.toStepDirectory(step)
+        executor.initialise(step, job)
+        executionObject = executor
+        error = False
+        if executor.emulationMode:
+            executionObject = executor.emulator
+
+        preOutcome = executionObject.pre()
+        if preOutcome is not None:
+            logging.info("Pre Executor Task Change: %s", preOutcome)
+            executor.saveReport()
+            self.toTaskDirectory()
+            myThread.watchdogMonitor.notifyStepEnd(step=step,
+                                                   stepReport=executor.report)
+            executor.saveReport()
+            return preOutcome
+        try:
+            executor.report.setStepStartTime(stepName=executor.stepName)
+            executionObject.execute()
+        except WMExecutionFailure as ex:
+            executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
+            executor.report.addError(executor.stepName, ex.code, "WMAgentStepExecutionError", str(ex))
+            error = True
+        except Exception as ex:
+            logging.error("Exception occured when executing step")
+            logging.error("Exception is %s", ex)
+            logging.error("Traceback: ")
+            logging.error(traceback.format_exc())
+            executor.diagnostic(99109, executor, ExceptionInstance=ex)
+            executor.report.addError(executor.stepName, 99109, "WMAgentStepExecutionError", str(ex))
+            error = True
+        executor.report.setStepStopTime(stepName=executor.stepName)
+        # TODO: Handle generic Exception that indicates development/code errors
+        executor.saveReport()
+
+        postOutcome = executionObject.post()
+        if postOutcome is not None:
+            logging.info("Post Executor Task Change: %s", postOutcome)
+            executor.saveReport()
+            self.toTaskDirectory()
+            myThread.watchdogMonitor.notifyStepEnd(step=step,
+                                                   stepReport=executor.report)
+            executor.saveReport()
+            return postOutcome
+
+        self.toTaskDirectory()
+
+        # Okay, we're done, set the job to successful
+        stepExitCode = executor.report.getExitCode()  # 0 is successful
+        if not error and not stepExitCode:
+            executor.report.setStepStatus(stepName=executor.stepName,
+                                          status=0)
+        executor.saveReport()
+
+        # Tell the watchdog that we're done with the step
+        myThread.watchdogMonitor.notifyStepEnd(step=step,
+                                               stepReport=executor.report)
+        executor.saveReport()
+
+        return stepExitCode
+
+    def toStepDirectory(self, step):
+        """
+        _toStepDirectory_
+
+        Switch current working directory to the step location
+        within WMTaskSpace
+
+        """
+        stepName = WMStepHelper(step).name()
+        from WMTaskSpace import taskSpace
+        stepSpace = taskSpace.stepSpace(stepName)
+
+        os.chdir(stepSpace.location)
+
+    def toTaskDirectory(self):
+        """
+        _toTaskDirectory_
+
+        Switch to current working directory to the task location
+        within WMTaskSpace
+
+        """
+        from WMTaskSpace import taskSpace
+        os.chdir(taskSpace.location)
+        return