wmglobalqueue 2.3.10rc10__py3-none-any.whl

WMCore/WMSpec/Steps/Executor.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+"""
+_Executor_
+
+Interface definition for a step executor
+
+
+"""
+from __future__ import absolute_import
+from builtins import object
+
+import json
+import logging
+import os
+import subprocess
+import sys
+
+from Utils.FileTools import getFullPath
+from Utils.PythonVersion import PY3
+from Utils.Utilities import zipEncodeStr, decodeBytesToUnicodeConditional
+from WMCore.FwkJobReport.Report import Report
+from WMCore.WMSpec.Steps.StepFactory import getStepEmulator
+from WMCore.WMSpec.WMStep import WMStepHelper
+
+getStepName = lambda step: WMStepHelper(step).name()
+getStepErrorDestination = lambda step: WMStepHelper(step).getErrorDestinationStep()
+
+
+def getStepSpace(stepName):
+    """
+    _getStepSpace_
+
+    Util to get the runtime step space.
+    This imports dynamic runtime libraries so be careful how
+    you use it
+
+    """
+    modName = "WMTaskSpace"
+    if modName in sys.modules:
+        taskspace = sys.modules[modName]
+    else:
+        try:
+            # taskspace = __import__(modName, globals(), locals(), ['taskSpace'], -1)
+            taskspace = __import__(modName, globals(), locals(), ['taskSpace'])
+
+        except ImportError as ex:
+            msg = "Unable to load WMTaskSpace module:\n"
+            msg += str(ex)
+            # TODO: Generic ExecutionException...
+            raise RuntimeError(msg)
+
+    try:
+        stepSpace = taskspace.taskSpace.stepSpace(stepName)
+    except Exception as ex:
+        msg = "Error retrieving stepSpace from TaskSpace:\n"
+        msg += str(ex)
+        raise RuntimeError(msg)
+    return stepSpace
+
+
+class Executor(object):
+    """
+    _Executor_
+
+    Define API for a step during execution
+
+    """
+
+    def __init__(self):
+        self.report = None
+        self.diagnostic = None
+        self.emulator = None
+        self.emulationMode = False
+        self.step = None
+        self.stepName = None
+        self.stepSpace = None
+        self.task = None
+        self.workload = None
+        self.job = None
+        self.errorDestination = None
+        self.logger = logging.getLogger()
+        self.logger.info("Steps.Executor logging started")
+
+    def initialise(self, step, job):
+        """
+        _initialise_
+
+
+        Initialise the executor attributes
+
+        """
+        self.step = step
+        self.job = job
+        self.stepName = getStepName(self.step)
+        self.stepSpace = getStepSpace(self.stepName)
+        self.task = self.stepSpace.getWMTask()
+        self.workload = self.stepSpace.taskSpace.workload
+        self.report = Report(self.stepName)
+        self.report.data.task = self.task.name()
+        self.report.data.workload = self.stepSpace.taskSpace.workloadName()
+        self.report.data.id = job['id']
+        self.errorDestination = getStepErrorDestination(self.step)
+
+        self.step.section_("execution")
+        self.step.execution.exitStatus = 0
+        self.step.execution.reportLocation = "%s/Report.pkl" % (
+            self.stepSpace.location,
+        )
+
+        # Set overall step status to 1 (failed)
+        self.report.setStepStatus(stepName=self.stepName, status=1)
+
+        #  //
+        # //  Does the step contain settings for an emulator?
+        # //   If so, load it up
+
+        emulatorName = getattr(self.step.emulator, "emulatorName", None)
+        if emulatorName != None:
+            self.emulator = getStepEmulator(emulatorName)
+            self.emulator.initialise(self)
+            self.emulationMode = True
+
+        return
+
+    def saveReport(self):
+        """
+        _saveReport_
+
+        Save the job report
+
+        """
+        self.report.persist(self.step.execution.reportLocation)
+        return
+
+    def pre(self, emulator=None):
+        """
+        _pre_
+
+        pre execution checks. Can alter flow of execution by returning
+        a different step in the task. If None, then current step will
+        be passed to execute.
+
+        TODO: Define better how to switch to different step within the task
+
+        """
+        return None
+
+    def execute(self, emulator=None):
+        """
+        _execute_
+
+        Override behaviour to execute this step type.
+        If Emulator is provided, execute the emulator instead.
+        Return a framework job report instance
+
+        """
+        msg = "WMSpec.Steps.Executor.execute method not overridden in "
+        msg += "implementation: %s\n" % self.__class__.__name__
+        raise NotImplementedError(msg)
+
+    def post(self, emulator=None):
+        """
+        _post_
+
+        post execution checks. Can alter flow of execution by returning
+        a different step in the task. If None, then the next step in the task
+        will be used next.
+
+        TODO: Define better how to switch to different step within the task
+
+        """
+        return None
+
+    def setCondorChirpAttrDelayed(self, key, value, compress=False, maxLen=5120):
+        """
+        _setCondorChirpAttrDelayed_
+
+        Util to call condor_chirp and publish the key/value pair
+
+        """
+
+        if compress:
+            value = zipEncodeStr(value, maxLen=maxLen)
+
+        # bytes object is not JSON serializable in Python3
+        value = decodeBytesToUnicodeConditional(value, condition=PY3)
+        # construct condor_chirp binary location from CONDOR_CONFIG
+        # Note: This works when we do not use containers.
+        condor_chirp_bin = None
+        condor_config = os.getenv('CONDOR_CONFIG', None)
+        if condor_config:
+            condor_config_dir = os.path.dirname(condor_config)
+            condor_chirp_bin = os.path.join(condor_config_dir, 'main/condor/libexec/condor_chirp')
+
+        # If the above fails, look for the executable in the environment
+        # This is the usual case for containers
+        if not condor_chirp_bin or not os.path.isfile(condor_chirp_bin):
+            condor_chirp_bin = getFullPath("condor_chirp")
+
+        if condor_chirp_bin and os.access(condor_chirp_bin, os.X_OK):
+            args = [condor_chirp_bin, 'set_job_attr_delayed', key, json.dumps(value)]
+            subprocess.call(args)
+        else:
+            if condor_chirp_bin and not os.access(condor_chirp_bin, os.X_OK):
+                msg = 'condor_chirp was found in: %s, but it was not an executable.' % condor_chirp_bin
+            else:
+                msg = 'condor_chirp was not found in the system.'
+            self.logger.warning(msg)
+
+        return

WMCore/WMSpec/Steps/StepFactory.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+"""
+_StepFactory_
+
+Factory implementation to retrieve Step Template instances
+
+"""
+
+
+from WMCore.WMFactory import WMFactory
+from WMCore.WMException import WMException
+
+class StepFactoryException(WMException):
+    """
+    _StepFactortyException_
+
+    Exception for missing objects or problems
+
+    """
+    pass
+
+
+
+class TemplateFactory(WMFactory):
+    """
+    _TemplateFactory_
+
+    Instantiate a WMFactory instance with the appropriate namespace
+
+    """
+    def __init__(self):
+        WMFactory.__init__(self, self.__class__.__name__,
+                           "WMCore.WMSpec.Steps.Templates")
+
+class BuilderFactory(WMFactory):
+    """
+    _BuilderFactory_
+
+    Instantiate a WMFactory instance with the appropriate namespace
+
+    """
+    def __init__(self):
+        WMFactory.__init__(self, self.__class__.__name__,
+                           "WMCore.WMSpec.Steps.Builders")
+
+class ExecutorFactory(WMFactory):
+    """
+    _ExecutorFactory_
+
+    Instantiate a WMFactory instance with the appropriate namespace
+
+    """
+    def __init__(self):
+        WMFactory.__init__(self, self.__class__.__name__,
+                           "WMCore.WMSpec.Steps.Executors")
+class EmulatorFactory(WMFactory):
+    """
+    _EmulatorFactory_
+
+    Instantiate a WMFactory instance with the appropriate namespace
+
+    """
+    def __init__(self):
+        WMFactory.__init__(self, self.__class__.__name__,
+                           "WMCore.WMSpec.Steps.Emulators")
+
+class FetcherFactory(WMFactory):
+    """
+    _FetcherFactory_
+
+    Instantiate a WMFactory instance for the Fetchers package
+
+    """
+    def __init__(self):
+        WMFactory.__init__(self, self.__class__.__name__,
+                           "WMCore.WMSpec.Steps.Fetchers")
+
+class DiagnosticFactory(WMFactory):
+    """
+    _DiagnosticFactory_
+
+    Instantiate a WMFactory instance for the Diagnostics package
+
+    """
+    def __init__(self):
+        WMFactory.__init__(self, self.__class__.__name__,
+                           "WMCore.WMSpec.Steps.Diagnostics")
+
+
+
+_TemplateFactory = TemplateFactory()
+_BuilderFactory  = BuilderFactory()
+_ExecutorFactory = ExecutorFactory()
+_EmulatorFactory = EmulatorFactory()
+_FetcherFactory  = FetcherFactory()
+_DiagnosticFactory = DiagnosticFactory()
+
+def getStepTemplate(stepType):
+    """
+    _getStepTemplate_
+
+    factory method to return a step template instance based on the
+    name of the step
+
+    """
+#    try:
+    return _TemplateFactory.loadObject(stepType)
+#    except WMException, wmEx:
+#        msg = "TemplateFactory Unable to load Object: %s" % stepType
+#        raise StepFactoryException(msg)
+#    except Exception, ex:
+#        msg = "Error creating object %s in TemplateFactory:\n" % stepType
+#        msg += str(ex)
+#        raise StepFactoryException(msg)
+
+def getStepTypeHelper(stepReference):
+    """
+    _getStepTypeHelper_
+
+    Given a step instance, get its type, use that to get a template
+    from the factory and use the template to instantiate the type
+    specific helper
+
+    """
+    stepType = getattr(stepReference, "stepType", None)
+    if stepType == None:
+        msg = "Unable to find stepType attribute for step reference passed\n"
+        msg += "to getStepTypeHelper method"
+        raise StepFactoryException(msg)
+    template = getStepTemplate(stepType)
+    helper = template.helper(stepReference)
+    return helper
+
+
+def getStepBuilder(stepType):
+    """
+    _getStepBuilder_
+
+    """
+    #try:
+    return _BuilderFactory.loadObject(stepType)
+#    except WMException, wmEx:
+#        msg = "BuilderFactory Unable to load Object: %s" % stepType
+#        raise StepFactoryException(msg)
+#    except Exception, ex:
+#        msg = "Error creating object %s in BuilderFactory:\n" % stepType
+#        msg += str(ex)
+#        raise StepFactoryException(msg)
+
+def getStepExecutor(stepType):
+    """
+    _getStepExecutor_
+
+    Get an Executor for the given step type
+
+    """
+#    try:
+    executor =_ExecutorFactory.loadObject(stepType)
+#    except WMException, wmEx:
+#        msg = "ExecutorFactory Unable to load Object: %s" % stepType
+#        raise StepFactoryException(msg)
+#    except Exception, ex:
+#        msg = "Error creating object %s in ExecutorFactory:\n" % stepType
+#        msg += str(ex)
+#        raise StepFactoryException(msg)
+
+    executor.diagnostic = getDiagnostic(stepType)
+    return executor
+
+def getStepEmulator(stepEmuName):
+    """
+    _getStepEmulator_
+
+    Get an instance of a given emulator, Note that this takes the name
+    of the emulator rather than the step Type since there are multiple
+    ways to emulate a given step
+
+    """
+    #try:
+    return _EmulatorFactory.loadObject(stepEmuName)
+    #except WMException, wmEx:
+    #    msg = "EmulatorFactory Unable to load Object: %s" % stepEmuName
+    #    raise StepFactoryException(msg)
+    #except Exception, ex:
+    #    msg = "Error creating object %s in EmulatorFactory:\n" % stepEmuName
+    #    msg += str(ex)
+    #    raise StepFactoryException(msg)
+
+def getFetcher(fetcherName):
+    """
+    _getFetcher_
+
+    Get an instance of the named Fetcher implementation
+
+    """
+    return _FetcherFactory.loadObject(fetcherName)
+
+def getDiagnostic(stepType):
+    """
+    _getDiagnostic_
+
+    Get the appropriate Diagnostic for the stepType, falling back
+    on a Generic Diagnostic if there isnt a specific impl
+
+    """
+    try:
+        return _DiagnosticFactory.loadObject(stepType)
+    except WMException as wmEx:
+        return _DiagnosticFactory.loadObject("Generic")
+#    except Exception, ex:
+#        msg = "Error creating object %s in DiagnosticFactory:\n" % stepType
+#        msg += str(ex)
+#        raise StepFactoryException(msg)

WMCore/WMSpec/Steps/TaskEmulator.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+"""
+_TaskEmulator_
+
+Top level emulator controller
+
+"""
+
+from builtins import object
+from WMCore.WMSpec.WMStep import WMStepHelper
+import WMCore.WMSpec.Steps.StepFactory as StepFactory
+
+
+class TaskEmulator(object):
+    """
+    _TaskEmulator_
+
+    Top Level Emulator that contains a map of all the steps to step emulators
+    and be able to run them
+
+    Instantiate with the task to be emulated
+
+    """
+    def __init__(self, task):
+        self.task = task
+        self.emulators = {}
+
+    def getEmulator(self, stepName):
+        """
+        _getEmulator_
+
+        Retrieve the Emulator for the step name provided, returns None if
+        there is no emulator
+
+        """
+        return self.emulators.get(stepName, None)
+
+
+    def addEmulator(self, nodeName, emulatorName):
+        """
+        _addEmulator_
+
+        Add an Emulator for the node provided, emulatorName is the name
+        of the emulator class to be loaded by the Emulator factory
+
+        TODO: Exception handling
+
+        """
+        emuInstance = StepFactory.getStepEmulator(emulatorName)
+        self.emulators[nodeName] = emuInstance
+        return
+
+    def emulateAll(self):
+        """
+        _emulateAll_
+
+        Traverse all Steps and load up the default Emulator based on
+        type.
+
+        """
+        for step in self.task.steps().nodeIterator():
+            helper = WMStepHelper(step)
+            stepType = helper.stepType()
+            stepName = helper.name()
+            self.addEmulator(stepName, stepType)
+
+
+
+    def __call__(self, step):
+        """
+        _operator(step)_
+
+        Invoke the emulator for the given step
+
+        """

WMCore/WMSpec/Steps/Template.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+"""
+_Template_
+
+Template class for all Step Template implementations to inherit and implement
+the API
+
+"""
+from builtins import object
+import os
+
+from WMCore.WMSpec.WMStep import WMStepHelper
+from WMCore.WMSpec.ConfigSectionTree import nodeName
+
+class CoreHelper(WMStepHelper):
+    """
+    _CoreHelper_
+
+    Helper API for core settings
+
+    """
+    def stepName(self):
+        """
+        _stepName_
+
+        Get the name of the step
+
+        """
+        return nodeName(self.data)
+
+
+    def addEnvironmentVariable(self, varname, setting):
+        """
+        _addEnvironmentVariable_
+
+        add a key = value style setting to the environment for this
+        step
+
+        """
+        setattr(self.data.environment.variables, varname, setting)
+        return
+
+    def addEnvironmentPath(self, pathname, setting):
+        """
+        _addEnvironmentPath_
+
+        add a key = value1:value2:value3 environment setting to this step
+
+        """
+        if getattr(self.data.environment.paths, pathname, None) == None:
+            setattr(self.data.environment.paths, pathname, [])
+        pathentry = getattr(self.data.environment.paths, pathname)
+        pathentry.append(setting)
+        return
+
+    def environment(self):
+        """
+        _environment_
+
+        Get the environment settings for this step
+
+        """
+        return self.data.environment
+
+    def setOverrideCatalog(self, overrideCatalog):
+        """
+        _setOverrideCatalog_
+        set the override catalog needed at least at CERN to use production castor pools
+        """
+        if overrideCatalog is not None:
+            self.data.application.overrideCatalog = overrideCatalog
+
+    def getOverrideCatalog(self):
+        """
+        _getOverrideCatalog_
+        return the TFC specified in overrideCatalog.
+        """
+        return getattr(self.data.application, "overrideCatalog", None)
+
+    def addDirectory(self, dirName):
+        """
+        _addDirectory_
+
+        Add a subdirectory structure to the template that will be built by
+        the builder
+
+        """
+        split = dirName.split("/")
+        split = [ x for x in split if x.strip() != "" ]
+
+        dirs = getattr(self.data.build.directories, self.stepName())
+        for subdir in split:
+            exists = getattr(dirs, subdir, None)
+            if exists == None:
+                dirs.section_(subdir)
+            dirs = getattr(dirs, subdir)
+        return dirs
+
+
+
+    def addFile(self, fileName, newLocation = None):
+        """
+        _addFile_
+
+        Add a file to the job at build time. This file must be
+        a local filesystem file available at fileName.
+
+        An optional location within the step can be specified which
+        may include a path structure that gets translated into calls
+        to addDirectory
+
+        """
+        dirs = getattr(self.data.build.directories, self.stepName())
+        if newLocation != None:
+            filename = os.path.basename(newLocation)
+            dirname = os.path.dirname(newLocation)
+            dirs = self.addDirectory(dirname)
+            setattr(dirs, filename, { "Source" : fileName, "Target" : filename})
+        else:
+            filename = os.path.basename(fileName)
+            setattr(dirs, filename, {"Target" : filename, "Source" : fileName })
+        return
+
+
+    def directoryStructure(self):
+        """
+        _directoryStructure_
+
+        Util to retrieve the directory structure
+
+        """
+        return self.data.build.directories
+
+class Template(object):
+    """
+    _Template_
+
+    Base interface definition for any WMStep Template
+
+    """
+
+    def __init__(self):
+        pass
+
+    def __call__(self, step):
+        """
+        _operator(step)_
+
+        Install the template on the step instance provided
+
+        """
+        self.coreInstall(step)
+        self.install(step)
+
+
+    def coreInstall(self, step):
+        """
+        _coreInstall_
+
+        Install attributes common to all steps
+
+        """
+        # Environment settings to pass to the step
+        step.section_("environment")
+        step.environment.section_("variables")
+        step.environment.section_("paths")
+
+        # Directory structure and files to be included in the job
+        # beyond those that would be added by a Step Specific builder
+        # Step Specific subclasses can simply append to these to get files
+        # and dirs into the job
+        step.section_("build")
+        step.build.section_("directories")
+        step.build.directories.section_(nodeName(step))
+
+
+
+
+
+
+    def install(self, step):
+        """
+        _install_
+
+        Override this method to install the required attributes
+        in the step Instance provided
+
+        """
+        msg = "WMSpec.Steps.Template.install method not overridden in "
+        msg += "implementation: %s\n" % self.__class__.__name__
+        raise NotImplementedError(msg)
+
+
+    def helper(self, step):
+        """
+        _helper_
+
+        Wrap the step instance in a helper class tailored to this particular
+        step type
+
+        """
+        msg = "WMSpec.Steps.Template.helper method not overridden in "
+        msg += "implementation: %s\n" % self.__class__.__name__
+        raise NotImplementedError(msg)