wmglobalqueue 2.3.10__py3-none-any.whl

WMCore/WMSpec/Steps/Templates/DQMUpload.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+"""
+_DQMUpload_
+
+Template for a DQMUpload Step
+
+"""
+
+from WMCore.WMSpec.Steps.Template import Template
+from WMCore.WMSpec.Steps.Template import CoreHelper
+from WMCore.WMSpec.ConfigSectionTree import nodeName
+
+
+class DQMUploadStepHelper(CoreHelper):
+    """
+    _DQMUploadStepHelper_
+
+    Add API calls and helper methods to the basic WMStepHelper to specialise
+    for DQMUpload tasks
+
+    """
+    def setProxyFile(self, proxy):
+        """
+        Sets the name of the proxy file in the input sandbox
+        """
+        self.data.upload.proxy = proxy
+
+    def setServerURL(self, url):
+        """
+        Sets the URL of the DQM GUI server.
+        """
+        self.data.upload.URL = url
+
+
+class DQMUpload(Template):
+    """
+    _DQMUpload_
+
+    Tools for creating a template DQMUpload Step
+
+    """
+
+    def install(self, step):
+        stepname = nodeName(step)
+        step.stepType = "DQMUpload"
+        step.section_("upload")
+        step.upload.URL = 'https://cmsweb.cern.ch/dqm/dev'
+        step.upload.proxy = None
+
+    def helper(self, step):
+        """
+        _helper_
+
+        Wrap the WMStep provided in the CMSSW helper class that
+        includes the ability to add and manipulate the details
+        of a CMSSW workflow step
+
+        """
+        return DQMUploadStepHelper(step)

WMCore/WMSpec/Steps/Templates/DeleteFiles.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+"""
+_DeleteFiles_
+
+Template for a DeleteFiles Step
+
+Mostly borrowed from StageOut since they share a similar function
+
+"""
+
+
+
+
+from WMCore.WMSpec.Steps.Template import Template
+from WMCore.WMSpec.Steps.Template import CoreHelper
+from WMCore.WMSpec.ConfigSectionTree import nodeName
+
+
+
+class DeleteFilesStepHelper(CoreHelper):
+    """
+    _StageOutStepHelper_
+
+    Add API calls and helper methods to the basic WMStepHelper to specialise
+    for StageOut tasks
+
+    This is very similar to StageOut since they have essentially the same function
+
+    """
+
+    def disableRetries(self):
+        """
+            handy for testing, without the 10 minute retry loop
+        """
+        self.data.retryCount = 1
+        self.data.retryDelay = 0
+
+    def addOutputDestination(self, lfn):
+        """
+        Adds an out location to put a tarball of all logs
+        """
+
+
+class DeleteFiles(Template):
+    """
+    _DeleteFiles_
+
+    Tools for creating a template DeleteFiles Step
+
+    """
+
+    def install(self, step):
+        stepname = nodeName(step)
+        step.stepType = "DeleteFiles"
+        step.section_("logs")
+        step.logcount = 0
+        step.retryCount = 3
+        step.retryDelay = 300
+
+
+    def helper(self, step):
+        """
+        _helper_
+
+        Wrap the WMStep provided in the CMSSW helper class that
+        includes the ability to add and manipulate the details
+        of a CMSSW workflow step
+
+        """
+        return DeleteFilesStepHelper(step)

WMCore/WMSpec/Steps/Templates/LogArchive.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+"""
+_LogArchive_
+
+Template for a LogArchive Step
+
+Mostly borrowed from StageOut since they share a similar function
+
+"""
+
+
+
+
+from WMCore.WMSpec.Steps.Template import Template
+from WMCore.WMSpec.Steps.Template import CoreHelper
+from WMCore.WMSpec.ConfigSectionTree import nodeName
+
+
+
+class LogArchiveStepHelper(CoreHelper):
+    """
+    _StageOutStepHelper_
+
+    Add API calls and helper methods to the basic WMStepHelper to specialise
+    for StageOut tasks
+
+    This is very similar to StageOut since they have essentially the same function
+
+    """
+    def addLog(self, infile):
+        """
+            Enqueues(sp?) a file to the StageOut step
+            infile must be a PFN
+            outfile must be a LFN
+        """
+        target = self.data.logs.section_("log%i" % self.data.filecount)
+        target.input  = infile
+        self.data.filecount += 1
+
+    def disableRetries(self):
+        """
+            handy for testing, without the 10 minute retry loop
+        """
+        self.data.retryCount = 1
+        self.data.retryDelay = 0
+
+    def addOutputDestination(self, lfn):
+        """
+        Adds an out location to put a tarball of all logs
+        """
+
+
+class LogArchive(Template):
+    """
+    _LogArchive_
+
+    Tools for creating a template LogArchive Step
+
+    """
+
+    def install(self, step):
+        stepname = nodeName(step)
+        step.stepType = "LogArchive"
+        step.section_("logs")
+        step.logcount = 0
+        step.retryCount = 3
+        step.retryDelay = 300
+
+        # Create output to put logs in
+        step.section_("output")
+        step.output.section_("modules")
+        step.output.modules.section_("logArchive")
+
+
+    def helper(self, step):
+        """
+        _helper_
+
+        Wrap the WMStep provided in the CMSSW helper class that
+        includes the ability to add and manipulate the details
+        of a CMSSW workflow step
+
+        """
+        return LogArchiveStepHelper(step)

WMCore/WMSpec/Steps/Templates/LogCollect.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+"""
+_LogCollect_
+
+Template for a LogCollect Step
+
+Mostly borrowed from StageOut since they share a similar function
+
+"""
+
+from future.utils import viewitems
+
+
+from WMCore.WMSpec.Steps.Template import Template
+from WMCore.WMSpec.Steps.Template import CoreHelper
+
+
+class LogCollectStepHelper(CoreHelper):
+    """
+    _StageOutStepHelper_
+
+    Add API calls and helper methods to the basic WMStepHelper to specialise
+    for StageOut tasks
+
+    This is very similar to StageOut since they have essentially the same function
+
+    """
+
+    def disableRetries(self):
+        """
+            handy for testing, without the 10 minute retry loop
+        """
+        self.data.retryCount = 1
+        self.data.retryDelay = 0
+
+    def addOutputDestination(self, lfn):
+        """
+        Adds an out location to put a tarball of all logs
+        """
+
+    def cmsswSetup(self, cmsswVersion, **options):
+        """
+        _cmsswSetup_
+        Provide setup details for CMSSW.
+        cmsswVersion - required - version of CMSSW to use
+        Optional:
+        scramCommand - defaults to scramv1
+        scramProject - defaults to CMSSW
+        scramArch    - optional scram architecture, defaults to None
+        buildArch    - optional scram build architecture, defaults to None
+        softwareEnvironment - setup command to bootstrap scram,defaults to None
+        """
+        self.data.application.setup.cmsswVersion = cmsswVersion
+        for k, v in viewitems(options):
+            setattr(self.data.application.setup, k, v)
+        return
+
+    def getScramArch(self):
+        """
+        _getScramArch_
+
+        Retrieve the scram architecture used for this step.
+        """
+        return self.data.application.setup.scramArch
+
+    def getCMSSWVersion(self):
+        """
+        _getCMSSWVersion_
+
+        Retrieve the version of the framework used for this step.
+        """
+        return self.data.application.setup.cmsswVersion
+
+class LogCollect(Template):
+    """
+    _LogCollect_
+
+    Tools for creating a template LogCollect Step
+
+    """
+
+    def install(self, step):
+        step.stepType = "LogCollect"
+        step.section_("logs")
+        step.logcount = 0
+        step.retryCount = 3
+        step.retryDelay = 300
+        step.application.section_("setup")
+        step.application.setup.scramCommand = "scramv1"
+        step.application.setup.scramProject = "CMSSW"
+        step.application.setup.cmsswVersion = None
+        step.application.setup.scramArch = None
+        step.application.setup.buildArch = None
+        step.application.setup.softwareEnvironment = None
+
+    def helper(self, step):
+        """
+        _helper_
+
+        Wrap the WMStep provided in the CMSSW helper class that
+        includes the ability to add and manipulate the details
+        of a CMSSW workflow step
+
+        """
+        return LogCollectStepHelper(step)

WMCore/WMSpec/Steps/Templates/StageOut.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+"""
+_StageOut_
+
+Template for a StageOut Step
+
+"""
+
+from WMCore.WMSpec.Steps.Template import Template
+from WMCore.WMSpec.Steps.Template import CoreHelper
+from WMCore.WMSpec.ConfigSectionTree import nodeName
+
+class StageOutStepHelper(CoreHelper):
+    """
+    _StageOutStepHelper_
+
+    Add API calls and helper methods to the basic WMStepHelper to specialise
+    for StageOut tasks
+
+    """
+    def addFile(self, infile, outfile, stageOut = True):
+        """
+            Enqueues(sp?) a file to the StageOut step
+            infile must be a LOCAL FILENAME. No file:/// in front
+            outfile must be a LFN which will be mapped to the remote
+        """
+        target = self.data.files.section_("file%i" % self.data.filecount)
+        target.input         = infile
+        target.output        = outfile
+        target.stageOut      = stageOut
+        self.data.filecount += 1
+
+    def disableRetries(self):
+        """
+            handy for testing, without the 10 minute retry loop
+        """
+        self.data.retryCount = 1
+        self.data.retryDelay = 0
+
+    def disableStraightToMerge(self):
+        """
+        _disableStraightToMerge_
+
+        Disable straight to merge for this step.
+        """
+        if hasattr(self.data.output, "minMergeSize"):
+            delattr(self.data.output, "minMergeSize")
+
+        return
+
+    def disableStraightToMergeForOutputModules(self, outputModules):
+        """
+        _disableStraightToMergeForOutputModules_
+
+        Disable straight to merge only for these output modules.
+        """
+        self.data.output.forceUnmergedOutputs = outputModules
+
+        return
+
+    def setMinMergeSize(self, minMergeSize, maxMergeEvents):
+        """
+        _setMinMergeSize_
+
+        Set the mininum size for promoting a file to merged status.
+        """
+        self.data.output.minMergeSize = minMergeSize
+        self.data.output.maxMergeEvents = maxMergeEvents
+        return
+
+    def minMergeSize(self):
+        """
+        _minMergeSize_
+
+        Retrieve the minimum size for promoting a file to merged status.  If
+        straight to merge is disabled -1 will be returned.
+        """
+        return getattr(self.data.output, "minMergeSize", -1)
+
+class StageOut(Template):
+    """
+    _StageOut_
+
+    Tools for creating a template StageOut Step
+
+    """
+
+    def install(self, step):
+        step.stepType = "StageOut"
+        step.section_("files")
+        step.filecount = 0
+        step.retryCount = 3
+        step.retryDelay = 300
+
+
+    def helper(self, step):
+        """
+        _helper_
+
+        Wrap the WMStep provided in the CMSSW helper class that
+        includes the ability to add and manipulate the details
+        of a CMSSW workflow step
+
+        """
+        return StageOutStepHelper(step)

WMCore/WMSpec/Steps/Templates/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+"""
+_Step.Templates_
+
+Package for containing Step Template implementations
+
+
+"""
+
+__all__ = []

WMCore/WMSpec/Steps/WMExecutionFailure.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+"""
+_WMExecutionFailure_
+
+Error class to generate when an executor implementation encounters a problem
+
+"""
+
+
+
+from WMCore.WMException import WMException
+
+
+class WMExecutionFailure(WMException):
+
+
+    def __init__(self, code, name, detail):
+        WMException.__init__(self, detail, code)
+        self.code = code
+        self.name = name
+        self.detail = detail

WMCore/WMSpec/Steps/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""
+_Steps_
+
+Templates for the various types of Step we deal with in WMTasks
+
+"""
+__all__ = []

WMCore/WMSpec/Utilities.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""
+_Utilities_
+
+Common/useful utils.
+Note that WMWorkload, WMTask and WMStep all depend on this module, so
+dont import them in it.
+
+"""
+
+
+def findObjectTypeAboveNode(node, objType):
+    """
+    _findObjectTypeAboveNode_
+
+    Given a config section (tree or not) traverse up the parent
+    structure until finding the first entry containing an objectType
+    setting that is set to the required type
+
+    """
+    if getattr(node, "objectType", None) == objType:
+        return node
+    if node._internal_parent_ref == None:
+        return None
+    return findObjectTypeAboveNode(node._internal_parent_ref, objType)
+
+
+def findTaskAboveNode(node):
+    """
+    _findTaskAboveNode_
+
+    Given a config section (tree or not) traverse up the parent
+    structure until finding the first entry containing an objectType
+    setting that is set to WMTask
+
+    """
+    return findObjectTypeAboveNode(node, "WMTask")
+
+def findWorkloadAboveNode(node):
+    """
+    _findWorkloadAboveNode_
+
+    Given a config section (tree or not) traverse up the parent
+    structure until finding the first entry containing an objectType
+    setting that is set to WMWorkload
+
+    """
+    return findObjectTypeAboveNode(node, "WMWorkload")
+
+
+
+def stepIdentifier(stepHelper):
+    """
+    _stepIdentifier_
+
+    Generate the /Workload/Task/Step id from the step instance
+    given.
+
+    """
+    task = findTaskAboveNode(stepHelper.data)
+    identifier = "%s/%s" % (task.pathName,
+                            stepHelper.name())
+    return identifier

WMCore/WMSpec/WMSpecErrors.py

@@ -0,0 +1,12 @@
+from WMCore.WMException import WMException
+
+class WMSpecFactoryException(WMException):
+    """
+    _WMSpecFactoryException_
+
+    This exception will be raised by validation functions if
+    the code fails validation.  It will then be changed into
+    a proper HTTPError in the ReqMgr, with the message you enter
+    used as the message for farther up the line.
+    """
+    pass