DIRAC 9.0.0a64__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py

@@ -1,266 +1,37 @@
-from diraccfg import CFG
+from __future__ import annotations
 
-from DIRAC import S_OK, S_ERROR
-from DIRAC.Core.Utilities import List
-from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
-from DIRAC.Core.Utilities.JDL import loadJDLAsCFG, dumpCFGAsJDL
-
-
-class JobManifest:
-    def __init__(self, manifest=""):
-        self.__manifest = CFG()
-        self.__dirty = False
-        self.__ops = False
-        if manifest:
-            result = self.load(manifest)
-            if not result["OK"]:
-                raise Exception(result["Message"])
-
-    def isDirty(self):
-        return self.__dirty
-
-    def setDirty(self):
-        self.__dirty = True
-
-    def clearDirty(self):
-        self.__dirty = False
-
-    def load(self, dataString):
-        """
-        Auto discover format type based on [ .. ] of JDL
-        """
-        dataString = dataString.strip()
-        if dataString[0] == "[" and dataString[-1] == "]":
-            return self.loadJDL(dataString)
-        else:
-            return self.loadCFG(dataString)
-
-    def loadJDL(self, jdlString):
-        """
-        Load job manifest from JDL format
-        """
-        result = loadJDLAsCFG(jdlString.strip())
-        if not result["OK"]:
-            self.__manifest = CFG()
-            return result
-        self.__manifest = result["Value"][0]
-        return S_OK()
-
-    def loadCFG(self, cfgString):
-        """
-        Load job manifest from CFG format
-        """
-        try:
-            self.__manifest.loadFromBuffer(cfgString)
-        except Exception as e:
-            return S_ERROR(f"Can't load manifest from cfg: {str(e)}")
-        return S_OK()
-
-    def dumpAsCFG(self):
-        return str(self.__manifest)
-
-    def getAsCFG(self):
-        return self.__manifest.clone()
-
-    def dumpAsJDL(self):
-        return dumpCFGAsJDL(self.__manifest)
-
-    def __getCSValue(self, varName, defaultVal=None):
-        if not self.__ops:
-            self.__ops = Operations(group=self.__manifest["OwnerGroup"])
-        if varName[0] != "/":
-            varName = f"JobDescription/{varName}"
-        return self.__ops.getValue(varName, defaultVal)
-
-    def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
-        """
-        Check a numerical var
-        """
-        initialVal = False
-        if varName not in self.__manifest:
-            varValue = self.__getCSValue(f"Default{varName}", defaultVal)
-        else:
-            varValue = self.__manifest[varName]
-            initialVal = varValue
-        try:
-            varValue = int(varValue)
-        except ValueError:
-            return S_ERROR(f"{varName} must be a number")
-        minVal = self.__getCSValue(f"Min{varName}", minVal)
-        maxVal = self.__getCSValue(f"Max{varName}", maxVal)
-        varValue = max(minVal, min(varValue, maxVal))
-        if initialVal != varValue:
-            self.__manifest.setOption(varName, varValue)
-        return S_OK(varValue)
-
-    def __checkChoiceVar(self, varName, defaultVal, choices):
-        """
-        Check a choice var
-        """
-        initialVal = False
-        if varName not in self.__manifest:
-            varValue = self.__getCSValue(f"Default{varName}", defaultVal)
-        else:
-            varValue = self.__manifest[varName]
-            initialVal = varValue
-        if varValue not in self.__getCSValue(f"Choices{varName}", choices):
-            return S_ERROR(f"{varValue} is not a valid value for {varName}")
-        if initialVal != varValue:
-            self.__manifest.setOption(varName, varValue)
-        return S_OK(varValue)
-
-    def __checkMultiChoice(self, varName, choices):
-        """
-        Check a multi choice var
-        """
-        initialVal = False
-        if varName not in self.__manifest:
-            return S_OK()
-        else:
-            varValue = self.__manifest[varName]
-            initialVal = varValue
-        choices = self.__getCSValue(f"Choices{varName}", choices)
-        for v in List.fromChar(varValue):
-            if v not in choices:
-                return S_ERROR(f"{v} is not a valid value for {varName}")
-        if initialVal != varValue:
-            self.__manifest.setOption(varName, varValue)
-        return S_OK(varValue)
+from DIRACCommon.WorkloadManagementSystem.Client.JobState.JobManifest import *  # noqa: F401, F403
 
-    def __checkMaxInputData(self, maxNumber):
-        """
-        Check Maximum Number of Input Data files allowed
-        """
-        varName = "InputData"
-        if varName not in self.__manifest:
-            return S_OK()
-        varValue = self.__manifest[varName]
-        if len(List.fromChar(varValue)) > maxNumber:
-            return S_ERROR(
-                "Number of Input Data Files (%s) greater than current limit: %s"
-                % (len(List.fromChar(varValue)), maxNumber)
-            )
-        return S_OK()
-
-    def __contains__(self, key):
-        """Check if the manifest has the required key"""
-        return key in self.__manifest
+from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 
-    def setOptionsFromDict(self, varDict):
-        for k in sorted(varDict):
-            self.setOption(k, varDict[k])
 
+def makeJobManifestConfig(ownerGroup: str) -> JobManifestConfig:
+    ops = Operations(group=ownerGroup)
+
+    allowedJobTypesForGroup = ops.getValue(
+        "JobDescription/ChoicesJobType",
+        ops.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
+        + ops.getValue("Transformations/DataProcessing", []),
+    )
+
+    return {
+        "defaultForGroup": {
+            "CPUTime": ops.getValue("JobDescription/DefaultCPUTime", 86400),
+            "Priority": ops.getValue("JobDescription/DefaultPriority", 1),
+        },
+        "minForGroup": {
+            "CPUTime": ops.getValue("JobDescription/MinCPUTime", 100),
+            "Priority": ops.getValue("JobDescription/MinPriority", 0),
+        },
+        "maxForGroup": {
+            "CPUTime": ops.getValue("JobDescription/MaxCPUTime", 500000),
+            "Priority": ops.getValue("JobDescription/MaxPriority", 10),
+        },
+        "allowedJobTypesForGroup": allowedJobTypesForGroup,
+        "maxInputData": Operations().getValue("JobDescription/MaxInputData", 500),
+    }
+
+
+class JobManifest(JobManifest):  # noqa: F405 pylint: disable=function-redefined
     def check(self):
-        """
-        Check that the manifest is OK
-        """
-        for k in ["Owner", "OwnerGroup"]:
-            if k not in self.__manifest:
-                return S_ERROR(f"Missing var {k} in manifest")
-
-        # Check CPUTime
-        result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
-        if not result["OK"]:
-            return result
-
-        result = self.__checkNumericalVar("Priority", 1, 0, 10)
-        if not result["OK"]:
-            return result
-
-        maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
-        result = self.__checkMaxInputData(maxInputData)
-        if not result["OK"]:
-            return result
-
-        operation = Operations(group=self.__manifest["OwnerGroup"])
-        allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
-        transformationTypes = operation.getValue("Transformations/DataProcessing", [])
-        result = self.__checkMultiChoice("JobType", allowedJobTypes + transformationTypes)
-        if not result["OK"]:
-            return result
-        return S_OK()
-
-    def createSection(self, secName, contents=False):
-        if secName not in self.__manifest:
-            if contents and not isinstance(contents, CFG):
-                return S_ERROR(f"Contents for section {secName} is not a cfg object")
-            self.__dirty = True
-            return S_OK(self.__manifest.createNewSection(secName, contents=contents))
-        return S_ERROR(f"Section {secName} already exists")
-
-    def getSection(self, secName):
-        self.__dirty = True
-        if secName not in self.__manifest:
-            return S_ERROR(f"{secName} does not exist")
-        sec = self.__manifest[secName]
-        if not sec:
-            return S_ERROR(f"{secName} section empty")
-        return S_OK(sec)
-
-    def setSectionContents(self, secName, contents):
-        if contents and not isinstance(contents, CFG):
-            return S_ERROR(f"Contents for section {secName} is not a cfg object")
-        self.__dirty = True
-        if secName in self.__manifest:
-            self.__manifest[secName].reset()
-            self.__manifest[secName].mergeWith(contents)
-        else:
-            self.__manifest.createNewSection(secName, contents=contents)
-
-    def setOption(self, varName, varValue):
-        """
-        Set a var in job manifest
-        """
-        self.__dirty = True
-        levels = List.fromChar(varName, "/")
-        cfg = self.__manifest
-        for l in levels[:-1]:
-            if l not in cfg:
-                cfg.createNewSection(l)
-            cfg = cfg[l]
-        cfg.setOption(levels[-1], varValue)
-
-    def remove(self, opName):
-        levels = List.fromChar(opName, "/")
-        cfg = self.__manifest
-        for l in levels[:-1]:
-            if l not in cfg:
-                return S_ERROR(f"{opName} does not exist")
-            cfg = cfg[l]
-        if cfg.deleteKey(levels[-1]):
-            self.__dirty = True
-            return S_OK()
-        return S_ERROR(f"{opName} does not exist")
-
-    def getOption(self, varName, defaultValue=None):
-        """
-        Get a variable from the job manifest
-        """
-        cfg = self.__manifest
-        return cfg.getOption(varName, defaultValue)
-
-    def getOptionList(self, section=""):
-        """
-        Get a list of variables in a section of the job manifest
-        """
-        cfg = self.__manifest.getRecursive(section)
-        if not cfg or "value" not in cfg:
-            return []
-        cfg = cfg["value"]
-        return cfg.listOptions()
-
-    def isOption(self, opName):
-        """
-        Check if it is a valid option
-        """
-        return self.__manifest.isOption(opName)
-
-    def getSectionList(self, section=""):
-        """
-        Get a list of sections in the job manifest
-        """
-        cfg = self.__manifest.getRecursive(section)
-        if not cfg or "value" not in cfg:
-            return []
-        cfg = cfg["value"]
-        return cfg.listSections()
+        return super().check(config=makeJobManifestConfig(self.__manifest["OwnerGroup"]))

DIRAC/WorkloadManagementSystem/Client/JobStatus.py

@@ -1,95 +1,10 @@
-"""
-This module contains constants and lists for the possible job states.
-"""
-
-from DIRAC.Core.Utilities.StateMachine import State, StateMachine
-
-#:
-SUBMITTING = "Submitting"
-#:
-RECEIVED = "Received"
-#:
-CHECKING = "Checking"
-#:
-STAGING = "Staging"
-#:
-SCOUTING = "Scouting"
-#:
-WAITING = "Waiting"
-#:
-MATCHED = "Matched"
-#: The Rescheduled status is effectively never stored in the DB.
-#: It could be considered a "virtual" status, and might even be dropped.
-RESCHEDULED = "Rescheduled"
-#:
-RUNNING = "Running"
-#:
-STALLED = "Stalled"
-#:
-COMPLETING = "Completing"
-#:
-DONE = "Done"
-#:
-COMPLETED = "Completed"
-#:
-FAILED = "Failed"
-#:
-DELETED = "Deleted"
-#:
-KILLED = "Killed"
-
-#: Possible job states
-JOB_STATES = [
-    SUBMITTING,
-    RECEIVED,
-    CHECKING,
-    SCOUTING,
-    STAGING,
-    WAITING,
-    MATCHED,
-    RESCHEDULED,
-    RUNNING,
-    STALLED,
-    COMPLETING,
-    DONE,
-    COMPLETED,
-    FAILED,
-    DELETED,
-    KILLED,
-]
+"""Backward compatibility wrapper - moved to DIRACCommon
 
-# Job States when the payload work has finished
-JOB_FINAL_STATES = [DONE, COMPLETED, FAILED, KILLED]
+This module has been moved to DIRACCommon.WorkloadManagementSystem.Client.JobStatus to avoid
+circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
 
-# WMS internal job States indicating the job object won't be updated
-JOB_REALLY_FINAL_STATES = [DELETED]
-
-
-class JobsStateMachine(StateMachine):
-    """Jobs state machine"""
-
-    def __init__(self, state):
-        """c'tor
-        Defines the state machine transactions
-        """
-        super().__init__(state)
-
-        # States transitions
-        self.states = {
-            DELETED: State(15),  # final state
-            KILLED: State(14, [DELETED], defState=KILLED),
-            FAILED: State(13, [RESCHEDULED, DELETED], defState=FAILED),
-            DONE: State(12, [DELETED], defState=DONE),
-            COMPLETED: State(11, [DONE, FAILED], defState=COMPLETED),
-            COMPLETING: State(10, [DONE, FAILED, COMPLETED, STALLED, KILLED], defState=COMPLETING),
-            STALLED: State(9, [RUNNING, FAILED, KILLED], defState=STALLED),
-            RUNNING: State(8, [STALLED, DONE, FAILED, RESCHEDULED, COMPLETING, KILLED, RECEIVED], defState=RUNNING),
-            RESCHEDULED: State(7, [WAITING, RECEIVED, DELETED, FAILED, KILLED], defState=RESCHEDULED),
-            MATCHED: State(6, [RUNNING, FAILED, RESCHEDULED, KILLED], defState=MATCHED),
-            WAITING: State(5, [MATCHED, RESCHEDULED, DELETED, KILLED], defState=WAITING),
-            STAGING: State(4, [CHECKING, WAITING, FAILED, KILLED], defState=STAGING),
-            SCOUTING: State(3, [CHECKING, FAILED, STALLED, KILLED], defState=SCOUTING),
-            CHECKING: State(2, [SCOUTING, STAGING, WAITING, RESCHEDULED, FAILED, DELETED, KILLED], defState=CHECKING),
-            RECEIVED: State(1, [SCOUTING, CHECKING, STAGING, WAITING, FAILED, DELETED, KILLED], defState=RECEIVED),
-            SUBMITTING: State(0, [RECEIVED, CHECKING, DELETED, KILLED], defState=SUBMITTING),  # initial state
-        }
+All exports are maintained for backward compatibility.
+"""
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.WorkloadManagementSystem.Client.JobStatus import *  # noqa: F401, F403

DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py

@@ -1,162 +1,33 @@
 from __future__ import annotations
 
-import base64
-import zlib
+# Import stateless functions from DIRACCommon for backward compatibility
+from DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils import *
 
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
-from DIRAC.Core.Utilities.DErrno import EWMSSUBM
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
-from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK, returnValueOrRaise
-from DIRAC.WorkloadManagementSystem.Client import JobStatus
-from DIRAC.WorkloadManagementSystem.Client.JobState.JobManifest import JobManifest
+from DIRAC.Core.Utilities.ReturnValues import returnValueOrRaise
+from DIRAC.WorkloadManagementSystem.Client.JobState.JobManifest import makeJobManifestConfig
 
 getDIRACPlatform = returnValueOrRaise(
     ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", "getDIRACPlatform")
 )
 
 
-def compressJDL(jdl):
-    """Return compressed JDL string."""
-    return base64.b64encode(zlib.compress(jdl.encode(), -1)).decode()
+def checkAndPrepareJob(
+    jobID, classAdJob, classAdReq, owner, ownerGroup, jobAttrs, vo
+):  # pylint: disable=function-redefined
+    from DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils import checkAndPrepareJob
 
+    config = {
+        "inputDataPolicyForVO": Operations(vo=vo).getValue("InputDataPolicy/InputDataModule"),
+        "softwareDistModuleForVO": Operations(vo=vo).getValue("SoftwareDistModule"),
+        "defaultCPUTimeForOwnerGroup": Operations(group=ownerGroup).getValue("JobDescription/DefaultCPUTime", 86400),
+        "getDIRACPlatform": getDIRACPlatform,
+    }
+    return checkAndPrepareJob(jobID, classAdJob, classAdReq, owner, ownerGroup, jobAttrs, vo, config=config)
 
-def extractJDL(compressedJDL):
-    """Return decompressed JDL string."""
-    # the starting bracket is guaranteeed by JobManager.submitJob
-    # we need the check to be backward compatible
-    if isinstance(compressedJDL, bytes):
-        if compressedJDL.startswith(b"["):
-            return compressedJDL.decode()
-    else:
-        if compressedJDL.startswith("["):
-            return compressedJDL
-    return zlib.decompress(base64.b64decode(compressedJDL)).decode()
 
+def checkAndAddOwner(jdl: str, owner: str, ownerGroup: str):  # pylint: disable=function-redefined
+    from DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils import checkAndAddOwner
 
-def checkAndAddOwner(jdl: str, owner: str, ownerGroup: str) -> JobManifest:
-    jobManifest = JobManifest()
-    res = jobManifest.load(jdl)
-    if not res["OK"]:
-        return res
-
-    jobManifest.setOptionsFromDict({"Owner": owner, "OwnerGroup": ownerGroup})
-    res = jobManifest.check()
-    if not res["OK"]:
-        return res
-
-    return S_OK(jobManifest)
-
-
-def fixJDL(jdl: str) -> str:
-    # 1.- insert original JDL on DB and get new JobID
-    # Fix the possible lack of the brackets in the JDL
-    if jdl.strip()[0].find("[") != 0:
-        jdl = "[" + jdl + "]"
-    return jdl
-
-
-def checkAndPrepareJob(jobID, classAdJob, classAdReq, owner, ownerGroup, jobAttrs, vo):
-    error = ""
-
-    jdlOwner = classAdJob.getAttributeString("Owner")
-    jdlOwnerGroup = classAdJob.getAttributeString("OwnerGroup")
-    jdlVO = classAdJob.getAttributeString("VirtualOrganization")
-
-    # The below is commented out since this is always overwritten by the submitter IDs
-    # but the check allows to findout inconsistent client environments
-    if jdlOwner and jdlOwner != owner:
-        error = "Wrong Owner in JDL"
-    elif jdlOwnerGroup and jdlOwnerGroup != ownerGroup:
-        error = "Wrong Owner Group in JDL"
-    elif jdlVO and jdlVO != vo:
-        error = "Wrong Virtual Organization in JDL"
-
-    classAdJob.insertAttributeString("Owner", owner)
-    classAdJob.insertAttributeString("OwnerGroup", ownerGroup)
-
-    if vo:
-        classAdJob.insertAttributeString("VirtualOrganization", vo)
-
-    classAdReq.insertAttributeString("Owner", owner)
-    classAdReq.insertAttributeString("OwnerGroup", ownerGroup)
-    if vo:
-        classAdReq.insertAttributeString("VirtualOrganization", vo)
-
-    inputDataPolicy = Operations(vo=vo).getValue("InputDataPolicy/InputDataModule")
-    if inputDataPolicy and not classAdJob.lookupAttribute("InputDataModule"):
-        classAdJob.insertAttributeString("InputDataModule", inputDataPolicy)
-
-    softwareDistModule = Operations(vo=vo).getValue("SoftwareDistModule")
-    if softwareDistModule and not classAdJob.lookupAttribute("SoftwareDistModule"):
-        classAdJob.insertAttributeString("SoftwareDistModule", softwareDistModule)
-
-    # priority
-    priority = classAdJob.getAttributeInt("Priority")
-    if priority is None:
-        priority = 0
-    classAdReq.insertAttributeInt("UserPriority", priority)
-
-    # CPU time
-    cpuTime = classAdJob.getAttributeInt("CPUTime")
-    if cpuTime is None:
-        opsHelper = Operations(group=ownerGroup)
-        cpuTime = opsHelper.getValue("JobDescription/DefaultCPUTime", 86400)
-    classAdReq.insertAttributeInt("CPUTime", cpuTime)
-
-    # platform(s)
-    platformList = classAdJob.getListFromExpression("Platform")
-    if platformList:
-        result = getDIRACPlatform(platformList)
-        if not result["OK"]:
-            return result
-        if result["Value"]:
-            classAdReq.insertAttributeVectorString("Platforms", result["Value"])
-        else:
-            error = "OS compatibility info not found"
-    if error:
-        retVal = S_ERROR(EWMSSUBM, error)
-        retVal["JobId"] = jobID
-        retVal["Status"] = JobStatus.FAILED
-        retVal["MinorStatus"] = error
-
-        jobAttrs["Status"] = JobStatus.FAILED
-
-        jobAttrs["MinorStatus"] = error
-        return retVal
-    return S_OK()
-
-
-def createJDLWithInitialStatus(
-    classAdJob, classAdReq, jdl2DBParameters, jobAttrs, initialStatus, initialMinorStatus, *, modern=False
-):
-    """
-    :param modern: if True, store boolean instead of string for VerifiedFlag (used by diracx only)
-    """
-    priority = classAdJob.getAttributeInt("Priority")
-    if priority is None:
-        priority = 0
-    jobAttrs["UserPriority"] = priority
-
-    for jdlName in jdl2DBParameters:
-        # Defaults are set by the DB.
-        jdlValue = classAdJob.getAttributeString(jdlName)
-        if jdlValue:
-            jobAttrs[jdlName] = jdlValue
-
-    jdlValue = classAdJob.getAttributeString("Site")
-    if jdlValue:
-        if jdlValue.find(",") != -1:
-            jobAttrs["Site"] = "Multiple"
-        else:
-            jobAttrs["Site"] = jdlValue
-
-    jobAttrs["VerifiedFlag"] = True if modern else "True"
-
-    jobAttrs["Status"] = initialStatus
-
-    jobAttrs["MinorStatus"] = initialMinorStatus
-
-    reqJDL = classAdReq.asJDL()
-    classAdJob.insertAttributeInt("JobRequirements", reqJDL)
-
-    return classAdJob.asJDL()
+    return checkAndAddOwner(jdl, owner, ownerGroup, job_manifest_config=makeJobManifestConfig(ownerGroup))