DIRAC 9.0.0a66__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Utilities/JobModel.py

@@ -1,209 +1,38 @@
-""" This module contains the JobModel class, which is used to validate the job description """
+from __future__ import annotations
 
-# pylint: disable=no-self-argument, no-self-use, invalid-name, missing-function-docstring
-
-from collections.abc import Iterable
-from typing import Any, Annotated, TypeAlias, Self
-
-from pydantic import BaseModel, BeforeValidator, model_validator, field_validator, ConfigDict
+from typing import ClassVar
+from pydantic import PrivateAttr
+from DIRACCommon.WorkloadManagementSystem.Utilities.JobModel import *  # noqa: F401, F403
 
 from DIRAC import gLogger
-from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
-from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACPlatforms, getSites
-
-
-# HACK: Convert appropriate iterables into sets
-def default_set_validator(value):
-    if value is None:
-        return set()
-    elif not isinstance(value, Iterable):
-        return value
-    elif isinstance(value, (str, bytes, bytearray)):
-        return value
-    else:
-        return set(value)
-
-
-CoercibleSetStr: TypeAlias = Annotated[set[str], BeforeValidator(default_set_validator)]
-
-
-class BaseJobDescriptionModel(BaseModel):
-    """Base model for the job description (not parametric)"""
-
-    model_config = ConfigDict(validate_assignment=True)
-
-    arguments: str = ""
-    bannedSites: CoercibleSetStr = set()
-    # TODO: This should use a field factory
-    cpuTime: int = Operations().getValue("JobDescription/DefaultCPUTime", 86400)
-    executable: str
-    executionEnvironment: dict = None
-    gridCE: str = ""
-    inputSandbox: CoercibleSetStr = set()
-    inputData: CoercibleSetStr = set()
-    inputDataPolicy: str = ""
-    jobConfigArgs: str = ""
-    jobGroup: str = ""
-    jobType: str = "User"
-    jobName: str = "Name"
-    # TODO: This should be an StrEnum
-    logLevel: str = "INFO"
-    # TODO: This can't be None with this type hint
-    maxNumberOfProcessors: int = None
-    minNumberOfProcessors: int = 1
-    outputData: CoercibleSetStr = set()
-    outputPath: str = ""
-    outputSandbox: CoercibleSetStr = set()
-    outputSE: str = ""
-    platform: str = ""
-    # TODO: This should use a field factory
-    priority: int = Operations().getValue("JobDescription/DefaultPriority", 1)
-    sites: CoercibleSetStr = set()
-    stderr: str = "std.err"
-    stdout: str = "std.out"
-    tags: CoercibleSetStr = set()
-    extraFields: dict[str, Any] = {}
-
-    @field_validator("cpuTime")
-    def checkCPUTimeBounds(cls, v):
-        minCPUTime = Operations().getValue("JobDescription/MinCPUTime", 100)
-        maxCPUTime = Operations().getValue("JobDescription/MaxCPUTime", 500000)
-        if not minCPUTime <= v <= maxCPUTime:
-            raise ValueError(f"cpuTime out of bounds (must be between {minCPUTime} and {maxCPUTime})")
-        return v
-
-    @field_validator("executable")
-    def checkExecutableIsNotAnEmptyString(cls, v: str):
-        if not v:
-            raise ValueError("executable must not be an empty string")
-        return v
-
-    @field_validator("jobType")
-    def checkJobTypeIsAllowed(cls, v: str):
-        jobTypes = Operations().getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
-        transformationTypes = Operations().getValue("Transformations/DataProcessing", [])
-        allowedTypes = jobTypes + transformationTypes
-        if v not in allowedTypes:
-            raise ValueError(f"jobType '{v}' is not allowed for this kind of user (must be in {allowedTypes})")
-        return v
-
-    @field_validator("inputData")
-    def checkInputDataDoesntContainDoubleSlashes(cls, v):
-        if v:
-            for lfn in v:
-                if lfn.find("//") > -1:
-                    raise ValueError("Input data contains //")
-        return v
-
-    @field_validator("inputData")
-    def addLFNPrefixIfStringStartsWithASlash(cls, v: set[str]):
-        if v:
-            v = {lfn.strip() for lfn in v if lfn.strip()}
-            v = {f"LFN:{lfn}" if lfn.startswith("/") else lfn for lfn in v}
-
-            for lfn in v:
-                if not lfn.startswith("LFN:/"):
-                    raise ValueError("Input data files must start with LFN:/")
-        return v
-
-    @model_validator(mode="after")
-    def checkNumberOfInputDataFiles(self) -> Self:
-        if self.inputData:
-            maxInputDataFiles = Operations().getValue("JobDescription/MaxInputData", 500)
-            if self.jobType == "User" and len(self.inputData) >= maxInputDataFiles:
-                raise ValueError(f"inputData contains too many files (must contain at most {maxInputDataFiles})")
-        return self
-
-    @field_validator("inputSandbox")
-    def checkLFNSandboxesAreWellFormated(cls, v: set[str]):
-        for inputSandbox in v:
-            if inputSandbox.startswith("LFN:") and not inputSandbox.startswith("LFN:/"):
-                raise ValueError("LFN files must start by LFN:/")
-        return v
-
-    @field_validator("logLevel")
-    def checkLogLevelIsValid(cls, v: str):
-        v = v.upper()
-        possibleLogLevels = gLogger.getAllPossibleLevels()
-        if v not in possibleLogLevels:
-            raise ValueError(f"Log level {v} not in {possibleLogLevels}")
-        return v
-
-    @field_validator("minNumberOfProcessors")
-    def checkMinNumberOfProcessorsBounds(cls, v):
-        minNumberOfProcessors = Operations().getValue("JobDescription/MinNumberOfProcessors", 1)
-        maxNumberOfProcessors = Operations().getValue("JobDescription/MaxNumberOfProcessors", 1024)
-        if not minNumberOfProcessors <= v <= maxNumberOfProcessors:
-            raise ValueError(
-                f"minNumberOfProcessors out of bounds (must be between {minNumberOfProcessors} and {maxNumberOfProcessors})"
-            )
-        return v
-
-    @field_validator("maxNumberOfProcessors")
-    def checkMaxNumberOfProcessorsBounds(cls, v):
-        minNumberOfProcessors = Operations().getValue("JobDescription/MinNumberOfProcessors", 1)
-        maxNumberOfProcessors = Operations().getValue("JobDescription/MaxNumberOfProcessors", 1024)
-        if not minNumberOfProcessors <= v <= maxNumberOfProcessors:
-            raise ValueError(
-                f"minNumberOfProcessors out of bounds (must be between {minNumberOfProcessors} and {maxNumberOfProcessors})"
-            )
-        return v
-
-    @model_validator(mode="after")
-    def checkThatMaxNumberOfProcessorsIsGreaterThanMinNumberOfProcessors(self) -> Self:
-        if self.maxNumberOfProcessors:
-            if self.maxNumberOfProcessors < self.minNumberOfProcessors:
-                raise ValueError("maxNumberOfProcessors must be greater than minNumberOfProcessors")
-        return self
-
-    @model_validator(mode="after")
-    def addTagsDependingOnNumberOfProcessors(self) -> Self:
-        if self.minNumberOfProcessors == self.maxNumberOfProcessors:
-            self.tags.add(f"{self.minNumberOfProcessors}Processors")
-        if self.minNumberOfProcessors > 1:
-            self.tags.add("MultiProcessor")
-        return self
+from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
 
-    @field_validator("sites")
-    def checkSites(cls, v: set[str]):
-        if v:
-            res = getSites()
-            if not res["OK"]:
-                raise ValueError(res["Message"])
-            invalidSites = v - set(res["Value"]).union({"ANY"})
-            if invalidSites:
-                raise ValueError(f"Invalid sites: {' '.join(invalidSites)}")
-        return v
 
-    @model_validator(mode="after")
-    def checkThatSitesAndBannedSitesAreNotMutuallyExclusive(self) -> Self:
-        if self.sites and self.bannedSites:
-            while self.bannedSites:
-                self.sites.discard(self.bannedSites.pop())
-            if not self.sites:
-                raise ValueError("sites and bannedSites are mutually exclusive")
-        return self
+def _make_model_config(cls=None) -> BaseJobDescriptionModelConfg:
+    from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 
-    @field_validator("priority")
-    def checkPriorityBounds(cls, v):
-        minPriority = Operations().getValue("JobDescription/MinPriority", 0)
-        maxPriority = Operations().getValue("JobDescription/MaxPriority", 10)
-        if not minPriority <= v <= maxPriority:
-            raise ValueError(f"priority out of bounds (must be between {minPriority} and {maxPriority})")
-        return v
+    ops = Operations()
+    allowedJobTypes = ops.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
+    allowedJobTypes += ops.getValue("Transformations/DataProcessing", [])
+    return {
+        "cpuTime": ops.getValue("JobDescription/DefaultCPUTime", 86400),
+        "priority": ops.getValue("JobDescription/DefaultPriority", 1),
+        "minCPUTime": ops.getValue("JobDescription/MinCPUTime", 100),
+        "maxCPUTime": ops.getValue("JobDescription/MaxCPUTime", 500000),
+        "allowedJobTypes": allowedJobTypes,
+        "maxInputDataFiles": ops.getValue("JobDescription/MaxInputData", 500),
+        "minNumberOfProcessors": ops.getValue("JobDescription/MinNumberOfProcessors", 1),
+        "maxNumberOfProcessors": ops.getValue("JobDescription/MaxNumberOfProcessors", 1024),
+        "minPriority": ops.getValue("JobDescription/MinPriority", 0),
+        "maxPriority": ops.getValue("JobDescription/MaxPriority", 10),
+        "possibleLogLevels": gLogger.getAllPossibleLevels(),
+        "sites": getSites(),
+    }
 
 
-class JobDescriptionModel(BaseJobDescriptionModel):
-    """Model for the job description (non parametric job with user credentials, i.e server side)"""
+class BaseJobDescriptionModel(BaseJobDescriptionModel):  # noqa: F405 pylint: disable=function-redefined
+    _config_builder: ClassVar = _make_model_config
 
-    owner: str
-    ownerGroup: str
-    vo: str
 
-    @model_validator(mode="after")
-    def checkLFNMatchesREGEX(self) -> Self:
-        if self.inputData:
-            for lfn in self.inputData:
-                if not lfn.startswith(f"LFN:/{self.vo}/"):
-                    raise ValueError(f"Input data not correctly specified (must start with LFN:/{self.vo}/)")
-        return self
+class JobDescriptionModel(JobDescriptionModel):  # noqa: F405 pylint: disable=function-redefined
+    _config_builder: ClassVar = _make_model_config

DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py

@@ -9,6 +9,7 @@ from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.Core.Utilities import TimeUtilities
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
+from DIRACCommon.WorkloadManagementSystem.Utilities.JobStatusUtility import getStartAndEndTime, getNewStatus
 
 if TYPE_CHECKING:
     from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
@@ -180,66 +181,3 @@ class JobStatusUtility:
                 return result
 
         return S_OK((attrNames, attrValues))
-
-
-def getStartAndEndTime(startTime, endTime, updateTimes, timeStamps, statusDict):
-    newStat = ""
-    firstUpdate = TimeUtilities.toEpoch(TimeUtilities.fromString(updateTimes[0]))
-    for ts, st in timeStamps:
-        if firstUpdate >= ts:
-            newStat = st
-    # Pick up start and end times from all updates
-    for updTime in updateTimes:
-        sDict = statusDict[updTime]
-        newStat = sDict.get("Status", newStat)
-
-        if not startTime and newStat == JobStatus.RUNNING:
-            # Pick up the start date when the job starts running if not existing
-            startTime = updTime
-        elif not endTime and newStat in JobStatus.JOB_FINAL_STATES:
-            # Pick up the end time when the job is in a final status
-            endTime = updTime
-
-    return startTime, endTime
-
-
-def getNewStatus(
-    jobID: int,
-    updateTimes: list[datetime],
-    lastTime: datetime,
-    statusDict: dict[datetime, Any],
-    currentStatus,
-    force: bool,
-    log,
-):
-    status = ""
-    minor = ""
-    application = ""
-    # Get the last status values looping on the most recent upupdateTimes in chronological order
-    for updTime in [dt for dt in updateTimes if dt >= lastTime]:
-        sDict = statusDict[updTime]
-        log.debug(f"\tTime {updTime} - Statuses {str(sDict)}")
-        status = sDict.get("Status", currentStatus)
-        # evaluate the state machine if the status is changing
-        if not force and status != currentStatus:
-            res = JobStatus.JobsStateMachine(currentStatus).getNextState(status)
-            if not res["OK"]:
-                return res
-            newStat = res["Value"]
-            # If the JobsStateMachine does not accept the candidate, don't update
-            if newStat != status:
-                # keeping the same status
-                log.error(
-                    f"Job Status Error: {jobID} can't move from {currentStatus} to {status}: using {newStat}",
-                )
-                status = newStat
-                sDict["Status"] = newStat
-                # Change the source to indicate this is not what was requested
-                source = sDict.get("Source", "")
-                sDict["Source"] = source + "(SM)"
-            # at this stage status == newStat. Set currentStatus to this new status
-            currentStatus = newStat
-
-        minor = sDict.get("MinorStatus", minor)
-        application = sDict.get("ApplicationStatus", application)
-    return S_OK((status, minor, application))

DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py

@@ -4,176 +4,12 @@
     getParameterVectorLength() - to get the total size of the bunch of parametric jobs
     generateParametricJobs() - to get a list of expanded descriptions of all the jobs
 """
-import re
 
-from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
-from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
-from DIRAC.Core.Utilities.DErrno import EWMSJDL
+# Import from DIRACCommon for backward compatibility
+from DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob import (
+    getParameterVectorLength,
+    generateParametricJobs,
+)
 
-
-def __getParameterSequence(nPar, parList=[], parStart=1, parStep=0, parFactor=1):
-    if parList:
-        if nPar != len(parList):
-            return []
-        else:
-            parameterList = list(parList)
-    else:
-        # The first parameter must have the same type as the other ones even if not defined explicitly
-        parameterList = [parStart * type(parFactor)(1) + type(parStep)(0)]
-        for np in range(1, nPar):
-            parameterList.append(parameterList[np - 1] * parFactor + parStep)
-
-    return parameterList
-
-
-def getParameterVectorLength(jobClassAd):
-    """Get the length of parameter vector in the parametric job description
-
-    :param jobClassAd: ClassAd job description object
-    :return: result structure with the Value: int number of parameter values, None if not a parametric job
-    """
-
-    nParValues = None
-    attributes = jobClassAd.getAttributes()
-    for attribute in attributes:
-        if attribute.startswith("Parameters"):
-            if jobClassAd.isAttributeList(attribute):
-                parameterList = jobClassAd.getListFromExpression(attribute)
-                nThisParValues = len(parameterList)
-            else:
-                nThisParValues = jobClassAd.getAttributeInt(attribute)
-            if nParValues is not None and nParValues != nThisParValues:
-                return S_ERROR(
-                    EWMSJDL,
-                    "Different length of parameter vectors: for %s, %s != %d" % (attribute, nParValues, nThisParValues),
-                )
-            nParValues = nThisParValues
-    if nParValues is not None and nParValues <= 0:
-        return S_ERROR(EWMSJDL, "Illegal number of job parameters %d" % (nParValues))
-    return S_OK(nParValues)
-
-
-def __updateAttribute(classAd, attribute, parName, parValue):
-    # If there is something to do:
-    pattern = r"%%\(%s\)s" % parName
-    if parName == "0":
-        pattern = "%s"
-    expr = classAd.get_expression(attribute)
-    if not re.search(pattern, expr):
-        return False
-
-    pattern = "%%(%s)s" % parName
-    if parName == "0":
-        pattern = "%s"
-
-    parValue = parValue.strip()
-    if classAd.isAttributeList(attribute):
-        parValue = parValue.strip()
-        if parValue.startswith("{"):
-            parValue = parValue.lstrip("{").rstrip("}").strip()
-
-    expr = classAd.get_expression(attribute)
-    newexpr = expr.replace(pattern, str(parValue))
-    classAd.set_expression(attribute, newexpr)
-    return True
-
-
-def generateParametricJobs(jobClassAd):
-    """Generate a series of ClassAd job descriptions expanding
-        job parameters
-
-    :param jobClassAd: ClassAd job description object
-    :return: list of ClassAd job description objects
-    """
-    if not jobClassAd.lookupAttribute("Parameters"):
-        return S_OK([jobClassAd.asJDL()])
-
-    result = getParameterVectorLength(jobClassAd)
-    if not result["OK"]:
-        return result
-    nParValues = result["Value"]
-    if nParValues is None:
-        return S_ERROR(EWMSJDL, "Can not determine the number of job parameters")
-
-    parameterDict = {}
-    attributes = jobClassAd.getAttributes()
-    for attribute in attributes:
-        for key in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
-            if attribute.startswith(key):
-                seqID = "0" if "." not in attribute else attribute.split(".")[1]
-                parameterDict.setdefault(seqID, {})
-                if key == "Parameters":
-                    if jobClassAd.isAttributeList(attribute):
-                        parList = jobClassAd.getListFromExpression(attribute)
-                        if len(parList) != nParValues:
-                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-                        parameterDict[seqID]["ParameterList"] = parList
-                    else:
-                        if attribute != "Parameters":
-                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-                        nPar = jobClassAd.getAttributeInt(attribute)
-                        if nPar is None:
-                            value = jobClassAd.get_expression(attribute)
-                            return S_ERROR(EWMSJDL, f"Inconsistent parametric job description: {attribute}={value}")
-                        parameterDict[seqID]["Parameters"] = nPar
-                else:
-                    value = jobClassAd.getAttributeInt(attribute)
-                    if value is None:
-                        value = jobClassAd.getAttributeFloat(attribute)
-                        if value is None:
-                            value = jobClassAd.get_expression(attribute)
-                            return S_ERROR(f"Illegal value for {attribute} JDL field: {value}")
-                    parameterDict[seqID][key] = value
-
-    if "0" in parameterDict and not parameterDict.get("0"):
-        parameterDict.pop("0")
-
-    parameterLists = {}
-    for seqID in parameterDict:
-        parList = __getParameterSequence(
-            nParValues,
-            parList=parameterDict[seqID].get("ParameterList", []),
-            parStart=parameterDict[seqID].get("ParameterStart", 1),
-            parStep=parameterDict[seqID].get("ParameterStep", 0),
-            parFactor=parameterDict[seqID].get("ParameterFactor", 1),
-        )
-        if not parList:
-            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-
-        parameterLists[seqID] = parList
-
-    jobDescList = []
-    jobDesc = jobClassAd.asJDL()
-    # Width of the sequential parameter number
-    zLength = len(str(nParValues - 1))
-    for n in range(nParValues):
-        newJobDesc = jobDesc
-        newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
-        newClassAd = ClassAd(newJobDesc)
-        for seqID in parameterLists:
-            parameter = parameterLists[seqID][n]
-            for attribute in newClassAd.getAttributes():
-                __updateAttribute(newClassAd, attribute, seqID, str(parameter))
-
-        for seqID in parameterLists:
-            for attribute in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
-                if seqID == "0":
-                    newClassAd.deleteAttribute(attribute)
-                else:
-                    newClassAd.deleteAttribute(f"{attribute}.{seqID}")
-
-            parameter = parameterLists[seqID][n]
-            if seqID == "0":
-                attribute = "Parameter"
-            else:
-                attribute = f"Parameter.{seqID}"
-            if isinstance(parameter, str) and parameter.startswith("{"):
-                newClassAd.insertAttributeInt(attribute, str(parameter))
-            else:
-                newClassAd.insertAttributeString(attribute, str(parameter))
-
-        newClassAd.insertAttributeInt("ParameterNumber", n)
-        newJDL = newClassAd.asJDL()
-        jobDescList.append(newJDL)
-
-    return S_OK(jobDescList)
+# Re-export for backward compatibility
+__all__ = ["getParameterVectorLength", "generateParametricJobs"]

DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py

@@ -175,11 +175,7 @@ def test_logLevelValidator_invalid():
 
 def test_platformValidator_valid():
     """Test the platform validator with valid input."""
-    with patch(
-        "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms",
-        return_value=S_OK(["x86_64-slc6-gcc62-opt"]),
-    ):
-        job = BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt")
+    job = BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt")
     assert job.platform == "x86_64-slc6-gcc62-opt"