DIRACCommon 9.0.0a66__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

DIRACCommon/WorkloadManagementSystem/Utilities/ParametricJob.py

@@ -0,0 +1,179 @@
+""" Utilities to process parametric job definitions and generate
+    bunches of parametric jobs. It exposes the following functions:
+
+    getParameterVectorLength() - to get the total size of the bunch of parametric jobs
+    generateParametricJobs() - to get a list of expanded descriptions of all the jobs
+"""
+import re
+
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities.DErrno import EWMSJDL
+
+
+def __getParameterSequence(nPar, parList=[], parStart=1, parStep=0, parFactor=1):
+    if parList:
+        if nPar != len(parList):
+            return []
+        else:
+            parameterList = list(parList)
+    else:
+        # The first parameter must have the same type as the other ones even if not defined explicitly
+        parameterList = [parStart * type(parFactor)(1) + type(parStep)(0)]
+        for np in range(1, nPar):
+            parameterList.append(parameterList[np - 1] * parFactor + parStep)
+
+    return parameterList
+
+
+def getParameterVectorLength(jobClassAd):
+    """Get the length of parameter vector in the parametric job description
+
+    :param jobClassAd: ClassAd job description object
+    :return: result structure with the Value: int number of parameter values, None if not a parametric job
+    """
+
+    nParValues = None
+    attributes = jobClassAd.getAttributes()
+    for attribute in attributes:
+        if attribute.startswith("Parameters"):
+            if jobClassAd.isAttributeList(attribute):
+                parameterList = jobClassAd.getListFromExpression(attribute)
+                nThisParValues = len(parameterList)
+            else:
+                nThisParValues = jobClassAd.getAttributeInt(attribute)
+            if nParValues is not None and nParValues != nThisParValues:
+                return S_ERROR(
+                    EWMSJDL,
+                    "Different length of parameter vectors: for %s, %s != %d" % (attribute, nParValues, nThisParValues),
+                )
+            nParValues = nThisParValues
+    if nParValues is not None and nParValues <= 0:
+        return S_ERROR(EWMSJDL, "Illegal number of job parameters %d" % (nParValues))
+    return S_OK(nParValues)
+
+
+def __updateAttribute(classAd, attribute, parName, parValue):
+    # If there is something to do:
+    pattern = r"%%\(%s\)s" % parName
+    if parName == "0":
+        pattern = "%s"
+    expr = classAd.get_expression(attribute)
+    if not re.search(pattern, expr):
+        return False
+
+    pattern = "%%(%s)s" % parName
+    if parName == "0":
+        pattern = "%s"
+
+    parValue = parValue.strip()
+    if classAd.isAttributeList(attribute):
+        parValue = parValue.strip()
+        if parValue.startswith("{"):
+            parValue = parValue.lstrip("{").rstrip("}").strip()
+
+    expr = classAd.get_expression(attribute)
+    newexpr = expr.replace(pattern, str(parValue))
+    classAd.set_expression(attribute, newexpr)
+    return True
+
+
+def generateParametricJobs(jobClassAd):
+    """Generate a series of ClassAd job descriptions expanding
+        job parameters
+
+    :param jobClassAd: ClassAd job description object
+    :return: list of ClassAd job description objects
+    """
+    if not jobClassAd.lookupAttribute("Parameters"):
+        return S_OK([jobClassAd.asJDL()])
+
+    result = getParameterVectorLength(jobClassAd)
+    if not result["OK"]:
+        return result
+    nParValues = result["Value"]
+    if nParValues is None:
+        return S_ERROR(EWMSJDL, "Can not determine the number of job parameters")
+
+    parameterDict = {}
+    attributes = jobClassAd.getAttributes()
+    for attribute in attributes:
+        for key in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
+            if attribute.startswith(key):
+                seqID = "0" if "." not in attribute else attribute.split(".")[1]
+                parameterDict.setdefault(seqID, {})
+                if key == "Parameters":
+                    if jobClassAd.isAttributeList(attribute):
+                        parList = jobClassAd.getListFromExpression(attribute)
+                        if len(parList) != nParValues:
+                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
+                        parameterDict[seqID]["ParameterList"] = parList
+                    else:
+                        if attribute != "Parameters":
+                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
+                        nPar = jobClassAd.getAttributeInt(attribute)
+                        if nPar is None:
+                            value = jobClassAd.get_expression(attribute)
+                            return S_ERROR(EWMSJDL, f"Inconsistent parametric job description: {attribute}={value}")
+                        parameterDict[seqID]["Parameters"] = nPar
+                else:
+                    value = jobClassAd.getAttributeInt(attribute)
+                    if value is None:
+                        value = jobClassAd.getAttributeFloat(attribute)
+                        if value is None:
+                            value = jobClassAd.get_expression(attribute)
+                            return S_ERROR(f"Illegal value for {attribute} JDL field: {value}")
+                    parameterDict[seqID][key] = value
+
+    if "0" in parameterDict and not parameterDict.get("0"):
+        parameterDict.pop("0")
+
+    parameterLists = {}
+    for seqID in parameterDict:
+        parList = __getParameterSequence(
+            nParValues,
+            parList=parameterDict[seqID].get("ParameterList", []),
+            parStart=parameterDict[seqID].get("ParameterStart", 1),
+            parStep=parameterDict[seqID].get("ParameterStep", 0),
+            parFactor=parameterDict[seqID].get("ParameterFactor", 1),
+        )
+        if not parList:
+            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
+
+        parameterLists[seqID] = parList
+
+    jobDescList = []
+    jobDesc = jobClassAd.asJDL()
+    # Width of the sequential parameter number
+    zLength = len(str(nParValues - 1))
+    for n in range(nParValues):
+        newJobDesc = jobDesc
+        newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
+        newClassAd = ClassAd(newJobDesc)
+        for seqID in parameterLists:
+            parameter = parameterLists[seqID][n]
+            for attribute in newClassAd.getAttributes():
+                __updateAttribute(newClassAd, attribute, seqID, str(parameter))
+
+        for seqID in parameterLists:
+            for attribute in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
+                if seqID == "0":
+                    newClassAd.deleteAttribute(attribute)
+                else:
+                    newClassAd.deleteAttribute(f"{attribute}.{seqID}")
+
+            parameter = parameterLists[seqID][n]
+            if seqID == "0":
+                attribute = "Parameter"
+            else:
+                attribute = f"Parameter.{seqID}"
+            if isinstance(parameter, str) and parameter.startswith("{"):
+                newClassAd.insertAttributeInt(attribute, str(parameter))
+            else:
+                newClassAd.insertAttributeString(attribute, str(parameter))
+
+        newClassAd.insertAttributeInt("ParameterNumber", n)
+        newJDL = newClassAd.asJDL()
+        jobDescList.append(newJDL)
+
+    return S_OK(jobDescList)

DIRACCommon/WorkloadManagementSystem/Utilities/__init__.py

@@ -0,0 +1 @@
+"""DIRACCommon WorkloadManagementSystem utilities"""

DIRACCommon/WorkloadManagementSystem/__init__.py

@@ -0,0 +1 @@
+"""DIRACCommon WorkloadManagementSystem utilities"""

{diraccommon-9.0.0a66.dist-info → diraccommon-9.0.0a67.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRACCommon
-Version: 9.0.0a66
+Version: 9.0.0a67
 Summary: Stateless utilities extracted from DIRAC for use by DiracX and other projects
 Project-URL: Homepage, https://github.com/DIRACGrid/DIRAC
 Project-URL: Documentation, https://dirac.readthedocs.io/
@@ -14,6 +14,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: System :: Distributed Computing
 Requires-Python: >=3.11
+Requires-Dist: diraccfg
+Requires-Dist: pydantic>=2.0.0
 Requires-Dist: typing-extensions>=4.0.0
 Provides-Extra: testing
 Requires-Dist: pytest-cov>=4.0.0; extra == 'testing'
@@ -30,8 +32,11 @@ This package solves the circular dependency issue where DiracX needs DIRAC utili
 
 ## Contents
 
-- `DIRACCommon.Utils.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
-- `DIRACCommon.Utils.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
+- `DIRACCommon.Core.Utilities.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ClassAd.ClassAdLight`: JDL parsing utilities
+- `DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils`: Job database utilities
+- `DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob`: Parametric job utilities
 
 ## Installation
 
@@ -42,7 +47,7 @@ pip install DIRACCommon
 ## Usage
 
 ```python
-from DIRACCommon.Utils.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
 
 def my_function():
     if success:

diraccommon-9.0.0a67.dist-info/RECORD

@@ -0,0 +1,25 @@
+DIRACCommon/__init__.py,sha256=o31FFvmIiU2Wsc9nfsPZVaXYK9d3KTN0vOa8mZS6x5E,587
+DIRACCommon/Core/__init__.py,sha256=Kv_Rkbm9hOPmHSnTG0_i7QR5Tck8bldITONhsoT2syg,33
+DIRACCommon/Core/Utilities/DErrno.py,sha256=LUaIcKkjuioTMO3O7H9T9Pk5pHUS63Nzx3FcaP40n5c,7998
+DIRACCommon/Core/Utilities/JDL.py,sha256=z4RID730tbCmbgldKfhSFc9ADfXDhUfjaBG8RpuK2Bs,6724
+DIRACCommon/Core/Utilities/List.py,sha256=AahQrlQqfnh8tyA1GjGqYJfo_3v5N77IQreA8VCNXks,3758
+DIRACCommon/Core/Utilities/ReturnValues.py,sha256=YDZEZrYLtI_Aa8qTnHX633F1cRupQ_UTAnI7aSlf0sI,8733
+DIRACCommon/Core/Utilities/StateMachine.py,sha256=5Pwpc2wmCgb6PwyUH2qe9HZwBMyzkhFTBA7cixso1Sg,6767
+DIRACCommon/Core/Utilities/TimeUtilities.py,sha256=wXuYIwECbLSFdVoNM0iSYoCrz6_iL-JrMV1yVv3FloA,9026
+DIRACCommon/Core/Utilities/__init__.py,sha256=hwOqqYwQDINF-D3eKKBC1r-B_jEnQ5ZSdJvc7rIv_fY,56
+DIRACCommon/Core/Utilities/ClassAd/ClassAdLight.py,sha256=7450ECzxnJQpIjzbaE-uFp3ISE55iL6YM-TRqyebZTY,9597
+DIRACCommon/Core/Utilities/ClassAd/__init__.py,sha256=z0yRe-wRxIHE-Sp3tnyC37S6z38dZHCxuqf3ZV1OyzQ,40
+DIRACCommon/WorkloadManagementSystem/__init__.py,sha256=XMOUD0OcPVo-mjuA0vU-mbtf9K0kBMy5I5sELrX3HEs,53
+DIRACCommon/WorkloadManagementSystem/Client/JobStatus.py,sha256=lBfPYa_RRjJlWEU5eqrbWFaSl1081CqSDjhI6B7-qeA,2813
+DIRACCommon/WorkloadManagementSystem/Client/__init__.py,sha256=ssMppR0P3iXlUaqWPjl1Y5AlQ6LgUAsRa9-5lVkBsL0,60
+DIRACCommon/WorkloadManagementSystem/Client/JobState/JobManifest.py,sha256=E3DVRzAHUFAFa9baqEbAMr-4cu6tTXuVRVdl7XgUX7I,7380
+DIRACCommon/WorkloadManagementSystem/Client/JobState/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+DIRACCommon/WorkloadManagementSystem/DB/JobDBUtils.py,sha256=YU97uk0fjwYN6O8nkD6XjnZ7k9ovhSLYjytqI3oLRfU,5777
+DIRACCommon/WorkloadManagementSystem/DB/__init__.py,sha256=CXWy4bdKeG6ml9T5r8-E7EyfDBNE79I_bs13qYI7yY8,56
+DIRACCommon/WorkloadManagementSystem/Utilities/JobModel.py,sha256=kpvsxQnR-j9Y9I38qUWLD7coFRssLwtJCtSNAbjFh1Y,8973
+DIRACCommon/WorkloadManagementSystem/Utilities/JobStatusUtility.py,sha256=Kf4o1bp25JLsRb-Vsc4NuM6w1Io2hg_ALtIyqBk-Qhw,3628
+DIRACCommon/WorkloadManagementSystem/Utilities/ParametricJob.py,sha256=iYd29TgGXZk31pQLPw_frIhcmZB9X-RmuDmIRxzgiww,7405
+DIRACCommon/WorkloadManagementSystem/Utilities/__init__.py,sha256=XMOUD0OcPVo-mjuA0vU-mbtf9K0kBMy5I5sELrX3HEs,53
+diraccommon-9.0.0a67.dist-info/METADATA,sha256=NA1PLIfXlFQ4tWmuf1m_7mSvSlHBuHVudhsOIC_VRDs,2575
+diraccommon-9.0.0a67.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+diraccommon-9.0.0a67.dist-info/RECORD,,

diraccommon-9.0.0a66.dist-info/RECORD

@@ -1,7 +0,0 @@
-DIRACCommon/__init__.py,sha256=o31FFvmIiU2Wsc9nfsPZVaXYK9d3KTN0vOa8mZS6x5E,587
-DIRACCommon/Utils/DErrno.py,sha256=LUaIcKkjuioTMO3O7H9T9Pk5pHUS63Nzx3FcaP40n5c,7998
-DIRACCommon/Utils/ReturnValues.py,sha256=9lEFPeu5MV1zCWTOgvd2779QMZeCKSnTW0LWa873eBE,8724
-DIRACCommon/Utils/__init__.py,sha256=hwOqqYwQDINF-D3eKKBC1r-B_jEnQ5ZSdJvc7rIv_fY,56
-diraccommon-9.0.0a66.dist-info/METADATA,sha256=0ci1O9p3PmrdZAmamil4OG-rXoZLygLuLXJuqsd3JJU,2248
-diraccommon-9.0.0a66.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-diraccommon-9.0.0a66.dist-info/RECORD,,