DIRACCommon 9.0.0a66__tar.gz → 9.0.0a67__tar.gz

{diraccommon-9.0.0a66 → diraccommon-9.0.0a67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRACCommon
-Version: 9.0.0a66
+Version: 9.0.0a67
 Summary: Stateless utilities extracted from DIRAC for use by DiracX and other projects
 Project-URL: Homepage, https://github.com/DIRACGrid/DIRAC
 Project-URL: Documentation, https://dirac.readthedocs.io/
@@ -14,6 +14,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: System :: Distributed Computing
 Requires-Python: >=3.11
+Requires-Dist: diraccfg
+Requires-Dist: pydantic>=2.0.0
 Requires-Dist: typing-extensions>=4.0.0
 Provides-Extra: testing
 Requires-Dist: pytest-cov>=4.0.0; extra == 'testing'
@@ -30,8 +32,11 @@ This package solves the circular dependency issue where DiracX needs DIRAC utili
 
 ## Contents
 
-- `DIRACCommon.Utils.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
-- `DIRACCommon.Utils.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
+- `DIRACCommon.Core.Utilities.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ClassAd.ClassAdLight`: JDL parsing utilities
+- `DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils`: Job database utilities
+- `DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob`: Parametric job utilities
 
 ## Installation
 
@@ -42,7 +47,7 @@ pip install DIRACCommon
 ## Usage
 
 ```python
-from DIRACCommon.Utils.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
 
 def my_function():
     if success:

{diraccommon-9.0.0a66 → diraccommon-9.0.0a67}/README.md

@@ -8,8 +8,11 @@ This package solves the circular dependency issue where DiracX needs DIRAC utili
 
 ## Contents
 
-- `DIRACCommon.Utils.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
-- `DIRACCommon.Utils.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
+- `DIRACCommon.Core.Utilities.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ClassAd.ClassAdLight`: JDL parsing utilities
+- `DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils`: Job database utilities
+- `DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob`: Parametric job utilities
 
 ## Installation
 
@@ -20,7 +23,7 @@ pip install DIRACCommon
 ## Usage
 
 ```python
-from DIRACCommon.Utils.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
 
 def my_function():
     if success:

{diraccommon-9.0.0a66 → diraccommon-9.0.0a67}/pyproject.toml

@@ -21,6 +21,8 @@ classifiers = [
 ]
 dependencies = [
     "typing-extensions>=4.0.0",
+    "diraccfg",
+    "pydantic>=2.0.0",
 ]
 dynamic = ["version"]
 

diraccommon-9.0.0a67/src/DIRACCommon/Core/Utilities/ClassAd/ClassAdLight.py

@@ -0,0 +1,295 @@
+""" ClassAd Class - a light purely Python representation of the
+    Condor ClassAd library.
+"""
+
+
+class ClassAd:
+    def __init__(self, jdl):
+        """ClassAd constructor from a JDL string"""
+        self.contents = {}
+        result = self.__analyse_jdl(jdl)
+        if result:
+            self.contents = result
+
+    def __analyse_jdl(self, jdl, index=0):
+        """Analyse one [] jdl enclosure"""
+
+        jdl = jdl.strip()
+
+        # Strip all the blanks first
+        # temp = jdl.replace(' ','').replace('\n','')
+        temp = jdl
+
+        result = {}
+
+        if temp[0] != "[" or temp[-1] != "]":
+            print("Invalid JDL: it should start with [ and end with ]")
+            return result
+
+        # Parse the jdl string now
+        body = temp[1:-1]
+        index = 0
+        namemode = 1
+        valuemode = 0
+        while index < len(body):
+            if namemode:
+                ind = body.find("=", index)
+                if ind != -1:
+                    name = body[index:ind]
+                    index = ind + 1
+                    valuemode = 1
+                    namemode = 0
+                else:
+                    break
+            elif valuemode:
+                ind1 = body.find("[", index)
+                ind2 = body.find(";", index)
+                if ind1 != -1 and ind1 < ind2:
+                    value, newind = self.__find_subjdl(body, ind1)
+                elif ind1 == -1 and ind2 == -1:
+                    value = body[index:]
+                    newind = len(body)
+                else:
+                    if index == ind2:
+                        return {}
+                    else:
+                        value = body[index:ind2]
+                        newind = ind2 + 1
+
+                result[name.strip()] = value.strip().replace("\n", "")
+                index = newind
+                valuemode = 0
+                namemode = 1
+
+        return result
+
+    def __find_subjdl(self, body, index):
+        """Find a full [] enclosure starting from index"""
+        result = ""
+        if body[index] != "[":
+            return (result, 0)
+
+        depth = 0
+        ind = index
+        while depth < 10:
+            ind1 = body.find("]", ind + 1)
+            ind2 = body.find("[", ind + 1)
+            if ind2 != -1 and ind2 < ind1:
+                depth += 1
+                ind = ind2
+            else:
+                if depth > 0:
+                    depth -= 1
+                    ind = ind1
+                else:
+                    result = body[index : ind1 + 1]
+                    if body[ind1 + 1] == ";":
+                        return (result, ind1 + 2)
+                    return result, 0
+
+        return result, 0
+
+    def insertAttributeInt(self, name, attribute):
+        """Insert a named integer attribute"""
+
+        self.contents[name] = str(attribute)
+
+    def insertAttributeBool(self, name, attribute):
+        """Insert a named boolean attribute"""
+
+        if attribute:
+            self.contents[name] = "true"
+        else:
+            self.contents[name] = "false"
+
+    def insertAttributeString(self, name, attribute):
+        """Insert a named string attribute"""
+
+        self.contents[name] = '"' + str(attribute) + '"'
+
+    def insertAttributeVectorString(self, name, attributelist):
+        """Insert a named string list attribute"""
+
+        tmp = ['"' + x + '"' for x in attributelist]
+        tmpstr = ",".join(tmp)
+        self.contents[name] = "{" + tmpstr + "}"
+
+    def insertAttributeVectorInt(self, name, attributelist):
+        """Insert a named string list attribute"""
+
+        tmp = [str(x) for x in attributelist]
+        tmpstr = ",".join(tmp)
+        self.contents[name] = "{" + tmpstr + "}"
+
+    def insertAttributeVectorStringList(self, name, attributelist):
+        """Insert a named list of string lists"""
+
+        listOfLists = []
+        for stringList in attributelist:
+            # tmp = map ( lambda x : '"' + x + '"', stringList )
+            tmpstr = ",".join(stringList)
+            listOfLists.append("{" + tmpstr + "}")
+        self.contents[name] = "{" + ",".join(listOfLists) + "}"
+
+    def lookupAttribute(self, name):
+        """Check the presence of the given attribute"""
+
+        return name in self.contents
+
+    def set_expression(self, name, attribute):
+        """Insert a named expression attribute"""
+
+        self.contents[name] = str(attribute)
+
+    def get_expression(self, name):
+        """Get expression corresponding to a named attribute"""
+
+        if name in self.contents:
+            if isinstance(self.contents[name], int):
+                return str(self.contents[name])
+            return self.contents[name]
+        return ""
+
+    def isAttributeList(self, name):
+        """Check if the given attribute is of the List type"""
+        attribute = self.get_expression(name).strip()
+        return attribute.startswith("{")
+
+    def getListFromExpression(self, name):
+        """Get a list of strings from a given expression"""
+
+        tempString = self.get_expression(name).strip()
+        listMode = False
+        if tempString.startswith("{"):
+            tempString = tempString[1:-1]
+            listMode = True
+
+        tempString = tempString.replace(" ", "").replace("\n", "")
+        if tempString.find("{") < 0:
+            if not listMode:
+                tempString = tempString.replace('"', "")
+                if not tempString:
+                    return []
+                return tempString.split(",")
+
+        resultList = []
+        while tempString:
+            if tempString.find("{") == 0:
+                end = tempString.find("}")
+                resultList.append(tempString[: end + 1])
+                tempString = tempString[end + 1 :]
+                if tempString.startswith(","):
+                    tempString = tempString[1:]
+            elif tempString.find('"') == 0:
+                end = tempString[1:].find('"')
+                resultList.append(tempString[1 : end + 1])
+                tempString = tempString[end + 2 :]
+                if tempString.startswith(","):
+                    tempString = tempString[1:]
+            else:
+                end = tempString.find(",")
+                if end < 0:
+                    resultList.append(tempString.replace('"', "").replace(" ", ""))
+                    break
+                else:
+                    resultList.append(tempString[:end].replace('"', "").replace(" ", ""))
+                    tempString = tempString[end + 1 :]
+
+        return resultList
+
+    def getDictionaryFromSubJDL(self, name):
+        """Get a dictionary of the JDL attributes from a subsection"""
+
+        tempList = self.get_expression(name)[1:-1]
+        resDict = {}
+        for item in tempList.split(";"):
+            if len(item.split("=")) == 2:
+                resDict[item.split("=")[0].strip()] = item.split("=")[1].strip().replace('"', "")
+            else:
+                return {}
+
+        return resDict
+
+    def deleteAttribute(self, name):
+        """Delete a named attribute"""
+
+        if name in self.contents:
+            del self.contents[name]
+            return 1
+        return 0
+
+    def isOK(self):
+        """Check the JDL validity - to be defined"""
+
+        if self.contents:
+            return 1
+        return 0
+
+    def asJDL(self):
+        """Convert the JDL description into a string"""
+
+        result = []
+        for name, value in sorted(self.contents.items()):
+            if value[0:1] == "{":
+                result += [4 * " " + name + " = \n"]
+                result += [8 * " " + "{\n"]
+                strings = value[1:-1].split(",")
+                for st in strings:
+                    result += [12 * " " + st.strip() + ",\n"]
+                result[-1] = result[-1][:-2]
+                result += ["\n" + 8 * " " + "};\n"]
+            elif value[0:1] == "[":
+                tempad = ClassAd(value)
+                tempjdl = tempad.asJDL() + ";"
+                lines = tempjdl.split("\n")
+                result += [4 * " " + name + " = \n"]
+                for line in lines:
+                    result += [8 * " " + line + "\n"]
+
+            else:
+                result += [4 * " " + name + " = " + str(value) + ";\n"]
+        if result:
+            result[-1] = result[-1][:-1]
+        return "[ \n" + "".join(result) + "\n]"
+
+    def getAttributeString(self, name):
+        """Get String type attribute value"""
+        value = ""
+        if self.lookupAttribute(name):
+            value = self.get_expression(name).replace('"', "")
+        return value
+
+    def getAttributeInt(self, name):
+        """Get Integer type attribute value"""
+        value = None
+        if self.lookupAttribute(name):
+            try:
+                value = int(self.get_expression(name).replace('"', ""))
+            except Exception:
+                value = None
+        return value
+
+    def getAttributeBool(self, name):
+        """Get Boolean type attribute value"""
+        if not self.lookupAttribute(name):
+            return False
+
+        value = self.get_expression(name).replace('"', "")
+        return value.lower() == "true"
+
+    def getAttributeFloat(self, name):
+        """Get Float type attribute value"""
+        value = None
+        if self.lookupAttribute(name):
+            try:
+                value = float(self.get_expression(name).replace('"', ""))
+            except Exception:
+                value = None
+        return value
+
+    def getAttributes(self) -> list[str]:
+        """Get the list of all the attribute names
+
+        :return: list of names as strings
+        """
+        return list(self.contents)

diraccommon-9.0.0a67/src/DIRACCommon/Core/Utilities/ClassAd/__init__.py

@@ -0,0 +1 @@
+"""ClassAd utilities for DIRACCommon"""

diraccommon-9.0.0a67/src/DIRACCommon/Core/Utilities/JDL.py

@@ -0,0 +1,199 @@
+"""Transformation classes around the JDL format."""
+
+from diraccfg import CFG
+from pydantic import ValidationError
+
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities import List
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
+from DIRACCommon.WorkloadManagementSystem.Utilities.JobModel import BaseJobDescriptionModel
+
+ARGUMENTS = "Arguments"
+BANNED_SITES = "BannedSites"
+CPU_TIME = "CPUTime"
+EXECUTABLE = "Executable"
+EXECUTION_ENVIRONMENT = "ExecutionEnvironment"
+GRID_CE = "GridCE"
+INPUT_DATA = "InputData"
+INPUT_DATA_POLICY = "InputDataPolicy"
+INPUT_SANDBOX = "InputSandbox"
+JOB_CONFIG_ARGS = "JobConfigArgs"
+JOB_TYPE = "JobType"
+JOB_GROUP = "JobGroup"
+LOG_LEVEL = "LogLevel"
+NUMBER_OF_PROCESSORS = "NumberOfProcessors"
+MAX_NUMBER_OF_PROCESSORS = "MaxNumberOfProcessors"
+MIN_NUMBER_OF_PROCESSORS = "MinNumberOfProcessors"
+OUTPUT_DATA = "OutputData"
+OUTPUT_PATH = "OutputPath"
+OUTPUT_SE = "OutputSE"
+PLATFORM = "Platform"
+PRIORITY = "Priority"
+STD_ERROR = "StdError"
+STD_OUTPUT = "StdOutput"
+OUTPUT_SANDBOX = "OutputSandbox"
+JOB_NAME = "JobName"
+SITE = "Site"
+TAGS = "Tags"
+
+OWNER = "Owner"
+OWNER_GROUP = "OwnerGroup"
+VO = "VirtualOrganization"
+
+CREDENTIALS_FIELDS = {OWNER, OWNER_GROUP, VO}
+
+
+def loadJDLAsCFG(jdl):
+    """
+    Load a JDL as CFG
+    """
+
+    def cleanValue(value):
+        value = value.strip()
+        if value[0] == '"':
+            entries = []
+            iPos = 1
+            current = ""
+            state = "in"
+            while iPos < len(value):
+                if value[iPos] == '"':
+                    if state == "in":
+                        entries.append(current)
+                        current = ""
+                        state = "out"
+                    elif state == "out":
+                        current = current.strip()
+                        if current not in (",",):
+                            return S_ERROR("value seems a list but is not separated in commas")
+                        current = ""
+                        state = "in"
+                else:
+                    current += value[iPos]
+                iPos += 1
+            if state == "in":
+                return S_ERROR('value is opened with " but is not closed')
+            return S_OK(", ".join(entries))
+        else:
+            return S_OK(value.replace('"', ""))
+
+    def assignValue(key, value, cfg):
+        key = key.strip()
+        if len(key) == 0:
+            return S_ERROR("Invalid key name")
+        value = value.strip()
+        if not value:
+            return S_ERROR(f"No value for key {key}")
+        if value[0] == "{":
+            if value[-1] != "}":
+                return S_ERROR("Value '%s' seems a list but does not end in '}'" % (value))
+            valList = List.fromChar(value[1:-1])
+            for i in range(len(valList)):
+                result = cleanValue(valList[i])
+                if not result["OK"]:
+                    return S_ERROR(f"Var {key} : {result['Message']}")
+                valList[i] = result["Value"]
+                if valList[i] is None:
+                    return S_ERROR(f"List value '{value}' seems invalid for item {i}")
+            value = ", ".join(valList)
+        else:
+            result = cleanValue(value)
+            if not result["OK"]:
+                return S_ERROR(f"Var {key} : {result['Message']}")
+            nV = result["Value"]
+            if nV is None:
+                return S_ERROR(f"Value '{value} seems invalid")
+            value = nV
+        cfg.setOption(key, value)
+        return S_OK()
+
+    if jdl[0] == "[":
+        iPos = 1
+    else:
+        iPos = 0
+    key = ""
+    value = ""
+    action = "key"
+    insideLiteral = False
+    cfg = CFG()
+    while iPos < len(jdl):
+        char = jdl[iPos]
+        if char == ";" and not insideLiteral:
+            if key.strip():
+                result = assignValue(key, value, cfg)
+                if not result["OK"]:
+                    return result
+            key = ""
+            value = ""
+            action = "key"
+        elif char == "[" and not insideLiteral:
+            key = key.strip()
+            if not key:
+                return S_ERROR("Invalid key in JDL")
+            if value.strip():
+                return S_ERROR(f"Key {key} seems to have a value and open a sub JDL at the same time")
+            result = loadJDLAsCFG(jdl[iPos:])
+            if not result["OK"]:
+                return result
+            subCfg, subPos = result["Value"]
+            cfg.createNewSection(key, contents=subCfg)
+            key = ""
+            value = ""
+            action = "key"
+            insideLiteral = False
+            iPos += subPos
+        elif char == "=" and not insideLiteral:
+            if action == "key":
+                action = "value"
+                insideLiteral = False
+            else:
+                value += char
+        elif char == "]" and not insideLiteral:
+            key = key.strip()
+            if len(key) > 0:
+                result = assignValue(key, value, cfg)
+                if not result["OK"]:
+                    return result
+            return S_OK((cfg, iPos))
+        else:
+            if action == "key":
+                key += char
+            else:
+                value += char
+                if char == '"':
+                    insideLiteral = not insideLiteral
+        iPos += 1
+
+    return S_OK((cfg, iPos))
+
+
+def dumpCFGAsJDL(cfg, level=1, tab="  "):
+    indent = tab * level
+    contents = [f"{tab * (level - 1)}["]
+    sections = cfg.listSections()
+
+    for key in cfg:
+        if key in sections:
+            contents.append(f"{indent}{key} =")
+            contents.append(f"{dumpCFGAsJDL(cfg[key], level + 1, tab)};")
+        else:
+            val = List.fromChar(cfg[key])
+            # Some attributes are never lists
+            if len(val) < 2 or key in [ARGUMENTS, EXECUTABLE, STD_OUTPUT, STD_ERROR]:
+                value = cfg[key]
+                try:
+                    try_value = float(value)
+                    contents.append(f"{tab * level}{key} = {value};")
+                except Exception:
+                    contents.append(f'{tab * level}{key} = "{value}";')
+            else:
+                contents.append(f"{indent}{key} =")
+                contents.append("%s{" % indent)
+                for iPos in range(len(val)):
+                    try:
+                        value = float(val[iPos])
+                    except Exception:
+                        val[iPos] = f'"{val[iPos]}"'
+                contents.append(",\n".join([f"{tab * (level + 1)}{value}" for value in val]))
+                contents.append("%s};" % indent)
+    contents.append(f"{tab * (level - 1)}]")
+    return "\n".join(contents)

diraccommon-9.0.0a67/src/DIRACCommon/Core/Utilities/List.py

@@ -0,0 +1,127 @@
+"""Collection of DIRAC useful list related modules.
+   By default on Error they return None.
+"""
+import random
+import sys
+from typing import Any, TypeVar
+from collections.abc import Iterable
+
+T = TypeVar("T")
+
+
+def uniqueElements(aList: list) -> list:
+    """Utility to retrieve list of unique elements in a list (order is kept)."""
+
+    # Use dict.fromkeys instead of set ensure the order is preserved
+    return list(dict.fromkeys(aList))
+
+
+def appendUnique(aList: list, anObject: Any):
+    """Append to list if object does not exist.
+
+    :param aList: list of elements
+    :param anObject: object you want to append
+    """
+    if anObject not in aList:
+        aList.append(anObject)
+
+
+def fromChar(inputString: str, sepChar: str = ","):
+    """Generates a list splitting a string by the required character(s)
+    resulting string items are stripped and empty items are removed.
+
+    :param inputString: list serialised to string
+    :param sepChar: separator
+    :return: list of strings or None if sepChar has a wrong type
+    """
+    # to prevent getting an empty String as argument
+    if not (isinstance(inputString, str) and isinstance(sepChar, str) and sepChar):
+        return None
+    return [fieldString.strip() for fieldString in inputString.split(sepChar) if len(fieldString.strip()) > 0]
+
+
+def randomize(aList: Iterable[T]) -> list[T]:
+    """Return a randomly sorted list.
+
+    :param aList: list to permute
+    """
+    tmpList = list(aList)
+    random.shuffle(tmpList)
+    return tmpList
+
+
+def pop(aList, popElement):
+    """Pop the first element equal to popElement from the list.
+
+    :param aList: list
+    :type aList: python:list
+    :param popElement: element to pop
+    """
+    if popElement in aList:
+        return aList.pop(aList.index(popElement))
+
+
+def stringListToString(aList: list) -> str:
+    """This function is used for making MySQL queries with a list of string elements.
+
+    :param aList: list to be serialized to string for making queries
+    """
+    return ",".join(f"'{x}'" for x in aList)
+
+
+def intListToString(aList: list) -> str:
+    """This function is used for making MySQL queries with a list of int elements.
+
+    :param aList: list to be serialized to string for making queries
+    """
+    return ",".join(str(x) for x in aList)
+
+
+def getChunk(aList: list, chunkSize: int):
+    """Generator yielding chunk from a list of a size chunkSize.
+
+    :param aList: list to be splitted
+    :param chunkSize: lenght of one chunk
+    :raise: StopIteration
+
+    Usage:
+
+    >>> for chunk in getChunk( aList, chunkSize=10):
+          process( chunk )
+
+    """
+    chunkSize = int(chunkSize)
+    for i in range(0, len(aList), chunkSize):
+        yield aList[i : i + chunkSize]
+
+
+def breakListIntoChunks(aList: list, chunkSize: int):
+    """This function takes a list as input and breaks it into list of size 'chunkSize'.
+       It returns a list of lists.
+
+    :param aList: list of elements
+    :param chunkSize: len of a single chunk
+    :return: list of lists of length of chunkSize
+    :raise: RuntimeError if numberOfFilesInChunk is less than 1
+    """
+    if chunkSize < 1:
+        raise RuntimeError("chunkSize cannot be less than 1")
+    if isinstance(aList, (set, dict, tuple, {}.keys().__class__, {}.items().__class__, {}.values().__class__)):
+        aList = list(aList)
+    return [chunk for chunk in getChunk(aList, chunkSize)]
+
+
+def getIndexInList(anItem: Any, aList: list) -> int:
+    """Return the index of the element x in the list l
+    or sys.maxint if it does not exist
+
+    :param anItem: element to look for
+    :param aList: list to look into
+
+    :return: the index or sys.maxint
+    """
+    # try:
+    if anItem in aList:
+        return aList.index(anItem)
+    else:
+        return sys.maxsize