DIRACCommon 9.0.0__py3-none-any.whl

DIRACCommon/Core/Utilities/JDL.py

@@ -0,0 +1,199 @@
+"""Transformation classes around the JDL format."""
+
+from diraccfg import CFG
+from pydantic import ValidationError
+
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities import List
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
+from DIRACCommon.WorkloadManagementSystem.Utilities.JobModel import BaseJobDescriptionModel
+
+ARGUMENTS = "Arguments"
+BANNED_SITES = "BannedSites"
+CPU_TIME = "CPUTime"
+EXECUTABLE = "Executable"
+EXECUTION_ENVIRONMENT = "ExecutionEnvironment"
+GRID_CE = "GridCE"
+INPUT_DATA = "InputData"
+INPUT_DATA_POLICY = "InputDataPolicy"
+INPUT_SANDBOX = "InputSandbox"
+JOB_CONFIG_ARGS = "JobConfigArgs"
+JOB_TYPE = "JobType"
+JOB_GROUP = "JobGroup"
+LOG_LEVEL = "LogLevel"
+NUMBER_OF_PROCESSORS = "NumberOfProcessors"
+MAX_NUMBER_OF_PROCESSORS = "MaxNumberOfProcessors"
+MIN_NUMBER_OF_PROCESSORS = "MinNumberOfProcessors"
+OUTPUT_DATA = "OutputData"
+OUTPUT_PATH = "OutputPath"
+OUTPUT_SE = "OutputSE"
+PLATFORM = "Platform"
+PRIORITY = "Priority"
+STD_ERROR = "StdError"
+STD_OUTPUT = "StdOutput"
+OUTPUT_SANDBOX = "OutputSandbox"
+JOB_NAME = "JobName"
+SITE = "Site"
+TAGS = "Tags"
+
+OWNER = "Owner"
+OWNER_GROUP = "OwnerGroup"
+VO = "VirtualOrganization"
+
+CREDENTIALS_FIELDS = {OWNER, OWNER_GROUP, VO}
+
+
+def loadJDLAsCFG(jdl):
+    """
+    Load a JDL as CFG
+    """
+
+    def cleanValue(value):
+        value = value.strip()
+        if value[0] == '"':
+            entries = []
+            iPos = 1
+            current = ""
+            state = "in"
+            while iPos < len(value):
+                if value[iPos] == '"':
+                    if state == "in":
+                        entries.append(current)
+                        current = ""
+                        state = "out"
+                    elif state == "out":
+                        current = current.strip()
+                        if current not in (",",):
+                            return S_ERROR("value seems a list but is not separated in commas")
+                        current = ""
+                        state = "in"
+                else:
+                    current += value[iPos]
+                iPos += 1
+            if state == "in":
+                return S_ERROR('value is opened with " but is not closed')
+            return S_OK(", ".join(entries))
+        else:
+            return S_OK(value.replace('"', ""))
+
+    def assignValue(key, value, cfg):
+        key = key.strip()
+        if len(key) == 0:
+            return S_ERROR("Invalid key name")
+        value = value.strip()
+        if not value:
+            return S_ERROR(f"No value for key {key}")
+        if value[0] == "{":
+            if value[-1] != "}":
+                return S_ERROR("Value '%s' seems a list but does not end in '}'" % (value))
+            valList = List.fromChar(value[1:-1])
+            for i in range(len(valList)):
+                result = cleanValue(valList[i])
+                if not result["OK"]:
+                    return S_ERROR(f"Var {key} : {result['Message']}")
+                valList[i] = result["Value"]
+                if valList[i] is None:
+                    return S_ERROR(f"List value '{value}' seems invalid for item {i}")
+            value = ", ".join(valList)
+        else:
+            result = cleanValue(value)
+            if not result["OK"]:
+                return S_ERROR(f"Var {key} : {result['Message']}")
+            nV = result["Value"]
+            if nV is None:
+                return S_ERROR(f"Value '{value} seems invalid")
+            value = nV
+        cfg.setOption(key, value)
+        return S_OK()
+
+    if jdl[0] == "[":
+        iPos = 1
+    else:
+        iPos = 0
+    key = ""
+    value = ""
+    action = "key"
+    insideLiteral = False
+    cfg = CFG()
+    while iPos < len(jdl):
+        char = jdl[iPos]
+        if char == ";" and not insideLiteral:
+            if key.strip():
+                result = assignValue(key, value, cfg)
+                if not result["OK"]:
+                    return result
+            key = ""
+            value = ""
+            action = "key"
+        elif char == "[" and not insideLiteral:
+            key = key.strip()
+            if not key:
+                return S_ERROR("Invalid key in JDL")
+            if value.strip():
+                return S_ERROR(f"Key {key} seems to have a value and open a sub JDL at the same time")
+            result = loadJDLAsCFG(jdl[iPos:])
+            if not result["OK"]:
+                return result
+            subCfg, subPos = result["Value"]
+            cfg.createNewSection(key, contents=subCfg)
+            key = ""
+            value = ""
+            action = "key"
+            insideLiteral = False
+            iPos += subPos
+        elif char == "=" and not insideLiteral:
+            if action == "key":
+                action = "value"
+                insideLiteral = False
+            else:
+                value += char
+        elif char == "]" and not insideLiteral:
+            key = key.strip()
+            if len(key) > 0:
+                result = assignValue(key, value, cfg)
+                if not result["OK"]:
+                    return result
+            return S_OK((cfg, iPos))
+        else:
+            if action == "key":
+                key += char
+            else:
+                value += char
+                if char == '"':
+                    insideLiteral = not insideLiteral
+        iPos += 1
+
+    return S_OK((cfg, iPos))
+
+
+def dumpCFGAsJDL(cfg, level=1, tab="  "):
+    indent = tab * level
+    contents = [f"{tab * (level - 1)}["]
+    sections = cfg.listSections()
+
+    for key in cfg:
+        if key in sections:
+            contents.append(f"{indent}{key} =")
+            contents.append(f"{dumpCFGAsJDL(cfg[key], level + 1, tab)};")
+        else:
+            val = List.fromChar(cfg[key])
+            # Some attributes are never lists
+            if len(val) < 2 or key in [ARGUMENTS, EXECUTABLE, STD_OUTPUT, STD_ERROR]:
+                value = cfg[key]
+                try:
+                    try_value = float(value)
+                    contents.append(f"{tab * level}{key} = {value};")
+                except Exception:
+                    contents.append(f'{tab * level}{key} = "{value}";')
+            else:
+                contents.append(f"{indent}{key} =")
+                contents.append("%s{" % indent)
+                for iPos in range(len(val)):
+                    try:
+                        value = float(val[iPos])
+                    except Exception:
+                        val[iPos] = f'"{val[iPos]}"'
+                contents.append(",\n".join([f"{tab * (level + 1)}{value}" for value in val]))
+                contents.append("%s};" % indent)
+    contents.append(f"{tab * (level - 1)}]")
+    return "\n".join(contents)

DIRACCommon/Core/Utilities/List.py

@@ -0,0 +1,127 @@
+"""Collection of DIRAC useful list related modules.
+   By default on Error they return None.
+"""
+import random
+import sys
+from typing import Any, TypeVar
+from collections.abc import Iterable
+
+T = TypeVar("T")
+
+
+def uniqueElements(aList: list) -> list:
+    """Utility to retrieve list of unique elements in a list (order is kept)."""
+
+    # Use dict.fromkeys instead of set ensure the order is preserved
+    return list(dict.fromkeys(aList))
+
+
+def appendUnique(aList: list, anObject: Any):
+    """Append to list if object does not exist.
+
+    :param aList: list of elements
+    :param anObject: object you want to append
+    """
+    if anObject not in aList:
+        aList.append(anObject)
+
+
+def fromChar(inputString: str, sepChar: str = ","):
+    """Generates a list splitting a string by the required character(s)
+    resulting string items are stripped and empty items are removed.
+
+    :param inputString: list serialised to string
+    :param sepChar: separator
+    :return: list of strings or None if sepChar has a wrong type
+    """
+    # to prevent getting an empty String as argument
+    if not (isinstance(inputString, str) and isinstance(sepChar, str) and sepChar):
+        return None
+    return [fieldString.strip() for fieldString in inputString.split(sepChar) if len(fieldString.strip()) > 0]
+
+
+def randomize(aList: Iterable[T]) -> list[T]:
+    """Return a randomly sorted list.
+
+    :param aList: list to permute
+    """
+    tmpList = list(aList)
+    random.shuffle(tmpList)
+    return tmpList
+
+
+def pop(aList, popElement):
+    """Pop the first element equal to popElement from the list.
+
+    :param aList: list
+    :type aList: python:list
+    :param popElement: element to pop
+    """
+    if popElement in aList:
+        return aList.pop(aList.index(popElement))
+
+
+def stringListToString(aList: list) -> str:
+    """This function is used for making MySQL queries with a list of string elements.
+
+    :param aList: list to be serialized to string for making queries
+    """
+    return ",".join(f"'{x}'" for x in aList)
+
+
+def intListToString(aList: list) -> str:
+    """This function is used for making MySQL queries with a list of int elements.
+
+    :param aList: list to be serialized to string for making queries
+    """
+    return ",".join(str(x) for x in aList)
+
+
+def getChunk(aList: list, chunkSize: int):
+    """Generator yielding chunk from a list of a size chunkSize.
+
+    :param aList: list to be splitted
+    :param chunkSize: lenght of one chunk
+    :raise: StopIteration
+
+    Usage:
+
+    >>> for chunk in getChunk( aList, chunkSize=10):
+          process( chunk )
+
+    """
+    chunkSize = int(chunkSize)
+    for i in range(0, len(aList), chunkSize):
+        yield aList[i : i + chunkSize]
+
+
+def breakListIntoChunks(aList: list, chunkSize: int):
+    """This function takes a list as input and breaks it into list of size 'chunkSize'.
+       It returns a list of lists.
+
+    :param aList: list of elements
+    :param chunkSize: len of a single chunk
+    :return: list of lists of length of chunkSize
+    :raise: RuntimeError if numberOfFilesInChunk is less than 1
+    """
+    if chunkSize < 1:
+        raise RuntimeError("chunkSize cannot be less than 1")
+    if isinstance(aList, (set, dict, tuple, {}.keys().__class__, {}.items().__class__, {}.values().__class__)):
+        aList = list(aList)
+    return [chunk for chunk in getChunk(aList, chunkSize)]
+
+
+def getIndexInList(anItem: Any, aList: list) -> int:
+    """Return the index of the element x in the list l
+    or sys.maxint if it does not exist
+
+    :param anItem: element to look for
+    :param aList: list to look into
+
+    :return: the index or sys.maxint
+    """
+    # try:
+    if anItem in aList:
+        return aList.index(anItem)
+    else:
+        return sys.maxsize

DIRACCommon/Core/Utilities/ReturnValues.py

@@ -0,0 +1,255 @@
+"""
+   DIRAC return dictionary
+
+   Message values are converted to string
+
+   keys are converted to string
+"""
+from __future__ import annotations
+
+import functools
+import sys
+import traceback
+from types import TracebackType
+from typing import Any, Callable, cast, Generic, Literal, overload, Type, TypeVar, Union
+from typing_extensions import TypedDict, ParamSpec, NotRequired
+
+from DIRACCommon.Core.Utilities.DErrno import strerror
+
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+
+class DOKReturnType(TypedDict, Generic[T]):
+    """used for typing the DIRAC return structure"""
+
+    OK: Literal[True]
+    Value: T
+
+
+class DErrorReturnType(TypedDict):
+    """used for typing the DIRAC return structure"""
+
+    OK: Literal[False]
+    Message: str
+    Errno: int
+    ExecInfo: NotRequired[tuple[type[BaseException], BaseException, TracebackType]]
+    CallStack: NotRequired[list[str]]
+
+
+DReturnType = Union[DOKReturnType[T], DErrorReturnType]
+
+
+def S_ERROR(*args: Any, **kwargs: Any) -> DErrorReturnType:
+    """return value on error condition
+
+    Arguments are either Errno and ErrorMessage or just ErrorMessage fro backward compatibility
+
+    :param int errno: Error number
+    :param string message: Error message
+    :param list callStack: Manually override the CallStack attribute better performance
+    """
+    callStack = kwargs.pop("callStack", None)
+
+    result: DErrorReturnType = {"OK": False, "Errno": 0, "Message": ""}
+
+    message = ""
+    if args:
+        if isinstance(args[0], int):
+            result["Errno"] = args[0]
+            if len(args) > 1:
+                message = args[1]
+        else:
+            message = args[0]
+
+    if result["Errno"]:
+        message = f"{strerror(result['Errno'])} ( {result['Errno']} : {message})"
+    result["Message"] = message
+
+    if callStack is None:
+        try:
+            callStack = traceback.format_stack()
+            callStack.pop()
+        except Exception:
+            callStack = []
+
+    result["CallStack"] = callStack
+
+    return result
+
+
+# mypy doesn't understand default parameter values with generics so use overloads (python/mypy#3737)
+@overload
+def S_OK() -> DOKReturnType[None]:
+    ...
+
+
+@overload
+def S_OK(value: T) -> DOKReturnType[T]:
+    ...
+
+
+def S_OK(value=None):  # type: ignore
+    """return value on success
+
+    :param value: value of the 'Value'
+    :return: dictionary { 'OK' : True, 'Value' : value }
+    """
+    return {"OK": True, "Value": value}
+
+
+def isReturnStructure(unk: Any) -> bool:
+    """Check if value is an `S_OK`/`S_ERROR` object"""
+    if not isinstance(unk, dict):
+        return False
+    if "OK" not in unk:
+        return False
+    if unk["OK"]:
+        return "Value" in unk
+    else:
+        return "Message" in unk
+
+
+def isSError(value: Any) -> bool:
+    """Check if value is an `S_ERROR` object"""
+    if not isinstance(value, dict):
+        return False
+    if "OK" not in value:
+        return False
+    return "Message" in value
+
+
+def reprReturnErrorStructure(struct: DErrorReturnType, full: bool = False) -> str:
+    errorNumber = struct.get("Errno", 0)
+    message = struct.get("Message", "")
+    if errorNumber:
+        reprStr = f"{strerror(errorNumber)} ( {errorNumber} : {message})"
+    else:
+        reprStr = message
+
+    if full:
+        callStack = struct.get("CallStack")
+        if callStack:
+            reprStr += "\n" + "".join(callStack)
+
+    return reprStr
+
+
+def returnSingleResult(dictRes: DReturnType[Any]) -> DReturnType[Any]:
+    """Transform the S_OK{Successful/Failed} dictionary convention into
+    an S_OK/S_ERROR return. To be used when a single returned entity
+    is expected from a generally bulk call.
+
+    :param dictRes: S_ERROR or S_OK( "Failed" : {}, "Successful" : {})
+    :returns: S_ERROR or S_OK(value)
+
+    The following rules are applied:
+
+    - if dictRes is an S_ERROR: returns it as is
+    - we start by looking at the Failed directory
+    - if there are several items in a dictionary, we return the first one
+    - if both dictionaries are empty, we return S_ERROR
+    - For an item in Failed, we return S_ERROR
+    - Far an item in Successful we return S_OK
+
+    Behavior examples (would be perfect unit test :-) )::
+
+      {'Message': 'Kaput', 'OK': False} -> {'Message': 'Kaput', 'OK': False}
+      {'OK': True, 'Value': {'Successful': {}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False}
+      {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {}}} -> {'OK': True, 'Value': 2}
+      {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False}
+      {'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1, 'c': 3}}} -> {'Message': '1', 'OK': False}
+      {'OK': True, 'Value': {'Successful': {'b': 2, 'd': 4}, 'Failed': {}}} -> {'OK': True, 'Value': 2}
+      {'OK': True, 'Value': {'Successful': {}, 'Failed': {}}} ->
+          {'Message': 'returnSingleResult: Failed and Successful dictionaries are empty', 'OK': False}
+    """
+    # if S_ERROR was returned, we return it as well
+    if not dictRes["OK"]:
+        return dictRes
+    # if there is a Failed, we return the first one in an S_ERROR
+    if "Failed" in dictRes["Value"] and len(dictRes["Value"]["Failed"]):
+        errorMessage = list(dictRes["Value"]["Failed"].values())[0]
+        if isinstance(errorMessage, dict):
+            if isReturnStructure(errorMessage):
+                return cast(DErrorReturnType, errorMessage)
+            else:
+                return S_ERROR(str(errorMessage))
+        return S_ERROR(errorMessage)
+    # if there is a Successful, we return the first one in an S_OK
+    elif "Successful" in dictRes["Value"] and len(dictRes["Value"]["Successful"]):
+        return S_OK(list(dictRes["Value"]["Successful"].values())[0])
+    else:
+        return S_ERROR("returnSingleResult: Failed and Successful dictionaries are empty")
+
+
+class SErrorException(Exception):
+    """Exception class for use with `convertToReturnValue`"""
+
+    def __init__(self, result: DErrorReturnType | str, errCode: int = 0):
+        """Create a new exception return value
+
+        If `result` is a `S_ERROR` return it directly else convert it to an
+        appropriate value using `S_ERROR(errCode, result)`.
+
+        :param result: The error to propagate
+        :param errCode: the error code to propagate
+        """
+        if not isSError(result):
+            result = S_ERROR(errCode, result)
+        self.result = cast(DErrorReturnType, result)
+
+
+def returnValueOrRaise(result: DReturnType[T], *, errorCode: int = 0) -> T:
+    """Unwrap an S_OK/S_ERROR response into a value or Exception
+
+    This method assists with using exceptions in DIRAC code by raising
+    :exc:`SErrorException` if `result` is an error. This can then by propagated
+    automatically as an `S_ERROR` by wrapping public facing functions with
+    `@convertToReturnValue`.
+
+    :param result: Result of a DIRAC function which returns `S_OK`/`S_ERROR`
+    :returns: The value associated with the `S_OK` object
+    :raises: If `result["OK"]` is falsey the original exception is re-raised.
+             If no exception is known an :exc:`SErrorException` is raised.
+    """
+    if not result["OK"]:
+        if "ExecInfo" in result:
+            raise result["ExecInfo"][0]
+        else:
+            raise SErrorException(result, errorCode)
+    return result["Value"]
+
+
+def convertToReturnValue(func: Callable[P, T]) -> Callable[P, DReturnType[T]]:
+    """Decorate a function to convert return values to `S_OK`/`S_ERROR`
+
+    If `func` returns, wrap the return value in `S_OK`.
+    If `func` raises :exc:`SErrorException`, return the associated `S_ERROR`
+    If `func` raises any other exception type, convert it to an `S_ERROR` object
+
+    :param result: The bare result of a function call
+    :returns: `S_OK`/`S_ERROR`
+    """
+
+    @functools.wraps(func)
+    def wrapped(*args: P.args, **kwargs: P.kwargs) -> DReturnType[T]:
+        try:
+            value = func(*args, **kwargs)
+        except SErrorException as e:
+            return e.result
+        except Exception as e:
+            retval = S_ERROR(f"{repr(e)}: {e}")
+            # Replace CallStack with the one from the exception
+            # Use cast as mypy doesn't understand that sys.exc_info can't return None in an exception block
+            retval["ExecInfo"] = cast(tuple[type[BaseException], BaseException, TracebackType], sys.exc_info())
+            exc_type, exc_value, exc_tb = retval["ExecInfo"]
+            retval["CallStack"] = traceback.format_tb(exc_tb)
+            return retval
+        else:
+            return S_OK(value)
+
+    # functools will copy the annotations. Since we change the return type
+    # we have to update it
+    wrapped.__annotations__["return"] = DReturnType
+    return wrapped