DIRAC 9.0.0a64__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

DIRAC/Core/Utilities/DErrno.py

@@ -37,320 +37,16 @@
                              DErrno.ERRX : ['An error message for ERRX that is specific to LHCb']}
 
 """
-import os
 import importlib
 import sys
 
-from DIRAC.Core.Utilities.Extensions import extensionsByPriority
-
-# To avoid conflict, the error numbers should be greater than 1000
-# We decided to group the by range of 100 per system
-
-# 1000: Generic
-# 1100: Core
-# 1200: Framework
-# 1300: Interfaces
-# 1400: Config
-# 1500: WMS + Workflow
-# 1600: DMS + StorageManagement
-# 1700: RMS
-# 1800: Accounting + Monitoring
-# 1900: TS + Production
-# 2000: Resources + RSS
-
-# ## Generic (10XX)
-# Python related: 0X
-ETYPE = 1000
-EIMPERR = 1001
-ENOMETH = 1002
-ECONF = 1003
-EVALUE = 1004
-EEEXCEPTION = 1005
-# Files manipulation: 1X
-ECTMPF = 1010
-EOF = 1011
-ERF = 1012
-EWF = 1013
-ESPF = 1014
-
-# ## Core (11XX)
-# Certificates and Proxy: 0X
-EX509 = 1100
-EPROXYFIND = 1101
-EPROXYREAD = 1102
-ECERTFIND = 1103
-ECERTREAD = 1104
-ENOCERT = 1105
-ENOCHAIN = 1106
-ENOPKEY = 1107
-ENOGROUP = 1108
-# DISET: 1X
-EDISET = 1110
-ENOAUTH = 1111
-# 3rd party security: 2X
-E3RDPARTY = 1120
-EVOMS = 1121
-# Databases : 3X
-EDB = 1130
-EMYSQL = 1131
-ESQLA = 1132
-# Message Queues: 4X
-EMQUKN = 1140
-EMQNOM = 1141
-EMQCONN = 1142
-# OpenSearch
-EELNOFOUND = 1146
-# Tokens
-EATOKENFIND = 1150
-EATOKENREAD = 1151
-ETOKENTYPE = 1152
-
-# config
-ESECTION = 1400
-
-# processes
-EEZOMBIE = 1147
-EENOPID = 1148
-
-# ## WMS/Workflow
-EWMSUKN = 1500
-EWMSJDL = 1501
-EWMSRESC = 1502
-EWMSSUBM = 1503
-EWMSJMAN = 1504
-EWMSSTATUS = 1505
-EWMSNOMATCH = 1510
-EWMSPLTVER = 1511
-EWMSNOPILOT = 1550
-
-# ## DMS/StorageManagement (16XX)
-EFILESIZE = 1601
-EGFAL = 1602
-EBADCKS = 1603
-EFCERR = 1604
-
-# ## RMS (17XX)
-ERMSUKN = 1700
-
-# ## TS (19XX)
-ETSUKN = 1900
-ETSDATA = 1901
-
-# ## Resources and RSS (20XX)
-ERESGEN = 2000
-ERESUNA = 2001
-ERESUNK = 2002
-
-# This translates the integer number into the name of the variable
-dErrorCode = {
-    # ## Generic (10XX)
-    # 100X: Python related
-    1000: "ETYPE",
-    1001: "EIMPERR",
-    1002: "ENOMETH",
-    1003: "ECONF",
-    1004: "EVALUE",
-    1005: "EEEXCEPTION",
-    # 101X: Files manipulation
-    1010: "ECTMPF",
-    1011: "EOF",
-    1012: "ERF",
-    1013: "EWF",
-    1014: "ESPF",
-    # ## Core
-    # 110X: Certificates and Proxy
-    1100: "EX509",
-    1101: "EPROXYFIND",
-    1102: "EPROXYREAD",
-    1103: "ECERTFIND",
-    1104: "ECERTREAD",
-    1105: "ENOCERT",
-    1106: "ENOCHAIN",
-    1107: "ENOPKEY",
-    1108: "ENOGROUP",
-    # 111X: DISET
-    1110: "EDISET",
-    1111: "ENOAUTH",
-    # 112X: 3rd party security
-    1120: "E3RDPARTY",
-    1121: "EVOMS",
-    # 113X: Databases
-    1130: "EDB",
-    1131: "EMYSQL",
-    1132: "ESQLA",
-    # 114X: Message Queues
-    1140: "EMQUKN",
-    1141: "EMQNOM",
-    1142: "EMQCONN",
-    # OpenSearch
-    1146: "EELNOFOUND",
-    # 115X: Tokens
-    1150: "EATOKENFIND",
-    1151: "EATOKENREAD",
-    1152: "ETOKENTYPE",
-    # Config
-    1400: "ESECTION",
-    # Processes
-    1147: "EEZOMBIE",
-    1148: "EENOPID",
-    # WMS/Workflow
-    1500: "EWMSUKN",
-    1501: "EWMSJDL",
-    1502: "EWMSRESC",
-    1503: "EWMSSUBM",
-    1504: "EWMSJMAN",
-    1505: "EWMSSTATUS",
-    1510: "EWMSNOMATCH",
-    1511: "EWMSPLTVER",
-    1550: "EWMSNOPILOT",
-    # DMS/StorageManagement
-    1601: "EFILESIZE",
-    1602: "EGFAL",
-    1603: "EBADCKS",
-    1604: "EFCERR",
-    # RMS
-    1700: "ERMSUKN",
-    # Resources and RSS
-    2000: "ERESGEN",
-    2001: "ERESUNA",
-    2002: "ERESUNK",
-    # TS
-    1900: "ETSUKN",
-    1901: "ETSDATA",
-}
-
-
-dStrError = {  # Generic (10XX)
-    # 100X: Python related
-    ETYPE: "Object Type Error",
-    EIMPERR: "Failed to import library",
-    ENOMETH: "No such method or function",
-    ECONF: "Configuration error",
-    EVALUE: "Wrong value passed",
-    EEEXCEPTION: "runtime general exception",
-    # 101X: Files manipulation
-    ECTMPF: "Failed to create temporary file",
-    EOF: "Cannot open file",
-    ERF: "Cannot read from file",
-    EWF: "Cannot write to file",
-    ESPF: "Cannot set permissions to file",
-    # ## Core
-    # 110X: Certificates and Proxy
-    EX509: "Generic Error with X509",
-    EPROXYFIND: "Can't find proxy",
-    EPROXYREAD: "Can't read proxy",
-    ECERTFIND: "Can't find certificate",
-    ECERTREAD: "Can't read certificate",
-    ENOCERT: "No certificate loaded",
-    ENOCHAIN: "No chain loaded",
-    ENOPKEY: "No private key loaded",
-    ENOGROUP: "No DIRAC group",
-    # 111X: DISET
-    EDISET: "DISET Error",
-    ENOAUTH: "Unauthorized query",
-    # 112X: 3rd party security
-    E3RDPARTY: "3rd party security service error",
-    EVOMS: "VOMS Error",
-    # 113X: Databases
-    EDB: "Database Error",
-    EMYSQL: "MySQL Error",
-    ESQLA: "SQLAlchemy Error",
-    # 114X: Message Queues
-    EMQUKN: "Unknown MQ Error",
-    EMQNOM: "No messages",
-    EMQCONN: "MQ connection failure",
-    # 114X OpenSearch
-    EELNOFOUND: "Index not found",
-    # 115X: Tokens
-    EATOKENFIND: "Can't find a bearer access token.",
-    EATOKENREAD: "Can't read a bearer access token.",
-    ETOKENTYPE: "Unsupported access token type.",
-    # Config
-    ESECTION: "Section is not found",
-    # processes
-    EEZOMBIE: "Zombie process",
-    EENOPID: "No PID of process",
-    # WMS/Workflow
-    EWMSUKN: "Unknown WMS error",
-    EWMSJDL: "Invalid job description",
-    EWMSRESC: "Job to reschedule",
-    EWMSSUBM: "Job submission error",
-    EWMSJMAN: "Job management error",
-    EWMSSTATUS: "Job status error",
-    EWMSNOPILOT: "No pilots found",
-    EWMSPLTVER: "Pilot version does not match",
-    EWMSNOMATCH: "No match found",
-    # DMS/StorageManagement
-    EFILESIZE: "Bad file size",
-    EGFAL: "Error with the gfal call",
-    EBADCKS: "Bad checksum",
-    EFCERR: "FileCatalog error",
-    # RMS
-    ERMSUKN: "Unknown RMS error",
-    # Resources and RSS
-    ERESGEN: "Unknown Resource Failure",
-    ERESUNA: "Resource not available",
-    ERESUNK: "Unknown Resource",
-    # TS
-    ETSUKN: "Unknown Transformation System Error",
-    ETSDATA: "Invalid Input Data definition",
-}
+# Import all the stateless parts from DIRACCommon
+from DIRACCommon.Core.Utilities.DErrno import *  # noqa: F401, F403
 
+from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 
-def strerror(code: int) -> str:
-    """This method wraps up os.strerror, and behave the same way.
-    It completes it with the DIRAC specific errors.
-    """
-
-    if code == 0:
-        return "Undefined error"
-
-    errMsg = f"Unknown error {code}"
-
-    try:
-        errMsg = dStrError[code]
-    except KeyError:
-        # It is not a DIRAC specific error, try the os one
-        try:
-            errMsg = os.strerror(code)
-            # On some system, os.strerror raises an exception with unknown code,
-            # on others, it returns a message...
-        except ValueError:
-            pass
-
-    return errMsg
-
-
-def cmpError(inErr, candidate):
-    """This function compares an error (in its old form (a string or dictionary) or in its int form
-    with a candidate error code.
-
-    :param inErr: a string, an integer, a S_ERROR dictionary
-    :type inErr: str or int or S_ERROR
-    :param int candidate: error code to compare with
-
-    :return: True or False
-
-    If an S_ERROR instance is passed, we compare the code with S_ERROR['Errno']
-    If it is a Integer, we do a direct comparison
-    If it is a String, we use strerror to check the error string
-    """
-
-    if isinstance(inErr, str):  # old style
-        # Compare error message strings
-        errMsg = strerror(candidate)
-        return errMsg in inErr
-    elif isinstance(inErr, dict):  # if the S_ERROR structure is given
-        # Check if Errno defined in the dict
-        errorNumber = inErr.get("Errno")
-        if errorNumber:
-            return errorNumber == candidate
-        errMsg = strerror(candidate)
-        return errMsg in inErr.get("Message", "")
-    elif isinstance(inErr, int):
-        return inErr == candidate
-    else:
-        raise TypeError(f"Unknown input error type {type(inErr)}")
+# compatErrorString is used by the extension mechanism but not in DIRACCommon
+compatErrorString = {}
 
 
 def includeExtensionErrors():

DIRAC/Core/Utilities/JDL.py

@@ -1,203 +1,9 @@
 """Transformation classes around the JDL format."""
 
-from diraccfg import CFG
-from pydantic import ValidationError
+from DIRACCommon.Core.Utilities.JDL import *  # noqa: F403,F401
 
-from DIRAC import S_OK, S_ERROR
-from DIRAC.Core.Utilities import List
-from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 from DIRAC.WorkloadManagementSystem.Utilities.JobModel import BaseJobDescriptionModel
 
-ARGUMENTS = "Arguments"
-BANNED_SITES = "BannedSites"
-CPU_TIME = "CPUTime"
-EXECUTABLE = "Executable"
-EXECUTION_ENVIRONMENT = "ExecutionEnvironment"
-GRID_CE = "GridCE"
-INPUT_DATA = "InputData"
-INPUT_DATA_POLICY = "InputDataPolicy"
-INPUT_SANDBOX = "InputSandbox"
-JOB_CONFIG_ARGS = "JobConfigArgs"
-JOB_TYPE = "JobType"
-JOB_GROUP = "JobGroup"
-LOG_LEVEL = "LogLevel"
-NUMBER_OF_PROCESSORS = "NumberOfProcessors"
-MAX_NUMBER_OF_PROCESSORS = "MaxNumberOfProcessors"
-MIN_NUMBER_OF_PROCESSORS = "MinNumberOfProcessors"
-OUTPUT_DATA = "OutputData"
-OUTPUT_PATH = "OutputPath"
-OUTPUT_SE = "OutputSE"
-PLATFORM = "Platform"
-PRIORITY = "Priority"
-STD_ERROR = "StdError"
-STD_OUTPUT = "StdOutput"
-OUTPUT_SANDBOX = "OutputSandbox"
-JOB_NAME = "JobName"
-SITE = "Site"
-TAGS = "Tags"
-
-OWNER = "Owner"
-OWNER_GROUP = "OwnerGroup"
-VO = "VirtualOrganization"
-
-CREDENTIALS_FIELDS = {OWNER, OWNER_GROUP, VO}
-
-
-def loadJDLAsCFG(jdl):
-    """
-    Load a JDL as CFG
-    """
-
-    def cleanValue(value):
-        value = value.strip()
-        if value[0] == '"':
-            entries = []
-            iPos = 1
-            current = ""
-            state = "in"
-            while iPos < len(value):
-                if value[iPos] == '"':
-                    if state == "in":
-                        entries.append(current)
-                        current = ""
-                        state = "out"
-                    elif state == "out":
-                        current = current.strip()
-                        if current not in (",",):
-                            return S_ERROR("value seems a list but is not separated in commas")
-                        current = ""
-                        state = "in"
-                else:
-                    current += value[iPos]
-                iPos += 1
-            if state == "in":
-                return S_ERROR('value is opened with " but is not closed')
-            return S_OK(", ".join(entries))
-        else:
-            return S_OK(value.replace('"', ""))
-
-    def assignValue(key, value, cfg):
-        key = key.strip()
-        if len(key) == 0:
-            return S_ERROR("Invalid key name")
-        value = value.strip()
-        if not value:
-            return S_ERROR(f"No value for key {key}")
-        if value[0] == "{":
-            if value[-1] != "}":
-                return S_ERROR("Value '%s' seems a list but does not end in '}'" % (value))
-            valList = List.fromChar(value[1:-1])
-            for i in range(len(valList)):
-                result = cleanValue(valList[i])
-                if not result["OK"]:
-                    return S_ERROR(f"Var {key} : {result['Message']}")
-                valList[i] = result["Value"]
-                if valList[i] is None:
-                    return S_ERROR(f"List value '{value}' seems invalid for item {i}")
-            value = ", ".join(valList)
-        else:
-            result = cleanValue(value)
-            if not result["OK"]:
-                return S_ERROR(f"Var {key} : {result['Message']}")
-            nV = result["Value"]
-            if nV is None:
-                return S_ERROR(f"Value '{value} seems invalid")
-            value = nV
-        cfg.setOption(key, value)
-        return S_OK()
-
-    if jdl[0] == "[":
-        iPos = 1
-    else:
-        iPos = 0
-    key = ""
-    value = ""
-    action = "key"
-    insideLiteral = False
-    cfg = CFG()
-    while iPos < len(jdl):
-        char = jdl[iPos]
-        if char == ";" and not insideLiteral:
-            if key.strip():
-                result = assignValue(key, value, cfg)
-                if not result["OK"]:
-                    return result
-            key = ""
-            value = ""
-            action = "key"
-        elif char == "[" and not insideLiteral:
-            key = key.strip()
-            if not key:
-                return S_ERROR("Invalid key in JDL")
-            if value.strip():
-                return S_ERROR(f"Key {key} seems to have a value and open a sub JDL at the same time")
-            result = loadJDLAsCFG(jdl[iPos:])
-            if not result["OK"]:
-                return result
-            subCfg, subPos = result["Value"]
-            cfg.createNewSection(key, contents=subCfg)
-            key = ""
-            value = ""
-            action = "key"
-            insideLiteral = False
-            iPos += subPos
-        elif char == "=" and not insideLiteral:
-            if action == "key":
-                action = "value"
-                insideLiteral = False
-            else:
-                value += char
-        elif char == "]" and not insideLiteral:
-            key = key.strip()
-            if len(key) > 0:
-                result = assignValue(key, value, cfg)
-                if not result["OK"]:
-                    return result
-            return S_OK((cfg, iPos))
-        else:
-            if action == "key":
-                key += char
-            else:
-                value += char
-                if char == '"':
-                    insideLiteral = not insideLiteral
-        iPos += 1
-
-    return S_OK((cfg, iPos))
-
-
-def dumpCFGAsJDL(cfg, level=1, tab="  "):
-    indent = tab * level
-    contents = [f"{tab * (level - 1)}["]
-    sections = cfg.listSections()
-
-    for key in cfg:
-        if key in sections:
-            contents.append(f"{indent}{key} =")
-            contents.append(f"{dumpCFGAsJDL(cfg[key], level + 1, tab)};")
-        else:
-            val = List.fromChar(cfg[key])
-            # Some attributes are never lists
-            if len(val) < 2 or key in [ARGUMENTS, EXECUTABLE, STD_OUTPUT, STD_ERROR]:
-                value = cfg[key]
-                try:
-                    try_value = float(value)
-                    contents.append(f"{tab * level}{key} = {value};")
-                except Exception:
-                    contents.append(f'{tab * level}{key} = "{value}";')
-            else:
-                contents.append(f"{indent}{key} =")
-                contents.append("%s{" % indent)
-                for iPos in range(len(val)):
-                    try:
-                        value = float(val[iPos])
-                    except Exception:
-                        val[iPos] = f'"{val[iPos]}"'
-                contents.append(",\n".join([f"{tab * (level + 1)}{value}" for value in val]))
-                contents.append("%s};" % indent)
-    contents.append(f"{tab * (level - 1)}]")
-    return "\n".join(contents)
-
 
 def jdlToBaseJobDescriptionModel(classAd: ClassAd):
     """

DIRAC/Core/Utilities/List.py

@@ -1,127 +1 @@
-"""Collection of DIRAC useful list related modules.
-   By default on Error they return None.
-"""
-import random
-import sys
-from typing import Any, TypeVar
-from collections.abc import Iterable
-
-T = TypeVar("T")
-
-
-def uniqueElements(aList: list) -> list:
-    """Utility to retrieve list of unique elements in a list (order is kept)."""
-
-    # Use dict.fromkeys instead of set ensure the order is preserved
-    return list(dict.fromkeys(aList))
-
-
-def appendUnique(aList: list, anObject: Any):
-    """Append to list if object does not exist.
-
-    :param aList: list of elements
-    :param anObject: object you want to append
-    """
-    if anObject not in aList:
-        aList.append(anObject)
-
-
-def fromChar(inputString: str, sepChar: str = ","):
-    """Generates a list splitting a string by the required character(s)
-    resulting string items are stripped and empty items are removed.
-
-    :param inputString: list serialised to string
-    :param sepChar: separator
-    :return: list of strings or None if sepChar has a wrong type
-    """
-    # to prevent getting an empty String as argument
-    if not (isinstance(inputString, str) and isinstance(sepChar, str) and sepChar):
-        return None
-    return [fieldString.strip() for fieldString in inputString.split(sepChar) if len(fieldString.strip()) > 0]
-
-
-def randomize(aList: Iterable[T]) -> list[T]:
-    """Return a randomly sorted list.
-
-    :param aList: list to permute
-    """
-    tmpList = list(aList)
-    random.shuffle(tmpList)
-    return tmpList
-
-
-def pop(aList, popElement):
-    """Pop the first element equal to popElement from the list.
-
-    :param aList: list
-    :type aList: python:list
-    :param popElement: element to pop
-    """
-    if popElement in aList:
-        return aList.pop(aList.index(popElement))
-
-
-def stringListToString(aList: list) -> str:
-    """This function is used for making MySQL queries with a list of string elements.
-
-    :param aList: list to be serialized to string for making queries
-    """
-    return ",".join(f"'{x}'" for x in aList)
-
-
-def intListToString(aList: list) -> str:
-    """This function is used for making MySQL queries with a list of int elements.
-
-    :param aList: list to be serialized to string for making queries
-    """
-    return ",".join(str(x) for x in aList)
-
-
-def getChunk(aList: list, chunkSize: int):
-    """Generator yielding chunk from a list of a size chunkSize.
-
-    :param aList: list to be splitted
-    :param chunkSize: lenght of one chunk
-    :raise: StopIteration
-
-    Usage:
-
-    >>> for chunk in getChunk( aList, chunkSize=10):
-          process( chunk )
-
-    """
-    chunkSize = int(chunkSize)
-    for i in range(0, len(aList), chunkSize):
-        yield aList[i : i + chunkSize]
-
-
-def breakListIntoChunks(aList: list, chunkSize: int):
-    """This function takes a list as input and breaks it into list of size 'chunkSize'.
-       It returns a list of lists.
-
-    :param aList: list of elements
-    :param chunkSize: len of a single chunk
-    :return: list of lists of length of chunkSize
-    :raise: RuntimeError if numberOfFilesInChunk is less than 1
-    """
-    if chunkSize < 1:
-        raise RuntimeError("chunkSize cannot be less than 1")
-    if isinstance(aList, (set, dict, tuple, {}.keys().__class__, {}.items().__class__, {}.values().__class__)):
-        aList = list(aList)
-    return [chunk for chunk in getChunk(aList, chunkSize)]
-
-
-def getIndexInList(anItem: Any, aList: list) -> int:
-    """Return the index of the element x in the list l
-    or sys.maxint if it does not exist
-
-    :param anItem: element to look for
-    :param aList: list to look into
-
-    :return: the index or sys.maxint
-    """
-    # try:
-    if anItem in aList:
-        return aList.index(anItem)
-    else:
-        return sys.maxsize
+from DIRACCommon.Core.Utilities.List import *  # noqa: F401,F403