DIRACCommon 9.0.0__py3-none-any.whl

DIRACCommon/ConfigurationSystem/Client/Helpers/Resources.py

@@ -0,0 +1,52 @@
+"""Platform utilities for DIRAC platform compatibility and management.
+
+This module provides functions for working with DIRAC platforms and OS compatibility.
+"""
+import re
+
+from DIRACCommon.Core.Utilities.ReturnValues import S_ERROR, S_OK, DReturnType
+
+
+def getDIRACPlatform(osList: list[str], osCompatibilityDict: dict[str, set[str]]) -> DReturnType[list[str]]:
+    """Get standard DIRAC platform(s) compatible with the argument.
+
+    NB: The returned value is a list, ordered by numeric components in the platform.
+    In practice the "highest" version (which should be the most "desirable" one is returned first)
+
+    :param list osList: list of platforms defined by resource providers
+    :param dict osCompatibilityDict: dictionary with OS compatibility information
+    :return: a list of DIRAC platforms that can be specified in job descriptions
+    """
+
+    if not osCompatibilityDict:
+        return S_ERROR("OS compatibility info not found")
+
+    # making an OS -> platforms dict
+    os2PlatformDict = dict()
+    for platform, osItems in osCompatibilityDict.items():
+        for osItem in osItems:
+            os2PlatformDict.setdefault(osItem, set()).add(platform)
+
+    platforms = set()
+    for os in osList:
+        platforms |= os2PlatformDict.get(os, set())
+
+    if not platforms:
+        return S_ERROR(f"No compatible DIRAC platform found for {','.join(osList)}")
+
+    return S_OK(sorted(platforms, key=_platformSortKey, reverse=True))
+
+
+def _platformSortKey(version: str) -> list[str]:
+    # Loosely based on distutils.version.LooseVersion
+    parts = []
+    for part in re.split(r"(\d+|[a-z]+|\.| -)", version.lower()):
+        if not part or part == ".":
+            continue
+        if part[:1] in "0123456789":
+            part = part.zfill(8)
+        else:
+            while parts and parts[-1] == "00000000":
+                parts.pop()
+        parts.append(part)
+    return parts

DIRACCommon/ConfigurationSystem/Client/Helpers/__init__.py

@@ -0,0 +1,3 @@
+"""
+DIRACCommon.ConfigurationSystem.Client.Helpers - Configuration system helper functions
+"""

DIRACCommon/ConfigurationSystem/Client/__init__.py

@@ -0,0 +1,3 @@
+"""
+DIRACCommon.ConfigurationSystem.Client - Configuration system client components
+"""

DIRACCommon/ConfigurationSystem/__init__.py

@@ -0,0 +1,3 @@
+"""
+DIRACCommon.ConfigurationSystem - Configuration system components
+"""

DIRACCommon/Core/Utilities/ClassAd/ClassAdLight.py

@@ -0,0 +1,295 @@
+""" ClassAd Class - a light purely Python representation of the
+    Condor ClassAd library.
+"""
+
+
+class ClassAd:
+    def __init__(self, jdl):
+        """ClassAd constructor from a JDL string"""
+        self.contents = {}
+        result = self.__analyse_jdl(jdl)
+        if result:
+            self.contents = result
+
+    def __analyse_jdl(self, jdl, index=0):
+        """Analyse one [] jdl enclosure"""
+
+        jdl = jdl.strip()
+
+        # Strip all the blanks first
+        # temp = jdl.replace(' ','').replace('\n','')
+        temp = jdl
+
+        result = {}
+
+        if temp[0] != "[" or temp[-1] != "]":
+            print("Invalid JDL: it should start with [ and end with ]")
+            return result
+
+        # Parse the jdl string now
+        body = temp[1:-1]
+        index = 0
+        namemode = 1
+        valuemode = 0
+        while index < len(body):
+            if namemode:
+                ind = body.find("=", index)
+                if ind != -1:
+                    name = body[index:ind]
+                    index = ind + 1
+                    valuemode = 1
+                    namemode = 0
+                else:
+                    break
+            elif valuemode:
+                ind1 = body.find("[", index)
+                ind2 = body.find(";", index)
+                if ind1 != -1 and ind1 < ind2:
+                    value, newind = self.__find_subjdl(body, ind1)
+                elif ind1 == -1 and ind2 == -1:
+                    value = body[index:]
+                    newind = len(body)
+                else:
+                    if index == ind2:
+                        return {}
+                    else:
+                        value = body[index:ind2]
+                        newind = ind2 + 1
+
+                result[name.strip()] = value.strip().replace("\n", "")
+                index = newind
+                valuemode = 0
+                namemode = 1
+
+        return result
+
+    def __find_subjdl(self, body, index):
+        """Find a full [] enclosure starting from index"""
+        result = ""
+        if body[index] != "[":
+            return (result, 0)
+
+        depth = 0
+        ind = index
+        while depth < 10:
+            ind1 = body.find("]", ind + 1)
+            ind2 = body.find("[", ind + 1)
+            if ind2 != -1 and ind2 < ind1:
+                depth += 1
+                ind = ind2
+            else:
+                if depth > 0:
+                    depth -= 1
+                    ind = ind1
+                else:
+                    result = body[index : ind1 + 1]
+                    if body[ind1 + 1] == ";":
+                        return (result, ind1 + 2)
+                    return result, 0
+
+        return result, 0
+
+    def insertAttributeInt(self, name, attribute):
+        """Insert a named integer attribute"""
+
+        self.contents[name] = str(attribute)
+
+    def insertAttributeBool(self, name, attribute):
+        """Insert a named boolean attribute"""
+
+        if attribute:
+            self.contents[name] = "true"
+        else:
+            self.contents[name] = "false"
+
+    def insertAttributeString(self, name, attribute):
+        """Insert a named string attribute"""
+
+        self.contents[name] = '"' + str(attribute) + '"'
+
+    def insertAttributeVectorString(self, name, attributelist):
+        """Insert a named string list attribute"""
+
+        tmp = ['"' + x + '"' for x in attributelist]
+        tmpstr = ",".join(tmp)
+        self.contents[name] = "{" + tmpstr + "}"
+
+    def insertAttributeVectorInt(self, name, attributelist):
+        """Insert a named string list attribute"""
+
+        tmp = [str(x) for x in attributelist]
+        tmpstr = ",".join(tmp)
+        self.contents[name] = "{" + tmpstr + "}"
+
+    def insertAttributeVectorStringList(self, name, attributelist):
+        """Insert a named list of string lists"""
+
+        listOfLists = []
+        for stringList in attributelist:
+            # tmp = map ( lambda x : '"' + x + '"', stringList )
+            tmpstr = ",".join(stringList)
+            listOfLists.append("{" + tmpstr + "}")
+        self.contents[name] = "{" + ",".join(listOfLists) + "}"
+
+    def lookupAttribute(self, name):
+        """Check the presence of the given attribute"""
+
+        return name in self.contents
+
+    def set_expression(self, name, attribute):
+        """Insert a named expression attribute"""
+
+        self.contents[name] = str(attribute)
+
+    def get_expression(self, name):
+        """Get expression corresponding to a named attribute"""
+
+        if name in self.contents:
+            if isinstance(self.contents[name], int):
+                return str(self.contents[name])
+            return self.contents[name]
+        return ""
+
+    def isAttributeList(self, name):
+        """Check if the given attribute is of the List type"""
+        attribute = self.get_expression(name).strip()
+        return attribute.startswith("{")
+
+    def getListFromExpression(self, name):
+        """Get a list of strings from a given expression"""
+
+        tempString = self.get_expression(name).strip()
+        listMode = False
+        if tempString.startswith("{"):
+            tempString = tempString[1:-1]
+            listMode = True
+
+        tempString = tempString.replace(" ", "").replace("\n", "")
+        if tempString.find("{") < 0:
+            if not listMode:
+                tempString = tempString.replace('"', "")
+                if not tempString:
+                    return []
+                return tempString.split(",")
+
+        resultList = []
+        while tempString:
+            if tempString.find("{") == 0:
+                end = tempString.find("}")
+                resultList.append(tempString[: end + 1])
+                tempString = tempString[end + 1 :]
+                if tempString.startswith(","):
+                    tempString = tempString[1:]
+            elif tempString.find('"') == 0:
+                end = tempString[1:].find('"')
+                resultList.append(tempString[1 : end + 1])
+                tempString = tempString[end + 2 :]
+                if tempString.startswith(","):
+                    tempString = tempString[1:]
+            else:
+                end = tempString.find(",")
+                if end < 0:
+                    resultList.append(tempString.replace('"', "").replace(" ", ""))
+                    break
+                else:
+                    resultList.append(tempString[:end].replace('"', "").replace(" ", ""))
+                    tempString = tempString[end + 1 :]
+
+        return resultList
+
+    def getDictionaryFromSubJDL(self, name):
+        """Get a dictionary of the JDL attributes from a subsection"""
+
+        tempList = self.get_expression(name)[1:-1]
+        resDict = {}
+        for item in tempList.split(";"):
+            if len(item.split("=")) == 2:
+                resDict[item.split("=")[0].strip()] = item.split("=")[1].strip().replace('"', "")
+            else:
+                return {}
+
+        return resDict
+
+    def deleteAttribute(self, name):
+        """Delete a named attribute"""
+
+        if name in self.contents:
+            del self.contents[name]
+            return 1
+        return 0
+
+    def isOK(self):
+        """Check the JDL validity - to be defined"""
+
+        if self.contents:
+            return 1
+        return 0
+
+    def asJDL(self):
+        """Convert the JDL description into a string"""
+
+        result = []
+        for name, value in sorted(self.contents.items()):
+            if value[0:1] == "{":
+                result += [4 * " " + name + " = \n"]
+                result += [8 * " " + "{\n"]
+                strings = value[1:-1].split(",")
+                for st in strings:
+                    result += [12 * " " + st.strip() + ",\n"]
+                result[-1] = result[-1][:-2]
+                result += ["\n" + 8 * " " + "};\n"]
+            elif value[0:1] == "[":
+                tempad = ClassAd(value)
+                tempjdl = tempad.asJDL() + ";"
+                lines = tempjdl.split("\n")
+                result += [4 * " " + name + " = \n"]
+                for line in lines:
+                    result += [8 * " " + line + "\n"]
+
+            else:
+                result += [4 * " " + name + " = " + str(value) + ";\n"]
+        if result:
+            result[-1] = result[-1][:-1]
+        return "[ \n" + "".join(result) + "\n]"
+
+    def getAttributeString(self, name):
+        """Get String type attribute value"""
+        value = ""
+        if self.lookupAttribute(name):
+            value = self.get_expression(name).replace('"', "")
+        return value
+
+    def getAttributeInt(self, name):
+        """Get Integer type attribute value"""
+        value = None
+        if self.lookupAttribute(name):
+            try:
+                value = int(self.get_expression(name).replace('"', ""))
+            except Exception:
+                value = None
+        return value
+
+    def getAttributeBool(self, name):
+        """Get Boolean type attribute value"""
+        if not self.lookupAttribute(name):
+            return False
+
+        value = self.get_expression(name).replace('"', "")
+        return value.lower() == "true"
+
+    def getAttributeFloat(self, name):
+        """Get Float type attribute value"""
+        value = None
+        if self.lookupAttribute(name):
+            try:
+                value = float(self.get_expression(name).replace('"', ""))
+            except Exception:
+                value = None
+        return value
+
+    def getAttributes(self) -> list[str]:
+        """Get the list of all the attribute names
+
+        :return: list of names as strings
+        """
+        return list(self.contents)

DIRACCommon/Core/Utilities/ClassAd/__init__.py

@@ -0,0 +1 @@
+"""ClassAd utilities for DIRACCommon"""

DIRACCommon/Core/Utilities/DErrno.py

@@ -0,0 +1,327 @@
+""" :mod: DErrno
+
+    ==========================
+
+    .. module: DErrno
+
+    :synopsis: Error list and utilities for handling errors in DIRAC
+
+
+    This module contains list of errors that can be encountered in DIRAC.
+    It complements the errno module of python.
+
+    It also contains utilities to manipulate these errors.
+
+    This is a stateless version extracted to DIRACCommon to avoid circular dependencies.
+    The extension loading functionality has been removed.
+"""
+import os
+
+# To avoid conflict, the error numbers should be greater than 1000
+# We decided to group the by range of 100 per system
+
+# 1000: Generic
+# 1100: Core
+# 1200: Framework
+# 1300: Interfaces
+# 1400: Config
+# 1500: WMS + Workflow
+# 1600: DMS + StorageManagement
+# 1700: RMS
+# 1800: Accounting + Monitoring
+# 1900: TS + Production
+# 2000: Resources + RSS
+
+# ## Generic (10XX)
+# Python related: 0X
+ETYPE = 1000
+EIMPERR = 1001
+ENOMETH = 1002
+ECONF = 1003
+EVALUE = 1004
+EEEXCEPTION = 1005
+# Files manipulation: 1X
+ECTMPF = 1010
+EOF = 1011
+ERF = 1012
+EWF = 1013
+ESPF = 1014
+
+# ## Core (11XX)
+# Certificates and Proxy: 0X
+EX509 = 1100
+EPROXYFIND = 1101
+EPROXYREAD = 1102
+ECERTFIND = 1103
+ECERTREAD = 1104
+ENOCERT = 1105
+ENOCHAIN = 1106
+ENOPKEY = 1107
+ENOGROUP = 1108
+# DISET: 1X
+EDISET = 1110
+ENOAUTH = 1111
+# 3rd party security: 2X
+E3RDPARTY = 1120
+EVOMS = 1121
+# Databases : 3X
+EDB = 1130
+EMYSQL = 1131
+ESQLA = 1132
+# Message Queues: 4X
+EMQUKN = 1140
+EMQNOM = 1141
+EMQCONN = 1142
+# OpenSearch
+EELNOFOUND = 1146
+# Tokens
+EATOKENFIND = 1150
+EATOKENREAD = 1151
+ETOKENTYPE = 1152
+
+# config
+ESECTION = 1400
+
+# processes
+EEZOMBIE = 1147
+EENOPID = 1148
+
+# ## WMS/Workflow
+EWMSUKN = 1500
+EWMSJDL = 1501
+EWMSRESC = 1502
+EWMSSUBM = 1503
+EWMSJMAN = 1504
+EWMSSTATUS = 1505
+EWMSNOMATCH = 1510
+EWMSPLTVER = 1511
+EWMSNOPILOT = 1550
+
+# ## DMS/StorageManagement (16XX)
+EFILESIZE = 1601
+EGFAL = 1602
+EBADCKS = 1603
+EFCERR = 1604
+
+# ## RMS (17XX)
+ERMSUKN = 1700
+
+# ## TS (19XX)
+ETSUKN = 1900
+ETSDATA = 1901
+
+# ## Resources and RSS (20XX)
+ERESGEN = 2000
+ERESUNA = 2001
+ERESUNK = 2002
+
+# This translates the integer number into the name of the variable
+dErrorCode = {
+    # ## Generic (10XX)
+    # 100X: Python related
+    1000: "ETYPE",
+    1001: "EIMPERR",
+    1002: "ENOMETH",
+    1003: "ECONF",
+    1004: "EVALUE",
+    1005: "EEEXCEPTION",
+    # 101X: Files manipulation
+    1010: "ECTMPF",
+    1011: "EOF",
+    1012: "ERF",
+    1013: "EWF",
+    1014: "ESPF",
+    # ## Core
+    # 110X: Certificates and Proxy
+    1100: "EX509",
+    1101: "EPROXYFIND",
+    1102: "EPROXYREAD",
+    1103: "ECERTFIND",
+    1104: "ECERTREAD",
+    1105: "ENOCERT",
+    1106: "ENOCHAIN",
+    1107: "ENOPKEY",
+    1108: "ENOGROUP",
+    # 111X: DISET
+    1110: "EDISET",
+    1111: "ENOAUTH",
+    # 112X: 3rd party security
+    1120: "E3RDPARTY",
+    1121: "EVOMS",
+    # 113X: Databases
+    1130: "EDB",
+    1131: "EMYSQL",
+    1132: "ESQLA",
+    # 114X: Message Queues
+    1140: "EMQUKN",
+    1141: "EMQNOM",
+    1142: "EMQCONN",
+    # OpenSearch
+    1146: "EELNOFOUND",
+    # 115X: Tokens
+    1150: "EATOKENFIND",
+    1151: "EATOKENREAD",
+    1152: "ETOKENTYPE",
+    # Config
+    1400: "ESECTION",
+    # Processes
+    1147: "EEZOMBIE",
+    1148: "EENOPID",
+    # WMS/Workflow
+    1500: "EWMSUKN",
+    1501: "EWMSJDL",
+    1502: "EWMSRESC",
+    1503: "EWMSSUBM",
+    1504: "EWMSJMAN",
+    1505: "EWMSSTATUS",
+    1510: "EWMSNOMATCH",
+    1511: "EWMSPLTVER",
+    1550: "EWMSNOPILOT",
+    # DMS/StorageManagement
+    1601: "EFILESIZE",
+    1602: "EGFAL",
+    1603: "EBADCKS",
+    1604: "EFCERR",
+    # RMS
+    1700: "ERMSUKN",
+    # Resources and RSS
+    2000: "ERESGEN",
+    2001: "ERESUNA",
+    2002: "ERESUNK",
+    # TS
+    1900: "ETSUKN",
+    1901: "ETSDATA",
+}
+
+
+dStrError = {  # Generic (10XX)
+    # 100X: Python related
+    ETYPE: "Object Type Error",
+    EIMPERR: "Failed to import library",
+    ENOMETH: "No such method or function",
+    ECONF: "Configuration error",
+    EVALUE: "Wrong value passed",
+    EEEXCEPTION: "runtime general exception",
+    # 101X: Files manipulation
+    ECTMPF: "Failed to create temporary file",
+    EOF: "Cannot open file",
+    ERF: "Cannot read from file",
+    EWF: "Cannot write to file",
+    ESPF: "Cannot set permissions to file",
+    # ## Core
+    # 110X: Certificates and Proxy
+    EX509: "Generic Error with X509",
+    EPROXYFIND: "Can't find proxy",
+    EPROXYREAD: "Can't read proxy",
+    ECERTFIND: "Can't find certificate",
+    ECERTREAD: "Can't read certificate",
+    ENOCERT: "No certificate loaded",
+    ENOCHAIN: "No chain loaded",
+    ENOPKEY: "No private key loaded",
+    ENOGROUP: "No DIRAC group",
+    # 111X: DISET
+    EDISET: "DISET Error",
+    ENOAUTH: "Unauthorized query",
+    # 112X: 3rd party security
+    E3RDPARTY: "3rd party security service error",
+    EVOMS: "VOMS Error",
+    # 113X: Databases
+    EDB: "Database Error",
+    EMYSQL: "MySQL Error",
+    ESQLA: "SQLAlchemy Error",
+    # 114X: Message Queues
+    EMQUKN: "Unknown MQ Error",
+    EMQNOM: "No messages",
+    EMQCONN: "MQ connection failure",
+    # 114X OpenSearch
+    EELNOFOUND: "Index not found",
+    # 115X: Tokens
+    EATOKENFIND: "Can't find a bearer access token.",
+    EATOKENREAD: "Can't read a bearer access token.",
+    ETOKENTYPE: "Unsupported access token type.",
+    # Config
+    ESECTION: "Section is not found",
+    # processes
+    EEZOMBIE: "Zombie process",
+    EENOPID: "No PID of process",
+    # WMS/Workflow
+    EWMSUKN: "Unknown WMS error",
+    EWMSJDL: "Invalid job description",
+    EWMSRESC: "Job to reschedule",
+    EWMSSUBM: "Job submission error",
+    EWMSJMAN: "Job management error",
+    EWMSSTATUS: "Job status error",
+    EWMSNOPILOT: "No pilots found",
+    EWMSPLTVER: "Pilot version does not match",
+    EWMSNOMATCH: "No match found",
+    # DMS/StorageManagement
+    EFILESIZE: "Bad file size",
+    EGFAL: "Error with the gfal call",
+    EBADCKS: "Bad checksum",
+    EFCERR: "FileCatalog error",
+    # RMS
+    ERMSUKN: "Unknown RMS error",
+    # Resources and RSS
+    ERESGEN: "Unknown Resource Failure",
+    ERESUNA: "Resource not available",
+    ERESUNK: "Unknown Resource",
+    # TS
+    ETSUKN: "Unknown Transformation System Error",
+    ETSDATA: "Invalid Input Data definition",
+}
+
+
+def strerror(code: int) -> str:
+    """This method wraps up os.strerror, and behave the same way.
+    It completes it with the DIRAC specific errors.
+    """
+
+    if code == 0:
+        return "Undefined error"
+
+    errMsg = f"Unknown error {code}"
+
+    try:
+        errMsg = dStrError[code]
+    except KeyError:
+        # It is not a DIRAC specific error, try the os one
+        try:
+            errMsg = os.strerror(code)
+            # On some system, os.strerror raises an exception with unknown code,
+            # on others, it returns a message...
+        except ValueError:
+            pass
+
+    return errMsg
+
+
+def cmpError(inErr: str | int | dict, candidate: int) -> bool:
+    """This function compares an error (in its old form (a string or dictionary) or in its int form
+    with a candidate error code.
+
+    :param inErr: a string, an integer, a S_ERROR dictionary
+    :type inErr: str or int or S_ERROR
+    :param int candidate: error code to compare with
+
+    :return: True or False
+
+    If an S_ERROR instance is passed, we compare the code with S_ERROR['Errno']
+    If it is a Integer, we do a direct comparison
+    If it is a String, we use strerror to check the error string
+    """
+
+    if isinstance(inErr, str):  # old style
+        # Compare error message strings
+        errMsg = strerror(candidate)
+        return errMsg in inErr
+    elif isinstance(inErr, dict):  # if the S_ERROR structure is given
+        # Check if Errno defined in the dict
+        errorNumber = inErr.get("Errno")
+        if errorNumber:
+            return errorNumber == candidate
+        errMsg = strerror(candidate)
+        return errMsg in inErr.get("Message", "")
+    elif isinstance(inErr, int):
+        return inErr == candidate
+    else:
+        raise TypeError(f"Unknown input error type {type(inErr)}")