DIRAC 9.0.0a66__py3-none-any.whl → 9.0.0a68__py3-none-any.whl

DIRAC/ConfigurationSystem/Client/Helpers/Resources.py

@@ -1,11 +1,14 @@
 """ Helper for the CS Resources section
 """
-import re
 from urllib import parse
 
 from DIRAC import S_ERROR, S_OK, gConfig, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
 from DIRAC.Core.Utilities.List import fromChar, uniqueElements
+from DIRACCommon.ConfigurationSystem.Client.Helpers.Resources import (
+    getDIRACPlatform as _getDIRACPlatform,
+    _platformSortKey,
+)
 
 gBaseResourcesSection = "/Resources"
 
@@ -328,8 +331,8 @@ def getCompatiblePlatforms(originalPlatforms):
     if not (result["OK"] and result["Value"]):
         return S_ERROR("OS compatibility info not found")
 
-    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}  # can be an iterator
-    for k, v in platformsDict.items():  # can be an iterator
+    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}
+    for k, v in platformsDict.items():
         if k not in v:
             v.append(k)
 
@@ -355,7 +358,6 @@ def getDIRACPlatform(OSList):
     :param list OSList: list of platforms defined by resource providers
     :return: a list of DIRAC platforms that can be specified in job descriptions
     """
-
     # For backward compatibility allow a single string argument
     osList = OSList
     if isinstance(OSList, str):
@@ -365,31 +367,12 @@ def getDIRACPlatform(OSList):
     if not (result["OK"] and result["Value"]):
         return S_ERROR("OS compatibility info not found")
 
-    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}  # can be an iterator
-    for k, v in platformsDict.items():  # can be an iterator
+    platformsDict = {k: set(v.replace(" ", "").split(",")) for k, v in result["Value"].items()}
+    for k, v in platformsDict.items():
         if k not in v:
-            v.append(k)
-
-    # making an OS -> platforms dict
-    os2PlatformDict = dict()
-    for platform, osItems in platformsDict.items():  # can be an iterator
-        for osItem in osItems:
-            if os2PlatformDict.get(osItem):
-                os2PlatformDict[osItem].append(platform)
-            else:
-                os2PlatformDict[osItem] = [platform]
-
-    platforms = []
-    for os in osList:
-        if os in os2PlatformDict:
-            platforms += os2PlatformDict[os]
-
-    if not platforms:
-        return S_ERROR(f"No compatible DIRAC platform found for {','.join(OSList)}")
+            v.add(k)
 
-    platforms.sort(key=_platformSortKey, reverse=True)
-
-    return S_OK(platforms)
+    return _getDIRACPlatform(osList, platformsDict)
 
 
 def getDIRACPlatforms():
@@ -451,7 +434,7 @@ def getInfoAboutProviders(of=None, providerName=None, option="", section=""):
             result = gConfig.getConfigurationTree(relPath)
             if not result["OK"]:
                 return result
-            for key, value in result["Value"].items():  # can be an iterator
+            for key, value in result["Value"].items():
                 if value:
                     resDict[key.replace(relPath, "")] = value
             return S_OK(resDict)
@@ -459,18 +442,3 @@ def getInfoAboutProviders(of=None, providerName=None, option="", section=""):
             return gConfig.getSections(f"{gBaseResourcesSection}/{of}Providers/{providerName}/{section}/")
     else:
         return S_OK(gConfig.getValue(f"{gBaseResourcesSection}/{of}Providers/{providerName}/{section}/{option}"))
-
-
-def _platformSortKey(version: str) -> list[str]:
-    # Loosely based on distutils.version.LooseVersion
-    parts = []
-    for part in re.split(r"(\d+|[a-z]+|\.| -)", version.lower()):
-        if not part or part == ".":
-            continue
-        if part[:1] in "0123456789":
-            part = part.zfill(8)
-        else:
-            while parts and parts[-1] == "00000000":
-                parts.pop()
-        parts.append(part)
-    return parts

DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py

@@ -9,7 +9,6 @@ from DIRAC.ConfigurationSystem.Client import ConfigurationData
 from DIRAC.ConfigurationSystem.Client.Helpers.Resources import (
     getDIRACPlatform,
     getCompatiblePlatforms,
-    _platformSortKey,
     getQueue,
 )
 
@@ -77,21 +76,6 @@ def test_getDIRACPlatform(mocker, mockGCReplyInput, requested, expectedRes, expe
         assert set(res["Value"]) == set(expectedValue), res["Value"]
 
 
-@pytest.mark.parametrize(
-    "string,expected",
-    [
-        ("Darwin_arm64_12.4", ["darwin", "_", "arm", "64", "_", "12", "4"]),
-        ("Linux_x86_64_glibc-2.17", ["linux", "_", "x", "86", "_", "64", "_", "glibc", "-", "2", "17"]),
-        ("Linux_aarch64_glibc-2.28", ["linux", "_", "aarch", "64", "_", "glibc", "-", "2", "28"]),
-    ],
-)
-def test_platformSortKey(string, expected):
-    actual = _platformSortKey(string)
-    for a, e in zip_longest(actual, expected):
-        # Numbers are padded with zeros so string comparison works
-        assert a.lstrip("0") == e
-
-
 @pytest.mark.parametrize(
     "mockGCReplyInput, requested, expectedRes, expectedValue",
     [

DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py

@@ -594,7 +594,7 @@ class VOMS2CSSynchronizer:
 
         # Try to fill in the DiracX section
         if self.useIAM:
-            iam_subs = self.iamSrv.getUsersSub()
+            iam_subs = self.iamSrv.getUsersSub(self.vo)
             diracx_vo_config = {"DiracX": {"CsSync": {"VOs": {self.vo: {"UserSubjects": iam_subs}}}}}
             iam_sub_cfg = CFG()
             iam_sub_cfg.loadFromDict(diracx_vo_config)

DIRAC/Core/Security/IAMService.py

@@ -144,7 +144,7 @@ class IAMService:
         result = S_OK({"Users": users, "Errors": errors})
         return result
 
-    def getUsersSub(self) -> dict[str, str]:
+    def getUsersSub(self, vo=None) -> dict[str, str]:
         """
         Return the mapping based on IAM sub:
         {nickname : sub}
@@ -152,6 +152,7 @@ class IAMService:
         iam_users_raw = self._getIamUserDump()
         diracx_user_section = {}
         for user_info in iam_users_raw:
+            userGroups = [grp["display"] for grp in user_info.get("groups", [])]
             # The nickname is available in the list of attributes
             # (if configured so)
             # in the form {'name': 'nickname', 'value': 'chaen'}
@@ -165,8 +166,8 @@ class IAMService:
             except (KeyError, IndexError):
                 nickname = user_info["userName"]
             sub = user_info["id"]
-
-            diracx_user_section[nickname] = sub
+            if not vo or vo in userGroups:
+                diracx_user_section[nickname] = sub
         # reorder it
         diracx_user_section = dict(sorted(diracx_user_section.items()))
 

DIRAC/Core/Utilities/ClassAd/ClassAdLight.py

@@ -2,294 +2,8 @@
     Condor ClassAd library.
 """
 
+# Import from DIRACCommon for backward compatibility
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 
-class ClassAd:
-    def __init__(self, jdl):
-        """ClassAd constructor from a JDL string"""
-        self.contents = {}
-        result = self.__analyse_jdl(jdl)
-        if result:
-            self.contents = result
-
-    def __analyse_jdl(self, jdl, index=0):
-        """Analyse one [] jdl enclosure"""
-
-        jdl = jdl.strip()
-
-        # Strip all the blanks first
-        # temp = jdl.replace(' ','').replace('\n','')
-        temp = jdl
-
-        result = {}
-
-        if temp[0] != "[" or temp[-1] != "]":
-            print("Invalid JDL: it should start with [ and end with ]")
-            return result
-
-        # Parse the jdl string now
-        body = temp[1:-1]
-        index = 0
-        namemode = 1
-        valuemode = 0
-        while index < len(body):
-            if namemode:
-                ind = body.find("=", index)
-                if ind != -1:
-                    name = body[index:ind]
-                    index = ind + 1
-                    valuemode = 1
-                    namemode = 0
-                else:
-                    break
-            elif valuemode:
-                ind1 = body.find("[", index)
-                ind2 = body.find(";", index)
-                if ind1 != -1 and ind1 < ind2:
-                    value, newind = self.__find_subjdl(body, ind1)
-                elif ind1 == -1 and ind2 == -1:
-                    value = body[index:]
-                    newind = len(body)
-                else:
-                    if index == ind2:
-                        return {}
-                    else:
-                        value = body[index:ind2]
-                        newind = ind2 + 1
-
-                result[name.strip()] = value.strip().replace("\n", "")
-                index = newind
-                valuemode = 0
-                namemode = 1
-
-        return result
-
-    def __find_subjdl(self, body, index):
-        """Find a full [] enclosure starting from index"""
-        result = ""
-        if body[index] != "[":
-            return (result, 0)
-
-        depth = 0
-        ind = index
-        while depth < 10:
-            ind1 = body.find("]", ind + 1)
-            ind2 = body.find("[", ind + 1)
-            if ind2 != -1 and ind2 < ind1:
-                depth += 1
-                ind = ind2
-            else:
-                if depth > 0:
-                    depth -= 1
-                    ind = ind1
-                else:
-                    result = body[index : ind1 + 1]
-                    if body[ind1 + 1] == ";":
-                        return (result, ind1 + 2)
-                    return result, 0
-
-        return result, 0
-
-    def insertAttributeInt(self, name, attribute):
-        """Insert a named integer attribute"""
-
-        self.contents[name] = str(attribute)
-
-    def insertAttributeBool(self, name, attribute):
-        """Insert a named boolean attribute"""
-
-        if attribute:
-            self.contents[name] = "true"
-        else:
-            self.contents[name] = "false"
-
-    def insertAttributeString(self, name, attribute):
-        """Insert a named string attribute"""
-
-        self.contents[name] = '"' + str(attribute) + '"'
-
-    def insertAttributeVectorString(self, name, attributelist):
-        """Insert a named string list attribute"""
-
-        tmp = ['"' + x + '"' for x in attributelist]
-        tmpstr = ",".join(tmp)
-        self.contents[name] = "{" + tmpstr + "}"
-
-    def insertAttributeVectorInt(self, name, attributelist):
-        """Insert a named string list attribute"""
-
-        tmp = [str(x) for x in attributelist]
-        tmpstr = ",".join(tmp)
-        self.contents[name] = "{" + tmpstr + "}"
-
-    def insertAttributeVectorStringList(self, name, attributelist):
-        """Insert a named list of string lists"""
-
-        listOfLists = []
-        for stringList in attributelist:
-            # tmp = map ( lambda x : '"' + x + '"', stringList )
-            tmpstr = ",".join(stringList)
-            listOfLists.append("{" + tmpstr + "}")
-        self.contents[name] = "{" + ",".join(listOfLists) + "}"
-
-    def lookupAttribute(self, name):
-        """Check the presence of the given attribute"""
-
-        return name in self.contents
-
-    def set_expression(self, name, attribute):
-        """Insert a named expression attribute"""
-
-        self.contents[name] = str(attribute)
-
-    def get_expression(self, name):
-        """Get expression corresponding to a named attribute"""
-
-        if name in self.contents:
-            if isinstance(self.contents[name], int):
-                return str(self.contents[name])
-            return self.contents[name]
-        return ""
-
-    def isAttributeList(self, name):
-        """Check if the given attribute is of the List type"""
-        attribute = self.get_expression(name).strip()
-        return attribute.startswith("{")
-
-    def getListFromExpression(self, name):
-        """Get a list of strings from a given expression"""
-
-        tempString = self.get_expression(name).strip()
-        listMode = False
-        if tempString.startswith("{"):
-            tempString = tempString[1:-1]
-            listMode = True
-
-        tempString = tempString.replace(" ", "").replace("\n", "")
-        if tempString.find("{") < 0:
-            if not listMode:
-                tempString = tempString.replace('"', "")
-                if not tempString:
-                    return []
-                return tempString.split(",")
-
-        resultList = []
-        while tempString:
-            if tempString.find("{") == 0:
-                end = tempString.find("}")
-                resultList.append(tempString[: end + 1])
-                tempString = tempString[end + 1 :]
-                if tempString.startswith(","):
-                    tempString = tempString[1:]
-            elif tempString.find('"') == 0:
-                end = tempString[1:].find('"')
-                resultList.append(tempString[1 : end + 1])
-                tempString = tempString[end + 2 :]
-                if tempString.startswith(","):
-                    tempString = tempString[1:]
-            else:
-                end = tempString.find(",")
-                if end < 0:
-                    resultList.append(tempString.replace('"', "").replace(" ", ""))
-                    break
-                else:
-                    resultList.append(tempString[:end].replace('"', "").replace(" ", ""))
-                    tempString = tempString[end + 1 :]
-
-        return resultList
-
-    def getDictionaryFromSubJDL(self, name):
-        """Get a dictionary of the JDL attributes from a subsection"""
-
-        tempList = self.get_expression(name)[1:-1]
-        resDict = {}
-        for item in tempList.split(";"):
-            if len(item.split("=")) == 2:
-                resDict[item.split("=")[0].strip()] = item.split("=")[1].strip().replace('"', "")
-            else:
-                return {}
-
-        return resDict
-
-    def deleteAttribute(self, name):
-        """Delete a named attribute"""
-
-        if name in self.contents:
-            del self.contents[name]
-            return 1
-        return 0
-
-    def isOK(self):
-        """Check the JDL validity - to be defined"""
-
-        if self.contents:
-            return 1
-        return 0
-
-    def asJDL(self):
-        """Convert the JDL description into a string"""
-
-        result = []
-        for name, value in sorted(self.contents.items()):
-            if value[0:1] == "{":
-                result += [4 * " " + name + " = \n"]
-                result += [8 * " " + "{\n"]
-                strings = value[1:-1].split(",")
-                for st in strings:
-                    result += [12 * " " + st.strip() + ",\n"]
-                result[-1] = result[-1][:-2]
-                result += ["\n" + 8 * " " + "};\n"]
-            elif value[0:1] == "[":
-                tempad = ClassAd(value)
-                tempjdl = tempad.asJDL() + ";"
-                lines = tempjdl.split("\n")
-                result += [4 * " " + name + " = \n"]
-                for line in lines:
-                    result += [8 * " " + line + "\n"]
-
-            else:
-                result += [4 * " " + name + " = " + str(value) + ";\n"]
-        if result:
-            result[-1] = result[-1][:-1]
-        return "[ \n" + "".join(result) + "\n]"
-
-    def getAttributeString(self, name):
-        """Get String type attribute value"""
-        value = ""
-        if self.lookupAttribute(name):
-            value = self.get_expression(name).replace('"', "")
-        return value
-
-    def getAttributeInt(self, name):
-        """Get Integer type attribute value"""
-        value = None
-        if self.lookupAttribute(name):
-            try:
-                value = int(self.get_expression(name).replace('"', ""))
-            except Exception:
-                value = None
-        return value
-
-    def getAttributeBool(self, name):
-        """Get Boolean type attribute value"""
-        if not self.lookupAttribute(name):
-            return False
-
-        value = self.get_expression(name).replace('"', "")
-        return value.lower() == "true"
-
-    def getAttributeFloat(self, name):
-        """Get Float type attribute value"""
-        value = None
-        if self.lookupAttribute(name):
-            try:
-                value = float(self.get_expression(name).replace('"', ""))
-            except Exception:
-                value = None
-        return value
-
-    def getAttributes(self) -> list[str]:
-        """Get the list of all the attribute names
-
-        :return: list of names as strings
-        """
-        return list(self.contents)
+# Re-export for backward compatibility
+__all__ = ["ClassAd"]

DIRAC/Core/Utilities/DErrno.py

@@ -41,7 +41,7 @@ import importlib
 import sys
 
 # Import all the stateless parts from DIRACCommon
-from DIRACCommon.Utils.DErrno import *  # noqa: F401, F403
+from DIRACCommon.Core.Utilities.DErrno import *  # noqa: F401, F403
 
 from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 

DIRAC/Core/Utilities/JDL.py

@@ -1,203 +1,9 @@
 """Transformation classes around the JDL format."""
 
-from diraccfg import CFG
-from pydantic import ValidationError
+from DIRACCommon.Core.Utilities.JDL import *  # noqa: F403,F401
 
-from DIRAC import S_OK, S_ERROR
-from DIRAC.Core.Utilities import List
-from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 from DIRAC.WorkloadManagementSystem.Utilities.JobModel import BaseJobDescriptionModel
 
-ARGUMENTS = "Arguments"
-BANNED_SITES = "BannedSites"
-CPU_TIME = "CPUTime"
-EXECUTABLE = "Executable"
-EXECUTION_ENVIRONMENT = "ExecutionEnvironment"
-GRID_CE = "GridCE"
-INPUT_DATA = "InputData"
-INPUT_DATA_POLICY = "InputDataPolicy"
-INPUT_SANDBOX = "InputSandbox"
-JOB_CONFIG_ARGS = "JobConfigArgs"
-JOB_TYPE = "JobType"
-JOB_GROUP = "JobGroup"
-LOG_LEVEL = "LogLevel"
-NUMBER_OF_PROCESSORS = "NumberOfProcessors"
-MAX_NUMBER_OF_PROCESSORS = "MaxNumberOfProcessors"
-MIN_NUMBER_OF_PROCESSORS = "MinNumberOfProcessors"
-OUTPUT_DATA = "OutputData"
-OUTPUT_PATH = "OutputPath"
-OUTPUT_SE = "OutputSE"
-PLATFORM = "Platform"
-PRIORITY = "Priority"
-STD_ERROR = "StdError"
-STD_OUTPUT = "StdOutput"
-OUTPUT_SANDBOX = "OutputSandbox"
-JOB_NAME = "JobName"
-SITE = "Site"
-TAGS = "Tags"
-
-OWNER = "Owner"
-OWNER_GROUP = "OwnerGroup"
-VO = "VirtualOrganization"
-
-CREDENTIALS_FIELDS = {OWNER, OWNER_GROUP, VO}
-
-
-def loadJDLAsCFG(jdl):
-    """
-    Load a JDL as CFG
-    """
-
-    def cleanValue(value):
-        value = value.strip()
-        if value[0] == '"':
-            entries = []
-            iPos = 1
-            current = ""
-            state = "in"
-            while iPos < len(value):
-                if value[iPos] == '"':
-                    if state == "in":
-                        entries.append(current)
-                        current = ""
-                        state = "out"
-                    elif state == "out":
-                        current = current.strip()
-                        if current not in (",",):
-                            return S_ERROR("value seems a list but is not separated in commas")
-                        current = ""
-                        state = "in"
-                else:
-                    current += value[iPos]
-                iPos += 1
-            if state == "in":
-                return S_ERROR('value is opened with " but is not closed')
-            return S_OK(", ".join(entries))
-        else:
-            return S_OK(value.replace('"', ""))
-
-    def assignValue(key, value, cfg):
-        key = key.strip()
-        if len(key) == 0:
-            return S_ERROR("Invalid key name")
-        value = value.strip()
-        if not value:
-            return S_ERROR(f"No value for key {key}")
-        if value[0] == "{":
-            if value[-1] != "}":
-                return S_ERROR("Value '%s' seems a list but does not end in '}'" % (value))
-            valList = List.fromChar(value[1:-1])
-            for i in range(len(valList)):
-                result = cleanValue(valList[i])
-                if not result["OK"]:
-                    return S_ERROR(f"Var {key} : {result['Message']}")
-                valList[i] = result["Value"]
-                if valList[i] is None:
-                    return S_ERROR(f"List value '{value}' seems invalid for item {i}")
-            value = ", ".join(valList)
-        else:
-            result = cleanValue(value)
-            if not result["OK"]:
-                return S_ERROR(f"Var {key} : {result['Message']}")
-            nV = result["Value"]
-            if nV is None:
-                return S_ERROR(f"Value '{value} seems invalid")
-            value = nV
-        cfg.setOption(key, value)
-        return S_OK()
-
-    if jdl[0] == "[":
-        iPos = 1
-    else:
-        iPos = 0
-    key = ""
-    value = ""
-    action = "key"
-    insideLiteral = False
-    cfg = CFG()
-    while iPos < len(jdl):
-        char = jdl[iPos]
-        if char == ";" and not insideLiteral:
-            if key.strip():
-                result = assignValue(key, value, cfg)
-                if not result["OK"]:
-                    return result
-            key = ""
-            value = ""
-            action = "key"
-        elif char == "[" and not insideLiteral:
-            key = key.strip()
-            if not key:
-                return S_ERROR("Invalid key in JDL")
-            if value.strip():
-                return S_ERROR(f"Key {key} seems to have a value and open a sub JDL at the same time")
-            result = loadJDLAsCFG(jdl[iPos:])
-            if not result["OK"]:
-                return result
-            subCfg, subPos = result["Value"]
-            cfg.createNewSection(key, contents=subCfg)
-            key = ""
-            value = ""
-            action = "key"
-            insideLiteral = False
-            iPos += subPos
-        elif char == "=" and not insideLiteral:
-            if action == "key":
-                action = "value"
-                insideLiteral = False
-            else:
-                value += char
-        elif char == "]" and not insideLiteral:
-            key = key.strip()
-            if len(key) > 0:
-                result = assignValue(key, value, cfg)
-                if not result["OK"]:
-                    return result
-            return S_OK((cfg, iPos))
-        else:
-            if action == "key":
-                key += char
-            else:
-                value += char
-                if char == '"':
-                    insideLiteral = not insideLiteral
-        iPos += 1
-
-    return S_OK((cfg, iPos))
-
-
-def dumpCFGAsJDL(cfg, level=1, tab="  "):
-    indent = tab * level
-    contents = [f"{tab * (level - 1)}["]
-    sections = cfg.listSections()
-
-    for key in cfg:
-        if key in sections:
-            contents.append(f"{indent}{key} =")
-            contents.append(f"{dumpCFGAsJDL(cfg[key], level + 1, tab)};")
-        else:
-            val = List.fromChar(cfg[key])
-            # Some attributes are never lists
-            if len(val) < 2 or key in [ARGUMENTS, EXECUTABLE, STD_OUTPUT, STD_ERROR]:
-                value = cfg[key]
-                try:
-                    try_value = float(value)
-                    contents.append(f"{tab * level}{key} = {value};")
-                except Exception:
-                    contents.append(f'{tab * level}{key} = "{value}";')
-            else:
-                contents.append(f"{indent}{key} =")
-                contents.append("%s{" % indent)
-                for iPos in range(len(val)):
-                    try:
-                        value = float(val[iPos])
-                    except Exception:
-                        val[iPos] = f'"{val[iPos]}"'
-                contents.append(",\n".join([f"{tab * (level + 1)}{value}" for value in val]))
-                contents.append("%s};" % indent)
-    contents.append(f"{tab * (level - 1)}]")
-    return "\n".join(contents)
-
 
 def jdlToBaseJobDescriptionModel(classAd: ClassAd):
     """