DIRAC 9.0.0a66__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py

@@ -594,7 +594,7 @@ class VOMS2CSSynchronizer:
 
         # Try to fill in the DiracX section
         if self.useIAM:
-            iam_subs = self.iamSrv.getUsersSub()
+            iam_subs = self.iamSrv.getUsersSub(self.vo)
             diracx_vo_config = {"DiracX": {"CsSync": {"VOs": {self.vo: {"UserSubjects": iam_subs}}}}}
             iam_sub_cfg = CFG()
             iam_sub_cfg.loadFromDict(diracx_vo_config)

DIRAC/Core/Security/IAMService.py

@@ -144,7 +144,7 @@ class IAMService:
         result = S_OK({"Users": users, "Errors": errors})
         return result
 
-    def getUsersSub(self) -> dict[str, str]:
+    def getUsersSub(self, vo=None) -> dict[str, str]:
         """
         Return the mapping based on IAM sub:
         {nickname : sub}
@@ -152,6 +152,7 @@ class IAMService:
         iam_users_raw = self._getIamUserDump()
         diracx_user_section = {}
         for user_info in iam_users_raw:
+            userGroups = [grp["display"] for grp in user_info.get("groups", [])]
             # The nickname is available in the list of attributes
             # (if configured so)
             # in the form {'name': 'nickname', 'value': 'chaen'}
@@ -165,8 +166,8 @@ class IAMService:
             except (KeyError, IndexError):
                 nickname = user_info["userName"]
             sub = user_info["id"]
-
-            diracx_user_section[nickname] = sub
+            if not vo or vo in userGroups:
+                diracx_user_section[nickname] = sub
         # reorder it
         diracx_user_section = dict(sorted(diracx_user_section.items()))
 

DIRAC/Core/Utilities/ClassAd/ClassAdLight.py

@@ -2,294 +2,8 @@
     Condor ClassAd library.
 """
 
+# Import from DIRACCommon for backward compatibility
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 
-class ClassAd:
-    def __init__(self, jdl):
-        """ClassAd constructor from a JDL string"""
-        self.contents = {}
-        result = self.__analyse_jdl(jdl)
-        if result:
-            self.contents = result
-
-    def __analyse_jdl(self, jdl, index=0):
-        """Analyse one [] jdl enclosure"""
-
-        jdl = jdl.strip()
-
-        # Strip all the blanks first
-        # temp = jdl.replace(' ','').replace('\n','')
-        temp = jdl
-
-        result = {}
-
-        if temp[0] != "[" or temp[-1] != "]":
-            print("Invalid JDL: it should start with [ and end with ]")
-            return result
-
-        # Parse the jdl string now
-        body = temp[1:-1]
-        index = 0
-        namemode = 1
-        valuemode = 0
-        while index < len(body):
-            if namemode:
-                ind = body.find("=", index)
-                if ind != -1:
-                    name = body[index:ind]
-                    index = ind + 1
-                    valuemode = 1
-                    namemode = 0
-                else:
-                    break
-            elif valuemode:
-                ind1 = body.find("[", index)
-                ind2 = body.find(";", index)
-                if ind1 != -1 and ind1 < ind2:
-                    value, newind = self.__find_subjdl(body, ind1)
-                elif ind1 == -1 and ind2 == -1:
-                    value = body[index:]
-                    newind = len(body)
-                else:
-                    if index == ind2:
-                        return {}
-                    else:
-                        value = body[index:ind2]
-                        newind = ind2 + 1
-
-                result[name.strip()] = value.strip().replace("\n", "")
-                index = newind
-                valuemode = 0
-                namemode = 1
-
-        return result
-
-    def __find_subjdl(self, body, index):
-        """Find a full [] enclosure starting from index"""
-        result = ""
-        if body[index] != "[":
-            return (result, 0)
-
-        depth = 0
-        ind = index
-        while depth < 10:
-            ind1 = body.find("]", ind + 1)
-            ind2 = body.find("[", ind + 1)
-            if ind2 != -1 and ind2 < ind1:
-                depth += 1
-                ind = ind2
-            else:
-                if depth > 0:
-                    depth -= 1
-                    ind = ind1
-                else:
-                    result = body[index : ind1 + 1]
-                    if body[ind1 + 1] == ";":
-                        return (result, ind1 + 2)
-                    return result, 0
-
-        return result, 0
-
-    def insertAttributeInt(self, name, attribute):
-        """Insert a named integer attribute"""
-
-        self.contents[name] = str(attribute)
-
-    def insertAttributeBool(self, name, attribute):
-        """Insert a named boolean attribute"""
-
-        if attribute:
-            self.contents[name] = "true"
-        else:
-            self.contents[name] = "false"
-
-    def insertAttributeString(self, name, attribute):
-        """Insert a named string attribute"""
-
-        self.contents[name] = '"' + str(attribute) + '"'
-
-    def insertAttributeVectorString(self, name, attributelist):
-        """Insert a named string list attribute"""
-
-        tmp = ['"' + x + '"' for x in attributelist]
-        tmpstr = ",".join(tmp)
-        self.contents[name] = "{" + tmpstr + "}"
-
-    def insertAttributeVectorInt(self, name, attributelist):
-        """Insert a named string list attribute"""
-
-        tmp = [str(x) for x in attributelist]
-        tmpstr = ",".join(tmp)
-        self.contents[name] = "{" + tmpstr + "}"
-
-    def insertAttributeVectorStringList(self, name, attributelist):
-        """Insert a named list of string lists"""
-
-        listOfLists = []
-        for stringList in attributelist:
-            # tmp = map ( lambda x : '"' + x + '"', stringList )
-            tmpstr = ",".join(stringList)
-            listOfLists.append("{" + tmpstr + "}")
-        self.contents[name] = "{" + ",".join(listOfLists) + "}"
-
-    def lookupAttribute(self, name):
-        """Check the presence of the given attribute"""
-
-        return name in self.contents
-
-    def set_expression(self, name, attribute):
-        """Insert a named expression attribute"""
-
-        self.contents[name] = str(attribute)
-
-    def get_expression(self, name):
-        """Get expression corresponding to a named attribute"""
-
-        if name in self.contents:
-            if isinstance(self.contents[name], int):
-                return str(self.contents[name])
-            return self.contents[name]
-        return ""
-
-    def isAttributeList(self, name):
-        """Check if the given attribute is of the List type"""
-        attribute = self.get_expression(name).strip()
-        return attribute.startswith("{")
-
-    def getListFromExpression(self, name):
-        """Get a list of strings from a given expression"""
-
-        tempString = self.get_expression(name).strip()
-        listMode = False
-        if tempString.startswith("{"):
-            tempString = tempString[1:-1]
-            listMode = True
-
-        tempString = tempString.replace(" ", "").replace("\n", "")
-        if tempString.find("{") < 0:
-            if not listMode:
-                tempString = tempString.replace('"', "")
-                if not tempString:
-                    return []
-                return tempString.split(",")
-
-        resultList = []
-        while tempString:
-            if tempString.find("{") == 0:
-                end = tempString.find("}")
-                resultList.append(tempString[: end + 1])
-                tempString = tempString[end + 1 :]
-                if tempString.startswith(","):
-                    tempString = tempString[1:]
-            elif tempString.find('"') == 0:
-                end = tempString[1:].find('"')
-                resultList.append(tempString[1 : end + 1])
-                tempString = tempString[end + 2 :]
-                if tempString.startswith(","):
-                    tempString = tempString[1:]
-            else:
-                end = tempString.find(",")
-                if end < 0:
-                    resultList.append(tempString.replace('"', "").replace(" ", ""))
-                    break
-                else:
-                    resultList.append(tempString[:end].replace('"', "").replace(" ", ""))
-                    tempString = tempString[end + 1 :]
-
-        return resultList
-
-    def getDictionaryFromSubJDL(self, name):
-        """Get a dictionary of the JDL attributes from a subsection"""
-
-        tempList = self.get_expression(name)[1:-1]
-        resDict = {}
-        for item in tempList.split(";"):
-            if len(item.split("=")) == 2:
-                resDict[item.split("=")[0].strip()] = item.split("=")[1].strip().replace('"', "")
-            else:
-                return {}
-
-        return resDict
-
-    def deleteAttribute(self, name):
-        """Delete a named attribute"""
-
-        if name in self.contents:
-            del self.contents[name]
-            return 1
-        return 0
-
-    def isOK(self):
-        """Check the JDL validity - to be defined"""
-
-        if self.contents:
-            return 1
-        return 0
-
-    def asJDL(self):
-        """Convert the JDL description into a string"""
-
-        result = []
-        for name, value in sorted(self.contents.items()):
-            if value[0:1] == "{":
-                result += [4 * " " + name + " = \n"]
-                result += [8 * " " + "{\n"]
-                strings = value[1:-1].split(",")
-                for st in strings:
-                    result += [12 * " " + st.strip() + ",\n"]
-                result[-1] = result[-1][:-2]
-                result += ["\n" + 8 * " " + "};\n"]
-            elif value[0:1] == "[":
-                tempad = ClassAd(value)
-                tempjdl = tempad.asJDL() + ";"
-                lines = tempjdl.split("\n")
-                result += [4 * " " + name + " = \n"]
-                for line in lines:
-                    result += [8 * " " + line + "\n"]
-
-            else:
-                result += [4 * " " + name + " = " + str(value) + ";\n"]
-        if result:
-            result[-1] = result[-1][:-1]
-        return "[ \n" + "".join(result) + "\n]"
-
-    def getAttributeString(self, name):
-        """Get String type attribute value"""
-        value = ""
-        if self.lookupAttribute(name):
-            value = self.get_expression(name).replace('"', "")
-        return value
-
-    def getAttributeInt(self, name):
-        """Get Integer type attribute value"""
-        value = None
-        if self.lookupAttribute(name):
-            try:
-                value = int(self.get_expression(name).replace('"', ""))
-            except Exception:
-                value = None
-        return value
-
-    def getAttributeBool(self, name):
-        """Get Boolean type attribute value"""
-        if not self.lookupAttribute(name):
-            return False
-
-        value = self.get_expression(name).replace('"', "")
-        return value.lower() == "true"
-
-    def getAttributeFloat(self, name):
-        """Get Float type attribute value"""
-        value = None
-        if self.lookupAttribute(name):
-            try:
-                value = float(self.get_expression(name).replace('"', ""))
-            except Exception:
-                value = None
-        return value
-
-    def getAttributes(self) -> list[str]:
-        """Get the list of all the attribute names
-
-        :return: list of names as strings
-        """
-        return list(self.contents)
+# Re-export for backward compatibility
+__all__ = ["ClassAd"]

DIRAC/Core/Utilities/DErrno.py

@@ -41,7 +41,7 @@ import importlib
 import sys
 
 # Import all the stateless parts from DIRACCommon
-from DIRACCommon.Utils.DErrno import *  # noqa: F401, F403
+from DIRACCommon.Core.Utilities.DErrno import *  # noqa: F401, F403
 
 from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 

DIRAC/Core/Utilities/JDL.py

@@ -1,203 +1,9 @@
 """Transformation classes around the JDL format."""
 
-from diraccfg import CFG
-from pydantic import ValidationError
+from DIRACCommon.Core.Utilities.JDL import *  # noqa: F403,F401
 
-from DIRAC import S_OK, S_ERROR
-from DIRAC.Core.Utilities import List
-from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 from DIRAC.WorkloadManagementSystem.Utilities.JobModel import BaseJobDescriptionModel
 
-ARGUMENTS = "Arguments"
-BANNED_SITES = "BannedSites"
-CPU_TIME = "CPUTime"
-EXECUTABLE = "Executable"
-EXECUTION_ENVIRONMENT = "ExecutionEnvironment"
-GRID_CE = "GridCE"
-INPUT_DATA = "InputData"
-INPUT_DATA_POLICY = "InputDataPolicy"
-INPUT_SANDBOX = "InputSandbox"
-JOB_CONFIG_ARGS = "JobConfigArgs"
-JOB_TYPE = "JobType"
-JOB_GROUP = "JobGroup"
-LOG_LEVEL = "LogLevel"
-NUMBER_OF_PROCESSORS = "NumberOfProcessors"
-MAX_NUMBER_OF_PROCESSORS = "MaxNumberOfProcessors"
-MIN_NUMBER_OF_PROCESSORS = "MinNumberOfProcessors"
-OUTPUT_DATA = "OutputData"
-OUTPUT_PATH = "OutputPath"
-OUTPUT_SE = "OutputSE"
-PLATFORM = "Platform"
-PRIORITY = "Priority"
-STD_ERROR = "StdError"
-STD_OUTPUT = "StdOutput"
-OUTPUT_SANDBOX = "OutputSandbox"
-JOB_NAME = "JobName"
-SITE = "Site"
-TAGS = "Tags"
-
-OWNER = "Owner"
-OWNER_GROUP = "OwnerGroup"
-VO = "VirtualOrganization"
-
-CREDENTIALS_FIELDS = {OWNER, OWNER_GROUP, VO}
-
-
-def loadJDLAsCFG(jdl):
-    """
-    Load a JDL as CFG
-    """
-
-    def cleanValue(value):
-        value = value.strip()
-        if value[0] == '"':
-            entries = []
-            iPos = 1
-            current = ""
-            state = "in"
-            while iPos < len(value):
-                if value[iPos] == '"':
-                    if state == "in":
-                        entries.append(current)
-                        current = ""
-                        state = "out"
-                    elif state == "out":
-                        current = current.strip()
-                        if current not in (",",):
-                            return S_ERROR("value seems a list but is not separated in commas")
-                        current = ""
-                        state = "in"
-                else:
-                    current += value[iPos]
-                iPos += 1
-            if state == "in":
-                return S_ERROR('value is opened with " but is not closed')
-            return S_OK(", ".join(entries))
-        else:
-            return S_OK(value.replace('"', ""))
-
-    def assignValue(key, value, cfg):
-        key = key.strip()
-        if len(key) == 0:
-            return S_ERROR("Invalid key name")
-        value = value.strip()
-        if not value:
-            return S_ERROR(f"No value for key {key}")
-        if value[0] == "{":
-            if value[-1] != "}":
-                return S_ERROR("Value '%s' seems a list but does not end in '}'" % (value))
-            valList = List.fromChar(value[1:-1])
-            for i in range(len(valList)):
-                result = cleanValue(valList[i])
-                if not result["OK"]:
-                    return S_ERROR(f"Var {key} : {result['Message']}")
-                valList[i] = result["Value"]
-                if valList[i] is None:
-                    return S_ERROR(f"List value '{value}' seems invalid for item {i}")
-            value = ", ".join(valList)
-        else:
-            result = cleanValue(value)
-            if not result["OK"]:
-                return S_ERROR(f"Var {key} : {result['Message']}")
-            nV = result["Value"]
-            if nV is None:
-                return S_ERROR(f"Value '{value} seems invalid")
-            value = nV
-        cfg.setOption(key, value)
-        return S_OK()
-
-    if jdl[0] == "[":
-        iPos = 1
-    else:
-        iPos = 0
-    key = ""
-    value = ""
-    action = "key"
-    insideLiteral = False
-    cfg = CFG()
-    while iPos < len(jdl):
-        char = jdl[iPos]
-        if char == ";" and not insideLiteral:
-            if key.strip():
-                result = assignValue(key, value, cfg)
-                if not result["OK"]:
-                    return result
-            key = ""
-            value = ""
-            action = "key"
-        elif char == "[" and not insideLiteral:
-            key = key.strip()
-            if not key:
-                return S_ERROR("Invalid key in JDL")
-            if value.strip():
-                return S_ERROR(f"Key {key} seems to have a value and open a sub JDL at the same time")
-            result = loadJDLAsCFG(jdl[iPos:])
-            if not result["OK"]:
-                return result
-            subCfg, subPos = result["Value"]
-            cfg.createNewSection(key, contents=subCfg)
-            key = ""
-            value = ""
-            action = "key"
-            insideLiteral = False
-            iPos += subPos
-        elif char == "=" and not insideLiteral:
-            if action == "key":
-                action = "value"
-                insideLiteral = False
-            else:
-                value += char
-        elif char == "]" and not insideLiteral:
-            key = key.strip()
-            if len(key) > 0:
-                result = assignValue(key, value, cfg)
-                if not result["OK"]:
-                    return result
-            return S_OK((cfg, iPos))
-        else:
-            if action == "key":
-                key += char
-            else:
-                value += char
-                if char == '"':
-                    insideLiteral = not insideLiteral
-        iPos += 1
-
-    return S_OK((cfg, iPos))
-
-
-def dumpCFGAsJDL(cfg, level=1, tab="  "):
-    indent = tab * level
-    contents = [f"{tab * (level - 1)}["]
-    sections = cfg.listSections()
-
-    for key in cfg:
-        if key in sections:
-            contents.append(f"{indent}{key} =")
-            contents.append(f"{dumpCFGAsJDL(cfg[key], level + 1, tab)};")
-        else:
-            val = List.fromChar(cfg[key])
-            # Some attributes are never lists
-            if len(val) < 2 or key in [ARGUMENTS, EXECUTABLE, STD_OUTPUT, STD_ERROR]:
-                value = cfg[key]
-                try:
-                    try_value = float(value)
-                    contents.append(f"{tab * level}{key} = {value};")
-                except Exception:
-                    contents.append(f'{tab * level}{key} = "{value}";')
-            else:
-                contents.append(f"{indent}{key} =")
-                contents.append("%s{" % indent)
-                for iPos in range(len(val)):
-                    try:
-                        value = float(val[iPos])
-                    except Exception:
-                        val[iPos] = f'"{val[iPos]}"'
-                contents.append(",\n".join([f"{tab * (level + 1)}{value}" for value in val]))
-                contents.append("%s};" % indent)
-    contents.append(f"{tab * (level - 1)}]")
-    return "\n".join(contents)
-
 
 def jdlToBaseJobDescriptionModel(classAd: ClassAd):
     """

DIRAC/Core/Utilities/List.py

@@ -1,127 +1 @@
-"""Collection of DIRAC useful list related modules.
-   By default on Error they return None.
-"""
-import random
-import sys
-from typing import Any, TypeVar
-from collections.abc import Iterable
-
-T = TypeVar("T")
-
-
-def uniqueElements(aList: list) -> list:
-    """Utility to retrieve list of unique elements in a list (order is kept)."""
-
-    # Use dict.fromkeys instead of set ensure the order is preserved
-    return list(dict.fromkeys(aList))
-
-
-def appendUnique(aList: list, anObject: Any):
-    """Append to list if object does not exist.
-
-    :param aList: list of elements
-    :param anObject: object you want to append
-    """
-    if anObject not in aList:
-        aList.append(anObject)
-
-
-def fromChar(inputString: str, sepChar: str = ","):
-    """Generates a list splitting a string by the required character(s)
-    resulting string items are stripped and empty items are removed.
-
-    :param inputString: list serialised to string
-    :param sepChar: separator
-    :return: list of strings or None if sepChar has a wrong type
-    """
-    # to prevent getting an empty String as argument
-    if not (isinstance(inputString, str) and isinstance(sepChar, str) and sepChar):
-        return None
-    return [fieldString.strip() for fieldString in inputString.split(sepChar) if len(fieldString.strip()) > 0]
-
-
-def randomize(aList: Iterable[T]) -> list[T]:
-    """Return a randomly sorted list.
-
-    :param aList: list to permute
-    """
-    tmpList = list(aList)
-    random.shuffle(tmpList)
-    return tmpList
-
-
-def pop(aList, popElement):
-    """Pop the first element equal to popElement from the list.
-
-    :param aList: list
-    :type aList: python:list
-    :param popElement: element to pop
-    """
-    if popElement in aList:
-        return aList.pop(aList.index(popElement))
-
-
-def stringListToString(aList: list) -> str:
-    """This function is used for making MySQL queries with a list of string elements.
-
-    :param aList: list to be serialized to string for making queries
-    """
-    return ",".join(f"'{x}'" for x in aList)
-
-
-def intListToString(aList: list) -> str:
-    """This function is used for making MySQL queries with a list of int elements.
-
-    :param aList: list to be serialized to string for making queries
-    """
-    return ",".join(str(x) for x in aList)
-
-
-def getChunk(aList: list, chunkSize: int):
-    """Generator yielding chunk from a list of a size chunkSize.
-
-    :param aList: list to be splitted
-    :param chunkSize: lenght of one chunk
-    :raise: StopIteration
-
-    Usage:
-
-    >>> for chunk in getChunk( aList, chunkSize=10):
-          process( chunk )
-
-    """
-    chunkSize = int(chunkSize)
-    for i in range(0, len(aList), chunkSize):
-        yield aList[i : i + chunkSize]
-
-
-def breakListIntoChunks(aList: list, chunkSize: int):
-    """This function takes a list as input and breaks it into list of size 'chunkSize'.
-       It returns a list of lists.
-
-    :param aList: list of elements
-    :param chunkSize: len of a single chunk
-    :return: list of lists of length of chunkSize
-    :raise: RuntimeError if numberOfFilesInChunk is less than 1
-    """
-    if chunkSize < 1:
-        raise RuntimeError("chunkSize cannot be less than 1")
-    if isinstance(aList, (set, dict, tuple, {}.keys().__class__, {}.items().__class__, {}.values().__class__)):
-        aList = list(aList)
-    return [chunk for chunk in getChunk(aList, chunkSize)]
-
-
-def getIndexInList(anItem: Any, aList: list) -> int:
-    """Return the index of the element x in the list l
-    or sys.maxint if it does not exist
-
-    :param anItem: element to look for
-    :param aList: list to look into
-
-    :return: the index or sys.maxint
-    """
-    # try:
-    if anItem in aList:
-        return aList.index(anItem)
-    else:
-        return sys.maxsize
+from DIRACCommon.Core.Utilities.List import *  # noqa: F401,F403

DIRAC/Core/Utilities/ReturnValues.py

@@ -1,10 +1,10 @@
 """Backward compatibility wrapper - moved to DIRACCommon
 
-This module has been moved to DIRACCommon.Utils.ReturnValues to avoid
+This module has been moved to DIRACCommon.Core.Utilities.ReturnValues to avoid
 circular dependencies and allow DiracX to use these utilities without
 triggering DIRAC's global state initialization.
 
 All exports are maintained for backward compatibility.
 """
 # Re-export everything from DIRACCommon for backward compatibility
-from DIRACCommon.Utils.ReturnValues import *  # noqa: F401, F403
+from DIRACCommon.Core.Utilities.ReturnValues import *  # noqa: F401, F403