DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/AccountingSystem/Client/AccountingCLI.py

@@ -57,99 +57,6 @@ class AccountingCLI(CLI):
         traceback.print_tb(sys.exc_info()[2])
         print("________________________\n")
 
-    def do_registerType(self, args):
-        """
-        Registers a new accounting type
-          Usage : registerType <typeName>
-          <DIRACRoot>/DIRAC/AccountingSystem/Client/Types/<typeName>
-           should exist and inherit the base type
-        """
-        try:
-            argList = args.split()
-            if argList:
-                typeName = argList[0].strip()
-            else:
-                gLogger.error("No type name specified")
-                return
-            # Try to import the type
-            result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}")
-            if not result["OK"]:
-                return result
-            typeClass = result["Value"]
-
-            gLogger.info(f"Loaded type {typeClass.__name__}")
-            typeDef = typeClass().getDefinition()
-            acClient = DataStoreClient()
-            retVal = acClient.registerType(*typeDef)
-            if retVal["OK"]:
-                gLogger.info("Type registered successfully")
-            else:
-                gLogger.error(f"Error: {retVal['Message']}")
-        except Exception:
-            self.showTraceback()
-
-    def do_resetBucketLength(self, args):
-        """
-        Set the bucket Length. Will trigger a recalculation of buckets. Can take a while.
-          Usage : resetBucketLength <typeName>
-          <DIRACRoot>/DIRAC/AccountingSystem/Client/Types/<typeName>
-           should exist and inherit the base type
-        """
-        try:
-            argList = args.split()
-            if argList:
-                typeName = argList[0].strip()
-            else:
-                gLogger.error("No type name specified")
-                return
-
-            # Try to import the type
-            result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}")
-            if not result["OK"]:
-                return result
-            typeClass = result["Value"]
-            gLogger.info(f"Loaded type {typeClass.__name__}")
-            typeDef = typeClass().getDefinition()
-            acClient = DataStoreClient()
-            retVal = acClient.setBucketsLength(typeDef[0], typeDef[3])
-            if retVal["OK"]:
-                gLogger.info("Type registered successfully")
-            else:
-                gLogger.error(f"Error: {retVal['Message']}")
-        except Exception:
-            self.showTraceback()
-
-    def do_regenerateBuckets(self, args):
-        """
-        Regenerate buckets for type. Can take a while.
-          Usage : regenerateBuckets <typeName>
-          <DIRACRoot>/DIRAC/AccountingSystem/Client/Types/<typeName>
-           should exist and inherit the base type
-        """
-        try:
-            argList = args.split()
-            if argList:
-                typeName = argList[0].strip()
-            else:
-                gLogger.error("No type name specified")
-                return
-
-            # Try to import the type
-            result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}")
-            if not result["OK"]:
-                return result
-            typeClass = result["Value"]
-            gLogger.info(f"Loaded type {typeClass.__name__}")
-            typeDef = typeClass().getDefinition()
-            acClient = DataStoreClient()
-            retVal = acClient.regenerateBuckets(typeDef[0])
-            if retVal["OK"]:
-                gLogger.info("Buckets recalculated!")
-            else:
-                gLogger.error(f"Error: {retVal['Message']}")
-        except Exception:
-            self.showTraceback()
-
     def do_showRegisteredTypes(self, args):
         """
         Get a list of registered types
@@ -170,50 +77,3 @@ class AccountingCLI(CLI):
                 print(" Value fields:\n  %s" % "\n  ".join(typeList[2]))
         except Exception:
             self.showTraceback()
-
-    def do_deleteType(self, args):
-        """
-        Delete a registered accounting type.
-          Usage : deleteType <typeName>
-          WARN! It will delete all data associated to that type! VERY DANGEROUS!
-          If you screw it, you'll discover a new dimension of pain and doom! :)
-        """
-        try:
-            argList = args.split()
-            if argList:
-                typeName = argList[0].strip()
-            else:
-                gLogger.error("No type name specified")
-                return
-
-            choice = input(
-                f"Are you completely sure you want to delete type {typeName} and all it's data? yes/no [no]: "
-            )
-            choice = choice.lower()
-            if choice not in ("yes", "y"):
-                print("Delete aborted")
-                return
-
-            acClient = DataStoreClient()
-            retVal = acClient.deleteType(typeName)
-            if not retVal["OK"]:
-                gLogger.error(f"Error: {retVal['Message']}")
-                return
-            print("Hope you meant it, because it's done")
-        except Exception:
-            self.showTraceback()
-
-    def do_compactBuckets(self, args):
-        """
-        Compact buckets table
-          Usage : compactBuckets
-        """
-        try:
-            acClient = DataStoreClient()
-            retVal = acClient.compactDB()
-            if not retVal["OK"]:
-                gLogger.error(f"Error: {retVal['Message']}")
-                return
-            gLogger.info("Done")
-        except Exception:
-            self.showTraceback()

DIRAC/AccountingSystem/Client/DataStoreClient.py

@@ -122,19 +122,6 @@ class DataStoreClient(Client):
 
         return S_OK()
 
-    def remove(self, register):
-        """
-        Remove a Register from the Accounting DataStore
-        """
-        if not self.__checkBaseType(register.__class__):
-            return S_ERROR("register is not a valid type (has to inherit from BaseAccountingType")
-        retVal = register.checkValues()
-        if not retVal["OK"]:
-            return retVal
-        if gConfig.getValue("/LocalSite/DisableAccounting", False):
-            return S_OK()
-        return self._getRPC().remove(*register.getValues())
-
 
 def _sendToFailover(rpcStub):
     """Create a ForwardDISET operation for failover"""

DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py

@@ -161,13 +161,6 @@ class BaseAccountingType:
                 cD[self.fieldsList[iPos]] = self.valuesList[iPos]
         return cD
 
-    def registerToServer(self):
-        """
-        Register type in server
-        """
-        rpcClient = Client(url="Accounting/DataStore")
-        return rpcClient.registerType(*self.getDefinition())
-
     def commit(self):
         """
         Commit register to server

DIRAC/AccountingSystem/ConfigTemplate.cfg

@@ -9,11 +9,6 @@ Services
     Authorization
     {
       Default = authenticated
-      compactDB = ServiceAdministrator
-      deleteType = ServiceAdministrator
-      registerType = ServiceAdministrator
-      setBucketsLength = ServiceAdministrator
-      regenerateBuckets = ServiceAdministrator
     }
   }
   ##END

DIRAC/AccountingSystem/Service/DataStoreHandler.py

@@ -14,7 +14,6 @@ import datetime
 from DIRAC import S_ERROR, S_OK
 from DIRAC.AccountingSystem.DB.MultiAccountingDB import MultiAccountingDB
 from DIRAC.ConfigurationSystem.Client import PathFinder
-from DIRAC.Core.Base.Client import Client
 from DIRAC.Core.DISET.RequestHandler import RequestHandler, getServiceOption
 from DIRAC.Core.Utilities import TimeUtilities
 from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler
@@ -39,30 +38,6 @@ class DataStoreHandler(RequestHandler):
             gThreadScheduler.addPeriodicTask(60, cls.__acDB.loadPendingRecords)
         return S_OK()
 
-    types_registerType = [str, list, list, list]
-
-    def export_registerType(self, typeName, definitionKeyFields, definitionAccountingFields, bucketsLength):
-        """
-        Register a new type. (Only for all powerful admins)
-        """
-        return self.__acDB.registerType(typeName, definitionKeyFields, definitionAccountingFields, bucketsLength)
-
-    types_setBucketsLength = [str, list]
-
-    def export_setBucketsLength(self, typeName, bucketsLength):
-        """
-        Change the buckets Length. (Only for all powerful admins)
-        """
-        return self.__acDB.changeBucketsLength(typeName, bucketsLength)
-
-    types_regenerateBuckets = [str]
-
-    def export_regenerateBuckets(self, typeName):
-        """
-        Recalculate buckets. (Only for all powerful admins)
-        """
-        return self.__acDB.regenerateBuckets(typeName)
-
     types_getRegisteredTypes = []
 
     def export_getRegisteredTypes(self):
@@ -106,51 +81,4 @@ class DataStoreHandler(RequestHandler):
             records.append((entry[0], startTime, endTime, entry[3]))
         return self.__acDB.insertRecordBundleThroughQueue(records)
 
-    types_compactDB = []
-
-    def export_compactDB(self):
-        """
-        Compact the db by grouping buckets
-        """
-        # if we are running workers (not only one service) we can redirect the request to the master
-        # For more information please read the Administrative guide Accounting part!
-        # ADVICE: If you want to trigger the bucketing, please make sure the bucketing is not running!!!!
-        if self.runBucketing:
-            return self.__acDB.compactBuckets()
-
-        return Client(url="Accounting/DataStoreMaster").compactDB()
-
     types_remove = [str, datetime.datetime, datetime.datetime, list]
-
-    def export_remove(self, typeName, startTime, endTime, valuesList):
-        """
-        Remove a record for a type
-        """
-        startTime = int(TimeUtilities.toEpoch(startTime))
-        endTime = int(TimeUtilities.toEpoch(endTime))
-        return self.__acDB.deleteRecord(typeName, startTime, endTime, valuesList)
-
-    types_removeRegisters = [list]
-
-    def export_removeRegisters(self, entriesList):
-        """
-        Remove a record for a type
-        """
-        expectedTypes = [str, datetime.datetime, datetime.datetime, list]
-        for entry in entriesList:
-            if len(entry) != 4:
-                return S_ERROR("Invalid records")
-            for i, en in enumerate(entry):
-                if not isinstance(en, expectedTypes[i]):
-                    return S_ERROR(f"{i} field in the records should be {expectedTypes[i]}")
-        ok = 0
-        for entry in entriesList:
-            startTime = int(TimeUtilities.toEpoch(entry[1]))
-            endTime = int(TimeUtilities.toEpoch(entry[2]))
-            record = entry[3]
-            result = self.__acDB.deleteRecord(entry[0], startTime, endTime, record)
-            if not result["OK"]:
-                return S_OK(ok)
-            ok += 1
-
-        return S_OK(ok)

DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py

@@ -4,15 +4,6 @@ Some Helper functions to retrieve common location from the CS
 from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 
 
-def getSetup() -> str:
-    """
-    Return setup name
-    """
-    from DIRAC import gConfig
-
-    return gConfig.getValue("/DIRAC/Setup", "")
-
-
 def getVO(defaultVO: str = "") -> str:
     """
     Return VO from configuration

DIRAC/ConfigurationSystem/Client/Helpers/Registry.py

@@ -1,18 +1,49 @@
-""" Helper for /Registry section
-"""
+"""Helper for /Registry section"""
+
 import errno
+import inspect
+import sys
+from collections.abc import Iterable
+from threading import Lock
+from typing import Optional
+
+from cachetools import TTLCache, cached
+from cachetools.keys import hashkey
 
-from DIRAC import S_OK, S_ERROR
+from DIRAC import S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client.Config import gConfig
 from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
 
 ID_DN_PREFIX = "/O=DIRAC/CN="
 
+# 300 is the default CS refresh time
+
+CACHE_REFRESH_TIME = 300
 # pylint: disable=missing-docstring
 
 gBaseRegistrySection = "/Registry"
 
 
+def reset_all_caches():
+    """This method is called to clear all caches.
+    It is necessary to reinitialize them after the central CS
+    has been loaded
+    """
+    for cache in [
+        obj
+        for name, obj in inspect.getmembers(sys.modules[__name__])
+        if (inspect.isfunction(obj) and hasattr(obj, "cache_clear"))
+    ]:
+        cache.cache_clear()
+
+
+def get_username_for_dn_key(dn: str, userList: Optional[Iterable[str]] = None):
+    if userList:
+        return hashkey(dn, *sorted(userList))
+    return hashkey(dn)
+
+
+@cached(TTLCache(maxsize=1000, ttl=CACHE_REFRESH_TIME), lock=Lock(), key=get_username_for_dn_key)
 def getUsernameForDN(dn, usersList=None):
     """Find DIRAC user for DN
 
@@ -33,6 +64,7 @@ def getUsernameForDN(dn, usersList=None):
     return S_ERROR(f"No username found for dn {dn}")
 
 
+@cached(TTLCache(maxsize=1000, ttl=CACHE_REFRESH_TIME), lock=Lock())
 def getDNForUsername(username):
     """Get user DN for user
 
@@ -413,6 +445,7 @@ def getBannedIPs():
     return gConfig.getValue(f"{gBaseRegistrySection}/BannedIPs", [])
 
 
+@cached(TTLCache(maxsize=1000, ttl=CACHE_REFRESH_TIME), lock=Lock())
 def getVOForGroup(group):
     """Search VO name for group
 
@@ -420,7 +453,7 @@ def getVOForGroup(group):
 
     :return: str
     """
-    return getVO() or gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VO", "")
+    return gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VO", "") or getVO()
 
 
 def getIdPForGroup(group):
@@ -451,28 +484,6 @@ def getVOMSAttributeForGroup(group):
     return gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VOMSRole", getDefaultVOMSAttribute())
 
 
-def getDefaultVOMSVO():
-    """Get default VOMS VO
-
-    :return: str
-    """
-    return gConfig.getValue(f"{gBaseRegistrySection}/DefaultVOMSVO", "") or getVO()
-
-
-def getVOMSVOForGroup(group):
-    """Search VOMS VO for group
-
-    :param str group: group name
-
-    :return: str
-    """
-    vomsVO = gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VOMSVO", getDefaultVOMSVO())
-    if not vomsVO:
-        vo = getVOForGroup(group)
-        vomsVO = getVOOption(vo, "VOMSName", "")
-    return vomsVO
-
-
 def getGroupsWithVOMSAttribute(vomsAttr):
     """Search groups with VOMS attribute
 
@@ -628,6 +639,7 @@ def getDNProperty(userDN, value, defaultValue=None):
     return S_OK(defaultValue)
 
 
+@cached(TTLCache(maxsize=1000, ttl=CACHE_REFRESH_TIME), lock=Lock())
 def getProxyProvidersForDN(userDN):
     """Get proxy providers by user DN
 

DIRAC/ConfigurationSystem/Client/Helpers/Resources.py

@@ -1,11 +1,14 @@
 """ Helper for the CS Resources section
 """
-import re
 from urllib import parse
 
 from DIRAC import S_ERROR, S_OK, gConfig, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
 from DIRAC.Core.Utilities.List import fromChar, uniqueElements
+from DIRACCommon.ConfigurationSystem.Client.Helpers.Resources import (
+    getDIRACPlatform as _getDIRACPlatform,
+    _platformSortKey,
+)
 
 gBaseResourcesSection = "/Resources"
 
@@ -328,8 +331,8 @@ def getCompatiblePlatforms(originalPlatforms):
     if not (result["OK"] and result["Value"]):
         return S_ERROR("OS compatibility info not found")
 
-    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}  # can be an iterator
-    for k, v in platformsDict.items():  # can be an iterator
+    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}
+    for k, v in platformsDict.items():
         if k not in v:
             v.append(k)
 
@@ -355,7 +358,6 @@ def getDIRACPlatform(OSList):
     :param list OSList: list of platforms defined by resource providers
     :return: a list of DIRAC platforms that can be specified in job descriptions
     """
-
     # For backward compatibility allow a single string argument
     osList = OSList
     if isinstance(OSList, str):
@@ -365,31 +367,12 @@ def getDIRACPlatform(OSList):
     if not (result["OK"] and result["Value"]):
         return S_ERROR("OS compatibility info not found")
 
-    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}  # can be an iterator
-    for k, v in platformsDict.items():  # can be an iterator
+    platformsDict = {k: set(v.replace(" ", "").split(",")) for k, v in result["Value"].items()}
+    for k, v in platformsDict.items():
         if k not in v:
-            v.append(k)
-
-    # making an OS -> platforms dict
-    os2PlatformDict = dict()
-    for platform, osItems in platformsDict.items():  # can be an iterator
-        for osItem in osItems:
-            if os2PlatformDict.get(osItem):
-                os2PlatformDict[osItem].append(platform)
-            else:
-                os2PlatformDict[osItem] = [platform]
-
-    platforms = []
-    for os in osList:
-        if os in os2PlatformDict:
-            platforms += os2PlatformDict[os]
-
-    if not platforms:
-        return S_ERROR(f"No compatible DIRAC platform found for {','.join(OSList)}")
+            v.add(k)
 
-    platforms.sort(key=_platformSortKey, reverse=True)
-
-    return S_OK(platforms)
+    return _getDIRACPlatform(osList, platformsDict)
 
 
 def getDIRACPlatforms():
@@ -451,7 +434,7 @@ def getInfoAboutProviders(of=None, providerName=None, option="", section=""):
             result = gConfig.getConfigurationTree(relPath)
             if not result["OK"]:
                 return result
-            for key, value in result["Value"].items():  # can be an iterator
+            for key, value in result["Value"].items():
                 if value:
                     resDict[key.replace(relPath, "")] = value
             return S_OK(resDict)
@@ -459,18 +442,3 @@ def getInfoAboutProviders(of=None, providerName=None, option="", section=""):
             return gConfig.getSections(f"{gBaseResourcesSection}/{of}Providers/{providerName}/{section}/")
     else:
         return S_OK(gConfig.getValue(f"{gBaseResourcesSection}/{of}Providers/{providerName}/{section}/{option}"))
-
-
-def _platformSortKey(version: str) -> list[str]:
-    # Loosely based on distutils.version.LooseVersion
-    parts = []
-    for part in re.split(r"(\d+|[a-z]+|\.| -)", version.lower()):
-        if not part or part == ".":
-            continue
-        if part[:1] in "0123456789":
-            part = part.zfill(8)
-        else:
-            while parts and parts[-1] == "00000000":
-                parts.pop()
-        parts.append(part)
-    return parts

DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py

@@ -9,7 +9,6 @@ from DIRAC.ConfigurationSystem.Client import ConfigurationData
 from DIRAC.ConfigurationSystem.Client.Helpers.Resources import (
     getDIRACPlatform,
     getCompatiblePlatforms,
-    _platformSortKey,
     getQueue,
 )
 
@@ -77,21 +76,6 @@ def test_getDIRACPlatform(mocker, mockGCReplyInput, requested, expectedRes, expe
         assert set(res["Value"]) == set(expectedValue), res["Value"]
 
 
-@pytest.mark.parametrize(
-    "string,expected",
-    [
-        ("Darwin_arm64_12.4", ["darwin", "_", "arm", "64", "_", "12", "4"]),
-        ("Linux_x86_64_glibc-2.17", ["linux", "_", "x", "86", "_", "64", "_", "glibc", "-", "2", "17"]),
-        ("Linux_aarch64_glibc-2.28", ["linux", "_", "aarch", "64", "_", "glibc", "-", "2", "28"]),
-    ],
-)
-def test_platformSortKey(string, expected):
-    actual = _platformSortKey(string)
-    for a, e in zip_longest(actual, expected):
-        # Numbers are padded with zeros so string comparison works
-        assert a.lstrip("0") == e
-
-
 @pytest.mark.parametrize(
     "mockGCReplyInput, requested, expectedRes, expectedValue",
     [

DIRAC/ConfigurationSystem/Client/LocalConfiguration.py

@@ -1,5 +1,5 @@
-""" This is the guy that parses and interprets the local configuration options.
-"""
+"""This is the guy that parses and interprets the local configuration options."""
+
 import re
 import os
 import sys
@@ -286,16 +286,18 @@ class LocalConfiguration:
             gLogger.exception()
             return S_ERROR(str(e))
 
-    def initialize(self, *, returnErrors=False):
+    def initialize(self, *, returnErrors=False, requireSuccessfulSync=False):
         """Entrypoint used by :py:class:`DIRAC.initialize`
 
+        :param requireSuccessfulSync: fails if syncing with the remote did not work
+
         TODO: This is currently a hack that returns a list of errors for so it
         can be used by ``__addUserDataToConfiguration``. This entire module
         should be refactored and simplified with ``Script.parseCommandLine``.
         """
         errorsList = self.__loadCFGFiles()
         if gConfigurationData.getServers():
-            retVal = self.syncRemoteConfiguration()
+            retVal = self.syncRemoteConfiguration(strict=requireSuccessfulSync)
             if not retVal["OK"]:
                 return retVal
         else:
@@ -321,7 +323,7 @@ class LocalConfiguration:
             gLogger.showHeaders(True)
             gLogger.enableLogsFromExternalLibs()
 
-    def loadUserData(self):
+    def loadUserData(self, requireSuccessfulSync=False):
         """
         This is the magic method that reads the command line and processes it
         It is used by the Script Base class and the dirac-service and dirac-agent scripts
@@ -329,6 +331,7 @@ class LocalConfiguration:
         - any additional switches to be processed
         - mandatory and default configuration configuration options must be defined.
 
+        :param requireSuccessfulSync: if True, will fail if the sync with remote server failed
         """
         if self.initialized:
             return S_OK()
@@ -336,7 +339,7 @@ class LocalConfiguration:
         try:
             if not self.isParsed:
                 self.__parseCommandLine()  # Parse command line
-            retVal = self.__addUserDataToConfiguration()
+            retVal = self.__addUserDataToConfiguration(requireSuccessfulSync=requireSuccessfulSync)
 
             for optionTuple in self.optionalEntryList:
                 optionPath = self.__getAbsolutePath(optionTuple[0])
@@ -496,8 +499,8 @@ class LocalConfiguration:
 
         return errorsList
 
-    def __addUserDataToConfiguration(self):
-        retVal = self.initialize(returnErrors=True)
+    def __addUserDataToConfiguration(self, requireSuccessfulSync=False):
+        retVal = self.initialize(returnErrors=True, requireSuccessfulSync=requireSuccessfulSync)
         if not retVal["OK"]:
             return retVal
         errorsList = retVal["Value"]
@@ -568,6 +571,9 @@ class LocalConfiguration:
             objLoader = ObjectLoader()
             objLoader.reloadRootModules()
             self.__initLogger(self.componentName, self.loggingSection, forceInit=True)
+            from DIRAC.ConfigurationSystem.Client.Helpers.Registry import reset_all_caches
+
+            reset_all_caches()
         return res
 
     def isCSEnabled(self):