DIRAC 9.0.0a60__tar.gz → 9.0.0a61__tar.gz

{dirac-9.0.0a60/src/DIRAC.egg-info → dirac-9.0.0a61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.0a60
+Version: 9.0.0a61
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only

{dirac-9.0.0a60 → dirac-9.0.0a61}/release.notes

@@ -1,3 +1,30 @@
+[v9.0.0a61]
+
+*WorkloadManagementSystem
+
+FIX: (#8237) The JobWrapper correctly resolves the destination SE lists
+
+*Core
+
+FIX: (#8235) do not randomized urls sorted by preferredURLPatterns
+
+*WorkloadManagement
+
+NEW: (#8234) add a warning log when the first pilot retrieval attempt fails
+
+*Resources
+
+NEW: (#8230) StorageBase.getParameters returns all parameters
+FIX: (#8225) Condor command line call compatible with v24
+
+*Accounting
+
+CHANGE: (#8216) removed few dangerous RPC calls
+
+*MonitoringSystem
+
+CHANGE: (#8216) removed useless RPC calls
+
 [v9.0.0a60]
 
 *DataManagementSystem

dirac-9.0.0a61/src/DIRAC/AccountingSystem/Client/AccountingCLI.py

@@ -0,0 +1,79 @@
+"""
+AccountingCLI class implementing command line administrative interface to
+DIRAC Accounting DataStore Service
+"""
+
+import sys
+
+from DIRAC import gLogger
+from DIRAC.Core.Base.CLI import CLI, colorize
+from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient
+from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
+
+
+class AccountingCLI(CLI):
+    def __init__(self):
+        CLI.__init__(self)
+        self.do_connect(None)
+        self.objectLoader = ObjectLoader()
+
+    def start(self):
+        """
+        Start the command loop
+        """
+        if not self.connected:
+            gLogger.error("Client is not connected")
+        try:
+            self.cmdloop()
+        except KeyboardInterrupt:
+            gLogger.warn("Received a keyboard interrupt.")
+            self.do_quit("")
+
+    def do_connect(self, args):
+        """
+        Tries to connect to the server
+            Usage: connect
+        """
+        gLogger.info("Trying to connect to server")
+        self.connected = False
+        self.prompt = f"({colorize('Not connected', 'red')})> "
+        acClient = DataStoreClient()
+        retVal = acClient.ping()
+        if retVal["OK"]:
+            self.prompt = f"({colorize('Connected', 'green')})> "
+            self.connected = True
+
+    def printComment(self, comment):
+        commentList = comment.split("\n")
+        for commentLine in commentList[:-1]:
+            print(f"# {commentLine.strip()}")
+
+    def showTraceback(self):
+        import traceback
+
+        type, value = sys.exc_info()[:2]
+        print("________________________\n")
+        print("Exception", type, ":", value)
+        traceback.print_tb(sys.exc_info()[2])
+        print("________________________\n")
+
+    def do_showRegisteredTypes(self, args):
+        """
+        Get a list of registered types
+          Usage : showRegisteredTypes
+        """
+        try:
+            acClient = DataStoreClient()
+            retVal = acClient.getRegisteredTypes()
+
+            print(retVal)
+
+            if not retVal["OK"]:
+                gLogger.error(f"Error: {retVal['Message']}")
+                return
+            for typeList in retVal["Value"]:
+                print(typeList[0])
+                print(" Key fields:\n  %s" % "\n  ".join(typeList[1]))
+                print(" Value fields:\n  %s" % "\n  ".join(typeList[2]))
+        except Exception:
+            self.showTraceback()

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/AccountingSystem/Client/DataStoreClient.py

@@ -122,19 +122,6 @@ class DataStoreClient(Client):
 
         return S_OK()
 
-    def remove(self, register):
-        """
-        Remove a Register from the Accounting DataStore
-        """
-        if not self.__checkBaseType(register.__class__):
-            return S_ERROR("register is not a valid type (has to inherit from BaseAccountingType")
-        retVal = register.checkValues()
-        if not retVal["OK"]:
-            return retVal
-        if gConfig.getValue("/LocalSite/DisableAccounting", False):
-            return S_OK()
-        return self._getRPC().remove(*register.getValues())
-
 
 def _sendToFailover(rpcStub):
     """Create a ForwardDISET operation for failover"""

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py

@@ -161,13 +161,6 @@ class BaseAccountingType:
                 cD[self.fieldsList[iPos]] = self.valuesList[iPos]
         return cD
 
-    def registerToServer(self):
-        """
-        Register type in server
-        """
-        rpcClient = Client(url="Accounting/DataStore")
-        return rpcClient.registerType(*self.getDefinition())
-
     def commit(self):
         """
         Commit register to server

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/AccountingSystem/ConfigTemplate.cfg

@@ -9,11 +9,6 @@ Services
     Authorization
     {
       Default = authenticated
-      compactDB = ServiceAdministrator
-      deleteType = ServiceAdministrator
-      registerType = ServiceAdministrator
-      setBucketsLength = ServiceAdministrator
-      regenerateBuckets = ServiceAdministrator
     }
   }
   ##END

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/AccountingSystem/Service/DataStoreHandler.py

@@ -14,7 +14,6 @@ import datetime
 from DIRAC import S_ERROR, S_OK
 from DIRAC.AccountingSystem.DB.MultiAccountingDB import MultiAccountingDB
 from DIRAC.ConfigurationSystem.Client import PathFinder
-from DIRAC.Core.Base.Client import Client
 from DIRAC.Core.DISET.RequestHandler import RequestHandler, getServiceOption
 from DIRAC.Core.Utilities import TimeUtilities
 from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler
@@ -39,30 +38,6 @@ class DataStoreHandler(RequestHandler):
             gThreadScheduler.addPeriodicTask(60, cls.__acDB.loadPendingRecords)
         return S_OK()
 
-    types_registerType = [str, list, list, list]
-
-    def export_registerType(self, typeName, definitionKeyFields, definitionAccountingFields, bucketsLength):
-        """
-        Register a new type. (Only for all powerful admins)
-        """
-        return self.__acDB.registerType(typeName, definitionKeyFields, definitionAccountingFields, bucketsLength)
-
-    types_setBucketsLength = [str, list]
-
-    def export_setBucketsLength(self, typeName, bucketsLength):
-        """
-        Change the buckets Length. (Only for all powerful admins)
-        """
-        return self.__acDB.changeBucketsLength(typeName, bucketsLength)
-
-    types_regenerateBuckets = [str]
-
-    def export_regenerateBuckets(self, typeName):
-        """
-        Recalculate buckets. (Only for all powerful admins)
-        """
-        return self.__acDB.regenerateBuckets(typeName)
-
     types_getRegisteredTypes = []
 
     def export_getRegisteredTypes(self):
@@ -106,51 +81,4 @@ class DataStoreHandler(RequestHandler):
             records.append((entry[0], startTime, endTime, entry[3]))
         return self.__acDB.insertRecordBundleThroughQueue(records)
 
-    types_compactDB = []
-
-    def export_compactDB(self):
-        """
-        Compact the db by grouping buckets
-        """
-        # if we are running workers (not only one service) we can redirect the request to the master
-        # For more information please read the Administrative guide Accounting part!
-        # ADVICE: If you want to trigger the bucketing, please make sure the bucketing is not running!!!!
-        if self.runBucketing:
-            return self.__acDB.compactBuckets()
-
-        return Client(url="Accounting/DataStoreMaster").compactDB()
-
     types_remove = [str, datetime.datetime, datetime.datetime, list]
-
-    def export_remove(self, typeName, startTime, endTime, valuesList):
-        """
-        Remove a record for a type
-        """
-        startTime = int(TimeUtilities.toEpoch(startTime))
-        endTime = int(TimeUtilities.toEpoch(endTime))
-        return self.__acDB.deleteRecord(typeName, startTime, endTime, valuesList)
-
-    types_removeRegisters = [list]
-
-    def export_removeRegisters(self, entriesList):
-        """
-        Remove a record for a type
-        """
-        expectedTypes = [str, datetime.datetime, datetime.datetime, list]
-        for entry in entriesList:
-            if len(entry) != 4:
-                return S_ERROR("Invalid records")
-            for i, en in enumerate(entry):
-                if not isinstance(en, expectedTypes[i]):
-                    return S_ERROR(f"{i} field in the records should be {expectedTypes[i]}")
-        ok = 0
-        for entry in entriesList:
-            startTime = int(TimeUtilities.toEpoch(entry[1]))
-            endTime = int(TimeUtilities.toEpoch(entry[2]))
-            record = entry[3]
-            result = self.__acDB.deleteRecord(entry[0], startTime, endTime, record)
-            if not result["OK"]:
-                return S_OK(ok)
-            ok += 1
-
-        return S_OK(ok)

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/Core/DISET/private/BaseClient.py

@@ -323,7 +323,7 @@ class BaseClient:
             pass
 
         # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
-        urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls
+        urlsList = List.fromChar(urls, ",") + failoverUrls
         self.__nbOfUrls = len(urlsList)
         self.__nbOfRetry = (
             2 if self.__nbOfUrls > 2 else 3
@@ -445,7 +445,6 @@ and this is thread {cThID}
             return self.__initStatus
         if self.__enableThreadCheck:
             self.__checkThreadID()
-
         gLogger.debug(f"Trying to connect to: {self.serviceURL}")
         try:
             # Calls the transport method of the apropriate protocol.

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/Core/scripts/dirac_apptainer_exec.py

@@ -100,7 +100,8 @@ def main():
         gLogger.error(result["Message"])
         DIRAC.exit(1)
     if result["Value"][0] != 0:
-        gLogger.error(result["Value"][2])
+        gLogger.error("Apptainer command failed with exit code", result["Value"][0])
+        gLogger.error("Command output:", result["Value"])
         DIRAC.exit(2)
     gLogger.notice(result["Value"][1])
 

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py

@@ -1,7 +1,8 @@
 """
-  This module contains helper methods for accessing operational attributes or parameters of DMS objects
+This module contains helper methods for accessing operational attributes or parameters of DMS objects
 
 """
+
 from collections import defaultdict
 from DIRAC import gConfig, gLogger, S_OK, S_ERROR
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
@@ -17,6 +18,9 @@ sLog = gLogger.getSubLogger(__name__)
 def resolveSEGroup(seGroupList, allSEs=None):
     """
     Resolves recursively a (list of) SEs that can be groupSEs
+    For modules, JobWrapper or whatever runs at a given site,
+    prefer using :py:func:`~DIRAC.DataManagementSystem.Utilities.ResolveSE.getDestinationSEList`
+
 
     :param seGroupList: list of SEs to resolve or comma-separated SEs
     :type seGroupList: list or string

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py

@@ -447,26 +447,6 @@ class MonitoringDB(ElasticDB):
                 records.append({paramName: getattr(resObj["_source"], paramName) for paramName in paramNames})
             return S_OK(records)
 
-    def getLastDayData(self, typeName, condDict):
-        """
-        It returns the last day data for a given monitoring type.
-
-        :returns: for example
-
-          .. code-block:: python
-
-           {'sort': [{'timestamp': {'order': 'desc'}}],
-            'query': {'bool': {'must': [{'match': {'host': 'dzmathe.cern.ch'}},
-                                        {'match': {'component': 'Bookkeeping_BookkeepingManager'}}]}}}
-
-        :param str typeName: name of the monitoring type
-        :param dict condDict: conditions for the query
-
-                      * key -> name of the field
-                      * value -> list of possible values
-        """
-        return self.__getRawData(typeName, condDict)
-
     def getLimitedData(self, typeName, condDict, size=10):
         """
         Returns a list of records for a given selection.

{dirac-9.0.0a60 → dirac-9.0.0a61}/src/DIRAC/MonitoringSystem/Service/MonitoringHandler.py

@@ -244,24 +244,6 @@ class MonitoringHandlerMixin:
         reportRequest["generatePlot"] = False
         return reporter.generate(reportRequest)
 
-    types_addMonitoringRecords = [str, list]
-
-    def export_addMonitoringRecords(self, monitoringtype, data):
-        """
-        Bulk insert data directly to the given monitoring type.
-
-        :param str monitoringtype: monitoring type name
-        :param list data: list of documents
-        :returns: S_OK or S_ERROR
-        """
-
-        retVal = self.__db.getIndexName(monitoringtype)
-        if not retVal["OK"]:
-            return retVal
-        prefix = retVal["Value"]
-        gLogger.debug("addMonitoringRecords:", prefix)
-        return self.__db.bulk_index(prefix, data)
-
     types_addRecords = [str, str, list]
 
     def export_addRecords(self, indexname, monitoringType, data):
@@ -290,21 +272,6 @@ class MonitoringHandlerMixin:
         gLogger.debug("delete index:", indexName)
         return self.__db.deleteIndex(indexName)
 
-    types_getLastDayData = [str, dict]
-
-    def export_getLastDayData(self, typeName, condDict):
-        """
-        It returns the data from the last day index. Note: we create daily indexes.
-
-        :param str typeName: name of the monitoring type
-        :param dict condDict: conditions for the query
-
-                       * key -> name of the field
-                       * value -> list of possible values
-        """
-
-        return self.__db.getLastDayData(typeName, condDict)
-
     types_getLimitedDat = [str, dict, int]
 
     def export_getLimitedData(self, typeName, condDict, size):