DIRAC 9.0.6__tar.gz → 9.0.7__tar.gz

{dirac-9.0.6/src/DIRAC.egg-info → dirac-9.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.6
+Version: 9.0.7
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.6
+Requires-Dist: DIRACCommon==v9.0.7
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1

{dirac-9.0.6 → dirac-9.0.7}/release.notes

@@ -1,3 +1,38 @@
+[v9.0.7]
+
+*DataManagementSystem
+
+FIX: (#8367) FTS reports correct accounting for failed transfers
+
+*WorkloadManagementSystem
+
+FIX: (#8365) SiteDirector: siteNames, ceTypes, ces and tags are None by default
+
+*FrameworkSystem
+
+FIX: (#8361) use cachedmethod instead of cached in ProxyDB
+
+*TransformationSystem
+
+FIX: (#8361) use cachedmethod instead of cached in Utilities
+
+*WorkloadManagement
+
+CHANGE: (#8360) use PilotManager to get the pilot output(s) from a job ID in WMSAdministrator service
+
+*ResourceStatusSystem
+
+FIX: (#8357) Avoid negative values in the Space Accounting
+
+*Resources
+
+FIX: (#8355) AREX CE: fix for understanding slightly different formats
+
+*Core
+
+NEW: (#8352) Report errors for every failed attempt in M2SSLTransport initAsClient
+FIX: (#8352) Use resolved socket address for SSL connections in M2SSLTransport
+
 [v9.0.6]
 
 *FrameworkSystem

{dirac-9.0.6 → dirac-9.0.7}/setup.cfg

@@ -26,7 +26,7 @@ install_requires =
 	certifi
 	cwltool
 	diraccfg
-	DIRACCommon==v9.0.6
+	DIRACCommon==v9.0.7
 	diracx-client >=v0.0.1
 	diracx-core >=v0.0.1
 	diracx-cli >=v0.0.1

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py

@@ -110,19 +110,18 @@ class SSLTransport(BaseTransport):
         if self.serverMode():
             raise RuntimeError("SSLTransport is in server mode.")
 
-        error = None
+        errors = []
         host, port = self.stServerAddress
 
         # The following piece of code was inspired by the python socket documentation
         # as well as the implementation of M2Crypto.httpslib.HTTPSConnection
 
-        # We ignore the returned sockaddr because SSL.Connection.connect needs
-        # a host name.
+        # Get all available addresses (IPv6 and IPv4) and try them in order
         try:
             addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
         except OSError as e:
             return S_ERROR(f"DNS lookup failed {e!r}")
-        for family, _socketType, _proto, _canonname, _socketAddress in addrInfoList:
+        for family, _socketType, _proto, _canonname, socketAddress in addrInfoList:
             try:
                 self.oSocket = SSL.Connection(self.__ctx, family=family)
 
@@ -138,7 +137,10 @@ class SSLTransport(BaseTransport):
                 # set SNI server name since we know it at this point
                 self.oSocket.set_tlsext_host_name(host)
 
-                self.oSocket.connect((host, port))
+                # tell the connection which host we are connecting to so we can
+                # use the address we obtained from DNS
+                self.oSocket.set1_host(host)
+                self.oSocket.connect(socketAddress)
 
                 # Once the connection is established, we can use the timeout
                 # asked for RPC
@@ -151,12 +153,12 @@ class SSLTransport(BaseTransport):
             # They should be propagated upwards and caught by the BaseClient
             # not to enter the retry loop
             except OSError as e:
-                error = f"{e}:{repr(e)}"
+                errors.append(f"{socketAddress} {e}:{repr(e)}")
 
                 if self.oSocket is not None:
                     self.close()
 
-        return S_ERROR(error)
+        return S_ERROR("; ".join(errors))
 
     def initAsServer(self):
         """Prepare this server socket for use."""

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py

@@ -719,10 +719,11 @@ class FTS3Agent(AgentModule):
         return self.dataOpSender.concludeSending()
 
     def __sendAccounting(self, ftsJob):
-        self.dataOpSender.sendData(
-            ftsJob.accountingDict,
-            commitFlag=True,
-            delayedCommit=True,
-            startTime=fromString(ftsJob.submitTime),
-            endTime=fromString(ftsJob.lastUpdate),
-        )
+        for accountingDict in ftsJob.accountingDicts:
+            self.dataOpSender.sendData(
+                accountingDict,
+                commitFlag=True,
+                delayedCommit=True,
+                startTime=fromString(ftsJob.submitTime),
+                endTime=fromString(ftsJob.lastUpdate),
+            )

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -120,7 +120,7 @@ class FTS3Job(JSerializable):
         # temporary used only for accounting
         # it is set by the monitor method
         # when a job is in a final state
-        self.accountingDict = None
+        self.accountingDicts = None
 
     @classmethod
     @cachedmethod(lambda cls: cls._idp_cache)
@@ -157,7 +157,6 @@ class FTS3Job(JSerializable):
 
         if not self.ftsGUID:
             return S_ERROR("FTSGUID not set, FTS job not submitted?")
-
         if not context:
             if not ftsServer:
                 ftsServer = self.ftsServer
@@ -184,13 +183,14 @@ class FTS3Job(JSerializable):
             self.error = jobStatusDict["reason"]
 
         if newStatus in self.FINAL_STATES:
-            self._fillAccountingDict(jobStatusDict)
+            self._fillAccountingDicts(jobStatusDict)
 
         filesInfoList = jobStatusDict["files"]
         filesStatus = {}
         statusSummary = {}
 
         # Make a copy, since we are potentially
+
         # deleting objects
         for fileDict in list(filesInfoList):
             file_state = fileDict["file_state"].capitalize()
@@ -245,7 +245,7 @@ class FTS3Job(JSerializable):
         # so we put this back into the monitoring data such that the accounting is done properly
         jobStatusDict["files"] = filesInfoList
         if newStatus in self.FINAL_STATES:
-            self._fillAccountingDict(jobStatusDict)
+            self._fillAccountingDicts(jobStatusDict)
 
         total = len(filesInfoList)
         completed = sum(statusSummary.get(state, 0) for state in FTS3File.FTS_FINAL_STATES)
@@ -902,9 +902,9 @@ class FTS3Job(JSerializable):
             gLogger.exception("Error generating context", repr(e))
             return S_ERROR(repr(e))
 
-    def _fillAccountingDict(self, jobStatusDict):
-        """This methods generates the necessary information to create a DataOperation
-        accounting record, and stores them as a instance attribute.
+    def _fillAccountingDicts(self, jobStatusDict):
+        """This methods generates the necessary information to create DataOperation
+        accounting records, and stores them as a instance attribute.
 
         For it to be relevant, it should be called only when the job is in a final state.
 
@@ -913,6 +913,7 @@ class FTS3Job(JSerializable):
         :returns: None
         """
 
+        accountingDicts = []
         accountingDict = dict()
         sourceSE = None
         targetSE = None
@@ -923,16 +924,24 @@ class FTS3Job(JSerializable):
         accountingDict["Protocol"] = "FTS3"
         accountingDict["ExecutionSite"] = self.ftsServer
 
+        # Registration values must be set anyway
+        accountingDict["RegistrationTime"] = 0.0
+        accountingDict["RegistrationOK"] = 0
+        accountingDict["RegistrationTotal"] = 0
+
         # We cannot rely on all the transient attributes (like self.filesToSubmit)
         # because it is probably not filed by the time we monitor !
 
         filesInfoList = jobStatusDict["files"]
         successfulFiles = []
+        failedFiles = []
 
         for fileDict in filesInfoList:
             file_state = fileDict["file_state"].capitalize()
             if file_state in FTS3File.FTS_SUCCESS_STATES:
                 successfulFiles.append(fileDict)
+            else:
+                failedFiles.append(fileDict)
 
         job_metadata = jobStatusDict["job_metadata"]
         # previous version of the code did not have dictionary as
@@ -941,23 +950,31 @@ class FTS3Job(JSerializable):
             sourceSE = job_metadata.get("sourceSE")
             targetSE = job_metadata.get("targetSE")
 
-        accountingDict["TransferOK"] = len(successfulFiles)
-        accountingDict["TransferTotal"] = len(filesInfoList)
-        # We need this if in the list comprehension because staging only jobs have `None` as filesize
-        accountingDict["TransferSize"] = sum(
-            fileDict["filesize"] for fileDict in successfulFiles if fileDict["filesize"]
-        )
-        accountingDict["FinalStatus"] = self.status
         accountingDict["Source"] = sourceSE
         accountingDict["Destination"] = targetSE
-        # We need this if in the list comprehension because staging only jobs have `None` as tx_duration
-        accountingDict["TransferTime"] = sum(
-            int(fileDict["tx_duration"]) for fileDict in successfulFiles if fileDict["tx_duration"]
-        )
 
-        # Registration values must be set anyway
-        accountingDict["RegistrationTime"] = 0.0
-        accountingDict["RegistrationOK"] = 0
-        accountingDict["RegistrationTotal"] = 0
+        if successfulFiles:
+            successfulDict = accountingDict.copy()
+            successfulDict["TransferOK"] = len(successfulFiles)
+            successfulDict["TransferTotal"] = len(successfulFiles)
+            # We need this if in the list comprehension because staging only jobs have `None` as filesize
+            successfulDict["TransferSize"] = sum(
+                fileDict["filesize"] for fileDict in successfulFiles if fileDict["filesize"]
+            )
+            successfulDict["FinalStatus"] = "Finished"
 
-        self.accountingDict = accountingDict
+            # We need this if in the list comprehension because staging only jobs have `None` as tx_duration
+            successfulDict["TransferTime"] = sum(
+                int(fileDict["tx_duration"]) for fileDict in successfulFiles if fileDict["tx_duration"]
+            )
+            accountingDicts.append(successfulDict)
+        if failedFiles:
+            failedDict = accountingDict.copy()
+            failedDict["TransferOK"] = 0
+            failedDict["TransferTotal"] = len(failedFiles)
+            failedDict["TransferSize"] = 0
+            failedDict["FinalStatus"] = "Failed"
+            failedDict["TransferTime"] = 0
+            accountingDicts.append(failedDict)
+
+        self.accountingDicts = accountingDicts

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/FrameworkSystem/DB/ProxyDB.py

@@ -11,7 +11,7 @@
 import textwrap
 from threading import Lock
 
-from cachetools import TTLCache, cached
+from cachetools import TTLCache, cachedmethod
 
 from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
@@ -25,6 +25,10 @@ from DIRAC.Resources.ProxyProvider.ProxyProviderFactory import ProxyProviderFact
 
 DEFAULT_MAIL_FROM = "proxymanager@diracgrid.org"
 
+# Module-level cache for getProxyStrength method (shared across ProxyDB instances)
+_get_proxy_strength_cache = TTLCache(maxsize=1000, ttl=600)
+_get_proxy_strength_lock = Lock()
+
 
 class ProxyDB(DB):
     NOTIFICATION_TIMES = [2592000, 1296000]
@@ -398,7 +402,7 @@ class ProxyDB(DB):
             return S_ERROR(", ".join(errMsgs))
         return result
 
-    @cached(TTLCache(maxsize=1000, ttl=600), lock=Lock())
+    @cachedmethod(lambda self: _get_proxy_strength_cache, lock=lambda self: _get_proxy_strength_lock)
     def getProxyStrength(self, userDN, userGroup=None, vomsAttr=None):
         """Load the proxy in cache corresponding to the criteria, and check its strength
 

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py

@@ -121,7 +121,9 @@ class FreeDiskSpaceCommand(Command):
             "Site": siteRes["Value"] if siteRes["Value"] else "unassigned",
         }
 
-        results["Used"] = results["Total"] - results["Free"]
+        # There are sometimes small discrepencies which can lead to negative
+        # used values.
+        results["Used"] = max(0, results["Total"] - results["Free"])
 
         for sType in ["Total", "Free", "Used"]:
             spaceTokenAccounting = StorageOccupancy()

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/Resources/Computing/AREXComputingElement.py

@@ -1,4 +1,4 @@
-""" AREX Computing Element (ARC REST interface)
+"""AREX Computing Element (ARC REST interface)
 
 Allows interacting with ARC AREX services via a REST interface.
 
@@ -807,7 +807,23 @@ class AREXComputingElement(ComputingElement):
             return S_ERROR(f"Failed decoding the status of the CE")
 
         # Look only in the relevant section out of the headache
-        queueInfo = ceData["Domains"]["AdminDomain"]["Services"]["ComputingService"]["ComputingShare"]
+        # This "safe_get" function allows to go down the dictionary
+        # even if some elements are lists instead of dictionaries
+        # and returns None if any element is not found
+        # FIXME: this is a temporary measure to be removed after https://github.com/DIRACGrid/DIRAC/issues/8354
+        def safe_get(d, *keys):
+            for k in keys:
+                if isinstance(d, list):
+                    d = d[0]  # assume first element
+                d = d.get(k) if isinstance(d, dict) else None
+                if d is None:
+                    break
+            return d
+
+        queueInfo = safe_get(ceData, "Domains", "AdminDomain", "Services", "ComputingService", "ComputingShare")
+        if queueInfo is None:
+            self.log.error("Failed to extract queue info")
+
         if not isinstance(queueInfo, list):
             queueInfo = [queueInfo]
 

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/TransformationSystem/Client/Utilities.py

@@ -9,7 +9,7 @@ Utilities for Transformation system
 import ast
 import random
 
-from cachetools import LRUCache, cached
+from cachetools import LRUCache, cachedmethod
 from cachetools.keys import hashkey
 from DIRAC import S_OK, S_ERROR, gLogger
 
@@ -24,6 +24,9 @@ from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
 
+# Module-level cache for isSameSEInList method (shared across PluginUtilities instances)
+_is_same_se_in_list_cache = LRUCache(maxsize=1024)
+
 
 class PluginUtilities:
     """
@@ -402,8 +405,8 @@ class PluginUtilities:
 
         return StorageElement(se1).isSameSE(StorageElement(se2))
 
-    @cached(
-        LRUCache(maxsize=1024),
+    @cachedmethod(
+        lambda self: _is_same_se_in_list_cache,
         key=lambda _, a, b: hashkey(a, *sorted(b)),
     )
     def isSameSEInList(self, se1, seList):

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -1,4 +1,4 @@
-"""  The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.
+"""The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.
 
 .. literalinclude:: ../ConfigTemplate.cfg
   :start-after: ##BEGIN SiteDirector
@@ -7,6 +7,7 @@
   :caption: SiteDirector options
 
 """
+
 import datetime
 import os
 from collections import defaultdict
@@ -147,10 +148,10 @@ class SiteDirector(AgentModule):
             self.sendSubmissionAccounting = True
 
         # Get the site description dictionary
-        siteNames = self.am_getOption("Site", [])
-        ceTypes = self.am_getOption("CETypes", [])
-        ces = self.am_getOption("CEs", [])
-        tags = self.am_getOption("Tags", [])
+        siteNames = self.am_getOption("Site")
+        ceTypes = self.am_getOption("CETypes")
+        ces = self.am_getOption("CEs")
+        tags = self.am_getOption("Tags")
 
         # Display options used
         self.log.always("VO:", self.vo)

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py

@@ -1,6 +1,7 @@
-""" The Download Input Data module wraps around the Replica Management
-    components to provide access to datasets by downloading locally
+"""The Download Input Data module wraps around the Replica Management
+components to provide access to datasets by downloading locally
 """
+
 import os
 import random
 import tempfile
@@ -226,7 +227,7 @@ class DownloadInputData:
         diskSpace = getDiskSpace(self.__getDownloadDir(False))  # MB
         availableBytes = diskSpace * 1024 * 1024  # bytes
         bufferGBs = gConfig.getValue(
-            os.path.join("/Systems/WorkloadManagement/JobWrapper", "JobWrapper", "MinOutputDataBufferGB"), 5.0
+            os.path.join("/Systems/WorkloadManagement/JobWrapper", "MinOutputDataBufferGB"), 5.0
         )
         data = bufferGBs * 1024 * 1024 * 1024  # bufferGBs in bytes
         if (data + totalSize) < availableBytes:

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg

@@ -263,14 +263,10 @@ Agents
     # the DN of the certificate proxy used to submit pilots. If not found here, what is in Operations/Pilot section of the CS will be used
     PilotDN =
 
-    # List of sites that will be treated by this SiteDirector (No value can refer to any Site defined in the CS)
-    Site =
-    # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
-    CEs =
-    # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
-    CETypes =
-    # List of Tags that are required to be present in the CE/Queue definition
-    Tags =
+    # Site = # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
+    # CEs =  # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
+    # CETypes = # List of Tags that are required to be present in the CE/Queue definition
+    # Tags =
 
     # How many cycles to skip if queue is not working
     FailedQueueCycleFactor = 10

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py

@@ -150,37 +150,24 @@ class WMSAdministratorHandlerMixin:
         :param str jobID: job ID
         :return: S_OK(dict)/S_ERROR()
         """
-        pilotReference = ""
-        # Get the pilot grid reference first from the job parameters
-
-        credDict = self.getRemoteCredentials()
-        vo = credDict.get("VO", Registry.getVOForGroup(credDict["group"]))
-        res = self.elasticJobParametersDB.getJobParameters(int(jobID), vo=vo, paramList=["Pilot_Reference"])
-        if not res["OK"]:
-            return res
-        if res["Value"].get(int(jobID)):
-            pilotReference = res["Value"][int(jobID)]["Pilot_Reference"]
-
-        if not pilotReference:
-            res = self.jobDB.getJobParameter(int(jobID), "Pilot_Reference")
-            if not res["OK"]:
-                return res
-            pilotReference = res["Value"]
-
-        if not pilotReference:
-            # Failed to get the pilot reference, try to look in the attic parameters
-            res = self.jobDB.getAtticJobParameters(int(jobID), ["Pilot_Reference"])
-            if res["OK"]:
-                c = -1
-                # Get the pilot reference for the last rescheduling cycle
-                for cycle in res["Value"]:
-                    if cycle > c:
-                        pilotReference = res["Value"][cycle]["Pilot_Reference"]
-                        c = cycle
-
-        if pilotReference:
-            return self.pilotManager.getPilotOutput(pilotReference)
-        return S_ERROR("No pilot job reference found")
+        result = self.pilotManager.getPilots(jobID)
+
+        if not result["OK"]:
+            return result
+        pilotJobReferences = result["Value"].keys()
+
+        outputs = {"StdOut": "", "StdErr": ""}
+        for pilotRef in pilotJobReferences:
+            result = self.pilotManager.getPilotOutput(pilotRef)
+            if not result["OK"]:
+                stdout = f"Could not retrieve output: {result['Message']}"
+                error = f"Could not retrieve error: {result['Message']}"
+            else:
+                stdout, error = result["Value"]["StdOut"], result["Value"]["StdErr"]
+            outputs["StdOut"] += f"# PilotJobReference: {pilotRef}\n\n{stdout}\n"
+            outputs["StdErr"] += f"# PilotJobReference: {pilotRef}\n\n{error}\n"
+
+        return S_OK(outputs)
 
 
 class WMSAdministratorHandler(WMSAdministratorHandlerMixin, RequestHandler):

{dirac-9.0.6 → dirac-9.0.7/src/DIRAC.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.6
+Version: 9.0.7
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.6
+Requires-Dist: DIRACCommon==v9.0.7
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1

{dirac-9.0.6 → dirac-9.0.7}/src/DIRAC.egg-info/requires.txt

@@ -4,7 +4,7 @@ cachetools
 certifi
 cwltool
 diraccfg
-DIRACCommon==v9.0.6
+DIRACCommon==v9.0.7
 diracx-client>=v0.0.1
 diracx-core>=v0.0.1
 diracx-cli>=v0.0.1