DIRAC 9.0.8__py3-none-any.whl → 9.0.10__py3-none-any.whl

DIRAC/Core/Utilities/ElasticSearchDB.py

@@ -82,7 +82,6 @@ def generateDocs(data, withTimeStamp=True):
 
 
 class ElasticSearchDB:
-
     """
     .. class:: ElasticSearchDB
 
@@ -506,7 +505,7 @@ class ElasticSearchDB:
             indexName = self.generateFullIndexName(indexPrefix, period)
         else:
             indexName = indexPrefix
-        sLog.debug(f"Bulk indexing into {indexName} of {data}")
+        sLog.debug(f"Bulk indexing into {indexName} of {len(data)}")
 
         res = self.existingIndex(indexName)
         if not res["OK"]:

DIRAC/Core/Utilities/Subprocess.py

@@ -1,7 +1,7 @@
 """
 DIRAC Wrapper to execute python and system commands with a wrapper, that might
 set a timeout.
-3 FUNCTIONS are provided:
+3 functions are provided:
 
      - shellCall( iTimeOut, cmdSeq, callbackFunction = None, env = None ):
        it uses subprocess.Popen class with "shell = True".
@@ -26,6 +26,7 @@ set a timeout.
        should be used to wrap third party python functions
 
 """
+
 import os
 import selectors
 import signal
@@ -161,7 +162,6 @@ class Subprocess:
 
         self.child = None
         self.childPID = 0
-        self.childKilled = False
         self.callback = None
         self.bufferList = []
         self.cmdSeq = []
@@ -193,7 +193,7 @@ class Subprocess:
                     f"First and last data in buffer: \n{dataString[:100]} \n....\n {dataString[-100:]} ",
                 )
                 retDict = S_ERROR(
-                    "Reached maximum allowed length (%d bytes) " "for called function return value" % self.bufferLimit
+                    "Reached maximum allowed length (%d bytes) for called function return value" % self.bufferLimit
                 )
                 retDict["Value"] = dataString
                 return retDict
@@ -241,60 +241,71 @@ class Subprocess:
         events = sel.select(timeout=timeout or self.timeout or None)
         return [key.fileobj for key, event in events if event & selectors.EVENT_READ]
 
-    def __killPid(self, pid, sig=9):
-        """send signal :sig: to process :pid:
-
-        :param int pid: process id
-        :param int sig: signal to send, default 9 (SIGKILL)
-        """
+    def __terminateProcess(self, process):
+        """Tries to terminate a process with SIGTERM. Returns a (gone, alive) tuple"""
+        self.log.verbose(f"Sending SIGTERM signal to PID {process.pid}")
         try:
-            os.kill(pid, sig)
-        except Exception as x:
-            if str(x) != "[Errno 3] No such process":
-                self.log.exception("Exception while killing timed out process")
-                raise x
-
-    def __poll(self, pid):
-        """wait for :pid:"""
+            process.terminate()
+        except psutil.NoSuchProcess:
+            return ([], [])
+        return psutil.wait_procs([process], timeout=60)
+
+    def __poll(self, process):
+        """Non-blocking check of whether process `pid` is still alive.
+        Returns:
+        - (0, 0) if process is still running  (like os.waitpid(pid, os.WNOHANG))
+        - (pid, exitcode) if process has terminated
+        - None if process info cannot be retrieved
+        """
         try:
-            return os.waitpid(pid, os.WNOHANG)
-        except os.error:
-            if self.childKilled:
-                return False
+            exitcode = process.wait(timeout=0)
+            return (process.pid, exitcode)  # exited
+        except psutil.TimeoutExpired:
+            return (0, 0)  # still running
+        except psutil.NoSuchProcess:
             return None
 
     def killChild(self, recursive=True):
-        """kill child process
-
-        :param boolean recursive: flag to kill all descendants
+        """Kills a process tree (including children) with signal SIGTERM. If that fails, escalate to SIGKILL
+        returns (gone, alive) tuple.
         """
-        pgid = os.getpgid(self.childPID)
-        if pgid != os.getpgrp():
-            try:
-                # Child is in its own group: kill the group
-                os.killpg(pgid, signal.SIGTERM)
-            except OSError:
-                # Process is already dead
-                pass
-        else:
-            # No separate group: walk the tree
-            parent = psutil.Process(self.childPID)
-            procs = parent.children(recursive=recursive)
-            procs.append(parent)
-            for p in procs:
+
+        self.log.info(f"Killing childPID {self.childPID}")
+
+        gone, alive = [], []
+        try:
+            child_process = psutil.Process(self.childPID)
+        except psutil.NoSuchProcess:
+            self.log.warn(f"Child PID {self.childPID} no longer exists")
+            return (gone, alive)
+
+        if recursive:
+            # grandchildren
+            children = child_process.children(recursive=True)
+            self.log.info(f"Sending kill signal to {len(children)} children PIDs")
+            for p in children:
                 try:
                     p.terminate()
                 except psutil.NoSuchProcess:
-                    pass
-            _gone, alive = psutil.wait_procs(procs, timeout=10)
-            # Escalate any survivors
+                    continue
+            g, a = psutil.wait_procs(children, timeout=60)
+            gone.extend(g)
+            alive.extend(a)
+
+        # now killing the child_process
+        g, a = self.__terminateProcess(child_process)
+        gone.extend(g)
+        alive.extend(a)
+
+        # if there's something still alive, use SIGKILL
+        if alive:
             for p in alive:
                 try:
                     p.kill()
                 except psutil.NoSuchProcess:
                     pass
 
-        self.childKilled = True
+        return psutil.wait_procs(alive, timeout=60)
 
     def pythonCall(self, function, *stArgs, **stKeyArgs):
         """call python function :function: with :stArgs: and :stKeyArgs:"""
@@ -309,8 +320,6 @@ class Subprocess:
         if pid == 0:
             os.close(readFD)
             self.__executePythonFunction(function, writeFD, *stArgs, **stKeyArgs)
-            # FIXME: the close it is done at __executePythonFunction, do we need it here?
-            os.close(writeFD)
         else:
             os.close(writeFD)
             readSeq = self.__selectFD([readFD])
@@ -319,14 +328,13 @@ class Subprocess:
             try:
                 if len(readSeq) == 0:
                     self.log.debug("Timeout limit reached for pythonCall", function.__name__)
-                    self.__killPid(pid)
-
-                    # HACK to avoid python bug
-                    # self.wait()
-                    retries = 10000
-                    while os.waitpid(pid, 0) == -1 and retries > 0:
-                        time.sleep(0.001)
-                        retries -= 1
+                    gone, alive = self.__terminateProcess(psutil.Process(pid))
+                    if alive:
+                        for p in alive:
+                            try:
+                                p.kill()
+                            except psutil.NoSuchProcess:
+                                continue
 
                     return S_ERROR('%d seconds timeout for "%s" call' % (self.timeout, function.__name__))
                 elif readSeq[0] == readFD:
@@ -400,7 +408,7 @@ class Subprocess:
         if len(dataString) + baseLength > self.bufferLimit:
             self.log.error("Maximum output buffer length reached")
             retDict = S_ERROR(
-                "Reached maximum allowed length (%d bytes) for called " "function return value" % self.bufferLimit
+                "Reached maximum allowed length (%d bytes) for called function return value" % self.bufferLimit
             )
             retDict["Value"] = dataString
             return retDict
@@ -446,6 +454,7 @@ class Subprocess:
                 start_new_session=start_new_session,
             )
             self.childPID = self.child.pid
+            child_process = psutil.Process(self.childPID)
         except OSError as v:
             retDict = S_ERROR(repr(v))
             retDict["Value"] = (-1, "", str(v))
@@ -464,9 +473,9 @@ class Subprocess:
             self.bufferList = [["", 0], ["", 0]]
             initialTime = time.time()
 
-            exitStatus = self.__poll(self.child.pid)
+            exitStatus = self.__poll(child_process)
 
-            while (0, 0) == exitStatus or exitStatus is None:
+            while (0, 0) == exitStatus:  # This means that the process is still alive
                 retDict = self.__readFromCommand()
                 if not retDict["OK"]:
                     return retDict
@@ -478,14 +487,14 @@ class Subprocess:
                         1, "Timeout (%d seconds) for '%s' call" % (self.timeout, cmdSeq)
                     )
                 time.sleep(0.01)
-                exitStatus = self.__poll(self.child.pid)
+                exitStatus = self.__poll(child_process)
 
             self.__readFromCommand()
 
             if exitStatus:
                 exitStatus = exitStatus[1]
 
-            if exitStatus >= 256:
+            if exitStatus and exitStatus >= 256:
                 exitStatus = int(exitStatus / 256)
             return S_OK((exitStatus, self.bufferList[0][0], self.bufferList[1][0]))
         finally:

DIRAC/Core/Utilities/test/Test_Subprocess.py

@@ -3,23 +3,25 @@
 # Date: 2012/12/11 18:04:25
 ########################################################################
 
-""" :mod: SubprocessTests
-    =======================
+""":mod: SubprocessTests
+=======================
 
-    .. module: SubprocessTests
-    :synopsis: unittest for Subprocess module
-    .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
+.. module: SubprocessTests
+:synopsis: unittest for Subprocess module
+.. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
 
-    unittest for Subprocess module
+unittest for Subprocess module
 """
-import time
+
 import platform
+import time
 from os.path import dirname, join
 from subprocess import Popen
 
+import psutil
 import pytest
 
-from DIRAC.Core.Utilities.Subprocess import systemCall, shellCall, pythonCall, getChildrenPIDs, Subprocess
+from DIRAC.Core.Utilities.Subprocess import Subprocess, getChildrenPIDs, pythonCall, shellCall, systemCall
 
 # Mark this entire module as slow
 pytestmark = pytest.mark.slow
@@ -72,3 +74,51 @@ def test_decodingCommandOutput():
     retVal = sp.systemCall(r"""python -c 'import os; os.fdopen(2, "wb").write(b"\xdf")'""", shell=True)
     assert retVal["OK"]
     assert retVal["Value"] == (0, "", "\ufffd")
+
+
+@pytest.fixture
+def subprocess_instance():
+    """Provides a Subprocess instance for testing."""
+    subp = Subprocess()
+    return subp
+
+
+@pytest.fixture
+def dummy_child():
+    """Spawn a dummy process tree: parent -> child."""
+    # Start a shell that sleeps, with a subprocess child
+    parent = Popen(["bash", "-c", "sleep 10 & wait"])
+    time.sleep(0.2)  # give it a moment to start
+    yield parent
+    # Ensure cleanup
+    try:
+        parent.terminate()
+        parent.wait(timeout=1)
+    except Exception:
+        pass
+
+
+def test_kill_child_process_tree(subprocess_instance, dummy_child):
+    """Test that killChild kills both parent and its children."""
+    subprocess_instance.childPID = dummy_child.pid
+    parent_proc = psutil.Process(subprocess_instance.childPID)
+
+    # Sanity check: parent should exist
+    assert parent_proc.is_running()
+
+    # It should have at least one sleeping child
+    children = parent_proc.children(recursive=True)
+    assert children, "Expected dummy process to have at least one child"
+
+    # Kill the tree
+    gone, alive = subprocess_instance.killChild(recursive=True)
+
+    # Verify the parent and children are terminated
+    for p in gone:
+        assert not p.is_running(), f"Process {p.pid} still alive"
+    for p in alive:
+        assert not p.is_running(), f"Process {p.pid} still alive"
+
+    # Verify parent is gone
+    with pytest.raises(psutil.NoSuchProcess):
+        psutil.Process(subprocess_instance.childPID)

DIRAC/DataManagementSystem/Client/DataManager.py

@@ -10,11 +10,11 @@ This module consists of DataManager and related classes.
 """
 
 # # imports
-from datetime import datetime, timedelta
+import errno
 import fnmatch
 import os
 import time
-import errno
+from datetime import datetime, timedelta
 
 # # from DIRAC
 import DIRAC
@@ -25,13 +25,13 @@ from DIRAC.Core.Utilities.File import makeGuid, getSize
 from DIRAC.Core.Utilities.List import randomize, breakListIntoChunks
 from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
+from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.DataManagementSystem.Client import MAX_FILENAME_LENGTH
-from DIRAC.MonitoringSystem.Client.DataOperationSender import DataOperationSender
 from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
+from DIRAC.MonitoringSystem.Client.DataOperationSender import DataOperationSender
 from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
 from DIRAC.Resources.Storage.StorageElement import StorageElement
-from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
 
 
 # # RSCID
@@ -89,7 +89,7 @@ class DataManager:
         :param vo: the VO for which the DataManager is created, get VO from the current proxy if not specified
         """
         self.log = gLogger.getSubLogger(self.__class__.__name__)
-        self.voName = vo
+        self.voName = vo if vo else getVOfromProxyGroup().get("Value", None)
 
         if catalogs is None:
             catalogs = []
@@ -97,10 +97,9 @@ class DataManager:
 
         self.fileCatalog = FileCatalog(catalogs=catalogsToUse, vo=self.voName)
         self.accountingClient = None
-        self.resourceStatus = ResourceStatus()
         self.ignoreMissingInFC = Operations(vo=self.voName).getValue("DataManagement/IgnoreMissingInFC", False)
         self.useCatalogPFN = Operations(vo=self.voName).getValue("DataManagement/UseCatalogPFN", True)
-        self.dmsHelper = DMSHelpers(vo=vo)
+        self.dmsHelper = DMSHelpers(vo=self.voName)
         self.registrationProtocol = self.dmsHelper.getRegistrationProtocols()
         self.thirdPartyProtocols = self.dmsHelper.getThirdPartyProtocols()
         self.dataOpSender = DataOperationSender()

DIRAC/Interfaces/Utilities/DConfigCache.py

@@ -9,6 +9,7 @@ from DIRAC import gLogger
 from DIRAC.Core.Base.Script import Script
 from DIRAC.Core.Utilities.File import secureOpenForWrite
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
+from DIRAC.ConfigurationSystem.Client.Helpers.Registry import reset_all_caches
 
 
 class ConfigCache:
@@ -69,5 +70,6 @@ class ConfigCache:
             try:
                 with open(self.configCacheName, "rb") as fh:
                     gConfigurationData.mergedCFG = pickle.load(fh)
+                    reset_all_caches()
             except:
                 gLogger.error("Cache corrupt or unreadable")

DIRAC/TransformationSystem/Agent/InputDataAgent.py

@@ -74,7 +74,10 @@ class InputDataAgent(AgentModule):
         """Main execution method"""
 
         # Get all the transformations
-        result = self.transClient.getTransformations({"Status": "Active", "Type": self.transformationTypes})
+        result = self.transClient.getTransformations(
+            {"Status": "Active", "Type": self.transformationTypes},
+            columns=["TransformationID", "AuthorDN", "AuthorGroup"],
+        )
         if not result["OK"]:
             self.log.error("InputDataAgent.execute: Failed to get transformations.", result["Message"])
             return S_OK()

DIRAC/TransformationSystem/Agent/MCExtensionAgent.py

@@ -1,4 +1,4 @@
-""" Agent to extend the number of tasks given the Transformation definition
+"""Agent to extend the number of tasks given the Transformation definition
 
 The following options can be set for the MCExtensionAgent.
 
@@ -8,6 +8,7 @@ The following options can be set for the MCExtensionAgent.
   :dedent: 2
   :caption: MCExtensionAgent options
 """
+
 from DIRAC import S_OK, gLogger
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
@@ -54,7 +55,9 @@ class MCExtensionAgent(AgentModule):
             return S_OK("Disabled via CS flag")
 
         # Obtain the transformations in Cleaning status and remove any mention of the jobs/files
-        res = self.transClient.getTransformations({"Status": "Active", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "Active", "Type": self.transformationTypes}, columns=["TransformationID", "MaxNumberOfTasks"]
+        )
         if res["OK"]:
             for transDict in res["Value"]:
                 transID = transDict["TransformationID"]

DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py

@@ -231,7 +231,9 @@ class TaskManagerAgentBase(AgentModule, TransformationAgentsUtilities):
             selectCond["Type"] = transType
         if agentType:
             selectCond["AgentType"] = agentType
-        res = self.transClient.getTransformations(condDict=selectCond)
+        res = self.transClient.getTransformations(
+            condDict=selectCond, columns=["TransformationID", "Body", "Author", "AuthorGroup"]
+        )
         if not res["OK"]:
             self.log.error("Failed to get transformations:", res["Message"])
         elif not res["Value"]:

DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py

@@ -144,7 +144,8 @@ class TransformationCleaningAgent(AgentModule):
 
         # Obtain the transformations in Cleaning status and remove any mention of the jobs/files
         res = self.transClient.getTransformations(
-            {"Status": TransformationStatus.CLEANING, "Type": self.transformationTypes}
+            {"Status": TransformationStatus.CLEANING, "Type": self.transformationTypes},
+            columns=["TransformationID", "Author", "AuthorGroup", "Type"],
         )
         if res["OK"]:
             for transDict in res["Value"]:
@@ -161,7 +162,10 @@ class TransformationCleaningAgent(AgentModule):
             self.log.error("Failed to get transformations", res["Message"])
 
         # Obtain the transformations in RemovingFiles status and removes the output files
-        res = self.transClient.getTransformations({"Status": "RemovingFiles", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "RemovingFiles", "Type": self.transformationTypes},
+            columns=["TransformationID", "Author", "AuthorGroup"],
+        )
         if res["OK"]:
             for transDict in res["Value"]:
                 if self.shifterProxy:
@@ -183,6 +187,7 @@ class TransformationCleaningAgent(AgentModule):
             {"Status": TransformationStatus.COMPLETED, "Type": self.transformationTypes},
             older=olderThanTime,
             timeStamp="LastUpdate",
+            columns=["TransformationID", "Author", "AuthorGroup"],
         )
         if res["OK"]:
             for transDict in res["Value"]:
@@ -230,7 +235,10 @@ class TransformationCleaningAgent(AgentModule):
             return res
         transformationIDs = res["Value"]
         if transformationIDs:
-            res = self.transClient.getTransformations({"TransformationID": transformationIDs})
+            res = self.transClient.getTransformations(
+                {"TransformationID": transformationIDs},
+                columns=["TransformationID", "Status", "Author", "AuthorGroup", "Type"],
+            )
             if not res["OK"]:
                 self.log.error("Failed to get transformations", res["Message"])
                 return res

DIRAC/TransformationSystem/Agent/ValidateOutputDataAgent.py

@@ -76,7 +76,9 @@ class ValidateOutputDataAgent(AgentModule):
         self.updateWaitingIntegrity()
         gLogger.info("-" * 40)
 
-        res = self.transClient.getTransformations({"Status": "ValidatingOutput", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "ValidatingOutput", "Type": self.transformationTypes}, columns=["TransformationID"]
+        )
         if not res["OK"]:
             gLogger.error("Failed to get ValidatingOutput transformations", res["Message"])
             return res
@@ -98,7 +100,7 @@ class ValidateOutputDataAgent(AgentModule):
     def updateWaitingIntegrity(self):
         """Get 'WaitingIntegrity' transformations, update to 'ValidatedOutput'"""
         gLogger.info("Looking for transformations in the WaitingIntegrity status to update")
-        res = self.transClient.getTransformations({"Status": "WaitingIntegrity"})
+        res = self.transClient.getTransformations({"Status": "WaitingIntegrity"}, columns=["TransformationID"])
         if not res["OK"]:
             gLogger.error("Failed to get WaitingIntegrity transformations", res["Message"])
             return res

DIRAC/TransformationSystem/Client/TransformationClient.py

@@ -114,13 +114,21 @@ class TransformationClient(Client):
         newer=None,
         timeStamp=None,
         orderAttribute=None,
-        limit=100,
+        limit=None,
         extraParams=False,
         columns=None,
     ):
         """gets all the transformations in the system, incrementally. "limit" here is just used to determine the offset."""
         rpcClient = self._getRPC()
 
+        # If the body is requested (or is served by default)
+        # we take smaller chunk, not to take too much memory
+        # on the server
+        if columns and "Body" not in columns:
+            limit = 100_000
+        else:
+            limit = 1_000
+
         transformations = []
         if condDict is None:
             condDict = {}

DIRAC/TransformationSystem/DB/TransformationDB.py

@@ -6,6 +6,9 @@ This class is typically used as a base class for more specific data processing
 databases
 """
 
+# Disable it because pylint does not understand decorator (convertToReturnValue)
+
+# pylint: disable=invalid-sequence-index
 import re
 import time
 import threading
@@ -15,6 +18,7 @@ from errno import ENOENT
 from DIRAC import gLogger, S_OK, S_ERROR
 from DIRAC.Core.Base.DB import DB
 from DIRAC.Core.Utilities.DErrno import cmpError
+from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueOrRaise
 from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
 from DIRAC.Core.Utilities.List import stringListToString, intListToString, breakListIntoChunks
@@ -25,6 +29,7 @@ from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery
 
 MAX_ERROR_COUNT = 10
 
+TMP_TABLE_JOIN_LIMIT = 100
 #############################################################################
 
 
@@ -270,6 +275,7 @@ class TransformationDB(DB):
         self.__updateTransformationLogging(transID, message, author, connection=connection)
         return S_OK(transID)
 
+    @convertToReturnValue
     def getTransformations(
         self,
         condDict=None,
@@ -289,32 +295,54 @@ class TransformationDB(DB):
             columns = self.TRANSPARAMS
         else:
             columns = [c for c in columns if c in self.TRANSPARAMS]
-        req = "SELECT {} FROM Transformations {}".format(
-            intListToString(columns),
-            self.buildCondition(condDict, older, newer, timeStamp, orderAttribute, limit, offset=offset),
-        )
-        res = self._query(req, conn=connection)
-        if not res["OK"]:
-            return res
-        if condDict is None:
-            condDict = {}
-        webList = []
+
+        join_query = ""
+
+        try:
+            # If we request multiple TransformationIDs, and they are more than TMP_TABLE_JOIN_LIMIT,
+            # we create a temporary table to speed up the query
+            if (
+                "TransformationID" in condDict
+                and isinstance(condDict["TransformationID"], list)
+                and len(condDict["TransformationID"]) > TMP_TABLE_JOIN_LIMIT
+            ):
+                # Create temporary table for TransformationIDs
+                transIDs = condDict.pop("TransformationID")
+                sqlCmd = "CREATE TEMPORARY TABLE to_query_TransformationIDs (TransID INTEGER NOT NULL, PRIMARY KEY (TransID)) ENGINE=MEMORY;"
+                returnValueOrRaise(self._update(sqlCmd, conn=connection))
+                join_query = " JOIN to_query_TransformationIDs t ON TransformationID = t.TransID"
+
+                # Insert TransformationIDs into temporary table
+                sqlCmd = "INSERT INTO to_query_TransformationIDs (TransID) VALUES ( %s )"
+                returnValueOrRaise(self._updatemany(sqlCmd, [(transID,) for transID in transIDs], conn=connection))
+
+            req = "SELECT {} FROM Transformations {} {}".format(
+                intListToString(columns),
+                join_query,
+                self.buildCondition(condDict, older, newer, timeStamp, orderAttribute, limit, offset=offset),
+            )
+            matching_transformations = returnValueOrRaise(self._query(req, conn=connection))
+
+        finally:
+            # Clean up temporary table
+            if join_query:
+                sqlCmd = "DROP TEMPORARY TABLE to_query_TransformationIDs"
+                self._update(sqlCmd, conn=connection)
+
+        # TODO: optimize by getting all the extra params at once
         resultList = []
-        for row in res["Value"]:
+        for row in matching_transformations:
             # Prepare the structure for the web
-            rList = [str(item) if not isinstance(item, int) else item for item in row]
             transDict = dict(zip(columns, row))
-            webList.append(rList)
             if extraParams:
-                res = self.__getAdditionalParameters(transDict["TransformationID"], connection=connection)
-                if not res["OK"]:
-                    return res
-                transDict.update(res["Value"])
+                trans_extra_param = returnValueOrRaise(
+                    self.__getAdditionalParameters(transDict["TransformationID"], connection=connection)
+                )
+
+                transDict.update(trans_extra_param)
             resultList.append(transDict)
-        result = S_OK(resultList)
-        result["Records"] = webList
-        result["ParameterNames"] = columns
-        return result
+
+        return resultList
 
     def getTransformation(self, transName, extraParams=False, connection=False):
         """Get Transformation definition and parameters of Transformation identified by TransformationID"""
@@ -710,21 +738,38 @@ class TransformationDB(DB):
         countDict["Total"] = sum(countDict.values())
         return S_OK(countDict)
 
+    @convertToReturnValue
     def __addFilesToTransformation(self, transID, fileIDs, connection=False):
-        req = "SELECT FileID from TransformationFiles"
-        req = req + " WHERE TransformationID = %d AND FileID IN (%s);" % (transID, intListToString(fileIDs))
-        res = self._query(req, conn=connection)
-        if not res["OK"]:
-            return res
-        for tupleIn in res["Value"]:
-            fileIDs.remove(tupleIn[0])
-        if not fileIDs:
-            return S_OK([])
-        values = [(transID, fileID) for fileID in fileIDs]
-        req = "INSERT INTO TransformationFiles (TransformationID,FileID,LastUpdate,InsertedTime) VALUES (%s, %s, UTC_TIMESTAMP(), UTC_TIMESTAMP())"
-        if not (res := self._updatemany(req, values, conn=connection))["OK"]:
-            return res
-        return S_OK(fileIDs)
+        # Create temporary table for FileIDs
+        sqlCmd = "CREATE TEMPORARY TABLE to_query_FileIDs (FileID INT(11) UNSIGNED NOT NULL, PRIMARY KEY (FileID)) ENGINE=MEMORY;"
+        returnValueOrRaise(self._update(sqlCmd, conn=connection))
+
+        try:
+            # Insert FileIDs into temporary table
+            sqlCmd = "INSERT INTO to_query_FileIDs (FileID) VALUES ( %s )"
+            returnValueOrRaise(self._updatemany(sqlCmd, [(fileID,) for fileID in fileIDs], conn=connection))
+
+            # Query existing files using JOIN
+            req = (
+                "SELECT tf.FileID FROM TransformationFiles tf JOIN to_query_FileIDs t ON tf.FileID = t.FileID WHERE tf.TransformationID = %d;"
+                % transID
+            )
+            res = returnValueOrRaise(self._query(req, conn=connection))
+
+            # Remove already existing fileIDs using set difference for efficiency
+            existingFileIDs = {tupleIn[0] for tupleIn in res}
+            fileIDs = list(set(fileIDs) - existingFileIDs)
+            if not fileIDs:
+                return []
+
+            values = [(transID, fileID) for fileID in fileIDs]
+            req = "INSERT INTO TransformationFiles (TransformationID,FileID,LastUpdate,InsertedTime) VALUES (%s, %s, UTC_TIMESTAMP(), UTC_TIMESTAMP())"
+            returnValueOrRaise(self._updatemany(req, values, conn=connection))
+            return fileIDs
+        finally:
+            # Clean up temporary table
+            sqlCmd = "DROP TEMPORARY TABLE to_query_FileIDs"
+            returnValueOrRaise(self._update(sqlCmd, conn=connection))
 
     def __insertExistingTransformationFiles(self, transID, fileTuplesList, connection=False):
         """Inserting already transformation files in TransformationFiles table (e.g. for deriving transformations)"""
@@ -1271,18 +1316,35 @@ class TransformationDB(DB):
     # These methods manipulate the DataFiles table
     #
 
+    @convertToReturnValue
     def __getFileIDsForLfns(self, lfns, connection=False):
         """Get file IDs for the given list of lfns
         warning: if the file is not present, we'll see no errors
         """
-        req = f"SELECT LFN,FileID FROM DataFiles WHERE LFN in ({stringListToString(lfns)});"
-        res = self._query(req, conn=connection)
-        if not res["OK"]:
-            return res
-        lfns = dict(res["Value"])
-        # Reverse dictionary
-        fids = {fileID: lfn for lfn, fileID in lfns.items()}
-        return S_OK((fids, lfns))
+
+        if not lfns:
+            return ({}, {})
+        # Create temporary table for LFNs
+        sqlCmd = "CREATE TEMPORARY TABLE to_query_LFNs (LFN VARCHAR(255) NOT NULL, PRIMARY KEY (LFN)) ENGINE=MEMORY;"
+        returnValueOrRaise(self._update(sqlCmd, conn=connection))
+
+        try:
+            # Insert LFNs into temporary table
+            sqlCmd = "INSERT INTO to_query_LFNs (LFN) VALUES ( %s )"
+            returnValueOrRaise(self._updatemany(sqlCmd, [(lfn,) for lfn in lfns], conn=connection))
+
+            # Query using JOIN with temporary table
+            req = "SELECT df.LFN, df.FileID FROM DataFiles df JOIN to_query_LFNs t ON df.LFN = t.LFN;"
+            res = returnValueOrRaise(self._query(req, conn=connection))
+
+            lfns = dict(res)
+            # Reverse dictionary
+            fids = {fileID: lfn for lfn, fileID in lfns.items()}
+            return (fids, lfns)
+        finally:
+            # Clean up temporary table
+            sqlCmd = "DROP TEMPORARY TABLE to_query_LFNs"
+            self._update(sqlCmd, conn=connection)
 
     def __getLfnsForFileIDs(self, fileIDs, connection=False):
         """Get lfns for the given list of fileIDs"""

DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py

@@ -1,22 +1,22 @@
-"""  The Watchdog class is used by the Job Wrapper to resolve and monitor
-     the system resource consumption.  The Watchdog can determine if
-     a running job is stalled and indicate this to the Job Wrapper.
-     Furthermore, the Watchdog will identify when the Job CPU limit has been
-     exceeded and fail jobs meaningfully.
-
-     Information is returned to the WMS via the heart-beat mechanism.  This
-     also interprets control signals from the WMS e.g. to kill a running
-     job.
-
-     - Still to implement:
-          - CPU normalization for correct comparison with job limit
+"""The Watchdog class is used by the Job Wrapper to resolve and monitor
+the system resource consumption.  The Watchdog can determine if
+a running job is stalled and indicate this to the Job Wrapper.
+Furthermore, the Watchdog will identify when the Job CPU limit has been
+exceeded and fail jobs meaningfully.
+
+Information is returned to the WMS via the heart-beat mechanism.  This
+also interprets control signals from the WMS e.g. to kill a running
+job.
+
+- Still to implement:
+     - CPU normalization for correct comparison with job limit
 """
+
 import datetime
 import errno
 import getpass
 import math
 import os
-import signal
 import socket
 import time
 from pathlib import Path
@@ -32,28 +32,6 @@ from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus
 from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
 
 
-def kill_proc_tree(pid, sig=signal.SIGTERM, includeParent=True):
-    """Kill a process tree (including grandchildren) with signal
-    "sig" and return a (gone, still_alive) tuple.
-    called as soon as a child terminates.
-
-    Taken from https://psutil.readthedocs.io/en/latest/index.html#kill-process-tree
-    """
-    assert pid != os.getpid(), "won't kill myself"
-    parent = psutil.Process(pid)
-    children = parent.children(recursive=True)
-    if includeParent:
-        children.append(parent)
-    for p in children:
-        try:
-            p.send_signal(sig)
-        except psutil.NoSuchProcess:
-            pass
-    _gone, alive = psutil.wait_procs(children, timeout=10)
-    for p in alive:
-        p.kill()
-
-
 class Watchdog:
     #############################################################################
     def __init__(self, pid, exeThread, spObject, jobCPUTime, memoryLimit=0, processors=1, jobArgs={}):
@@ -212,7 +190,7 @@ class Watchdog:
             if self.littleTimeLeftCount == 0 and self.__timeLeft() == -1:
                 self.checkError = JobMinorStatus.JOB_EXCEEDED_CPU
                 self.log.error(self.checkError, self.timeLeft)
-                self.__killRunningThread()
+                self.spObject.killChild()
                 return S_OK()
 
             self.littleTimeLeftCount -= 1
@@ -321,7 +299,7 @@ class Watchdog:
 
                     self.log.info("=================END=================")
 
-            self.__killRunningThread()
+            self.spObject.killChild()
             return S_OK()
 
         recentStdOut = "None"
@@ -408,7 +386,7 @@ class Watchdog:
             if "Kill" in signalDict:
                 self.log.info("Received Kill signal, stopping job via control signal")
                 self.checkError = JobMinorStatus.RECEIVED_KILL_SIGNAL
-                self.__killRunningThread()
+                self.spObject.killChild()
             else:
                 self.log.info("The following control signal was sent but not understood by the watchdog:")
                 self.log.info(signalDict)
@@ -862,13 +840,6 @@ class Watchdog:
 
         return result
 
-    #############################################################################
-    def __killRunningThread(self):
-        """Will kill the running thread process and any child processes."""
-        self.log.info("Sending kill signal to application PID", self.spObject.getChildPID())
-        self.spObject.killChild()
-        return S_OK("Thread killed")
-
     #############################################################################
     def __sendSignOfLife(self, jobID, heartBeatDict, staticParamDict):
         """Sends sign of life 'heartbeat' signal and triggers control signal

DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py

@@ -1,5 +1,5 @@
-""" Test class for JobWrapper
-"""
+"""Test class for JobWrapper"""
+
 import os
 import shutil
 import tempfile
@@ -314,8 +314,7 @@ def test_processKilledSubprocess(mocker):
         result = jw.process("sleep 20", {})
 
     assert result["OK"]
-    assert result["Value"]["payloadStatus"] == 15  # SIGTERM
-    assert not result["Value"]["payloadOutput"]
+    assert result["Value"]["payloadStatus"] is None
     assert not result["Value"]["payloadExecutorError"]
     assert result["Value"]["watchdogError"] == "Job is stalled!"  # Error message from the watchdog
 

DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py

@@ -218,7 +218,7 @@ class PilotCStoJSONSynchronizer:
 
         preferredURLPatterns = gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns")
         if preferredURLPatterns:
-            pilotDict["PreferredURLPatterns"] = preferredURLPatterns
+            pilotDict["PreferredURLPatterns"] = preferredURLPatterns.replace(" ", "").split(",")
 
         self.log.debug("Got pilotDict", str(pilotDict))
 

{dirac-9.0.8.dist-info → dirac-9.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.8
+Version: 9.0.10
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.8
+Requires-Dist: DIRACCommon==v9.0.10
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1

{dirac-9.0.8.dist-info → dirac-9.0.10.dist-info}/RECORD

@@ -231,7 +231,7 @@ DIRAC/Core/Utilities/Devloader.py,sha256=nPEinBId-NzyG1zcjcnUQOO-pWmNJsmWkj-Xkt5
 DIRAC/Core/Utilities/DictCache.py,sha256=8Yo5RJnTKwL8NO960hnbNgrzGCnd-Iupk6JExoFFYxQ,12661
 DIRAC/Core/Utilities/Dictionaries.py,sha256=yYxhB0lycfU2rDxgvviqqPpEsaVxNPNS8ZlulHG1LLw,549
 DIRAC/Core/Utilities/DirectoryExplorer.py,sha256=t8__3M_1zparfvKbB_fR8hIB56mZtmBPgklm5FTxYFA,1212
-DIRAC/Core/Utilities/ElasticSearchDB.py,sha256=CxYdPsqCEpe3QOHRK5jiXIlfVj9K74LbTr-QW7jl49E,22731
+DIRAC/Core/Utilities/ElasticSearchDB.py,sha256=Kr4hmm5PbsGpEdeQ1GgjkRM9m7evCB9yR4BNgWxCiMQ,22735
 DIRAC/Core/Utilities/EventDispatcher.py,sha256=wL2jA2gFRrzMPORJ10WEQloxZPNdcfDjGHq8ENDJ86Y,3142
 DIRAC/Core/Utilities/ExecutorDispatcher.py,sha256=npBVxBxKNmSDklJIfUMv-eadHO3oHbaASGoFZp54A0o,28893
 DIRAC/Core/Utilities/Extensions.py,sha256=k0kgUjc50oFZ1e2XwOERJLQMSSCbnx3HwBO_xFTffSU,7535
@@ -259,7 +259,7 @@ DIRAC/Core/Utilities/ReturnValues.py,sha256=EHtnn1I3Lvq6RO9tTVZMlf19a8QGX1B6Qa7l
 DIRAC/Core/Utilities/Shifter.py,sha256=EkJGPR_zbL0SS3E2_cq1-MjkTH91ZaFJlh-tQ2PxDGk,2571
 DIRAC/Core/Utilities/SiteSEMapping.py,sha256=Uynx_evXQPmfbMKH-TfqzLq4e_7APVrP6gC20cJEBYs,5683
 DIRAC/Core/Utilities/StateMachine.py,sha256=e1rc0X6J8Cf_W9HlLnLgroLkdrksUQUxdkH_qSoZVoA,769
-DIRAC/Core/Utilities/Subprocess.py,sha256=FcdEKKEySh_62S8VLjWp_HnuFYkKAjqsS8FLpPMS1b8,22756
+DIRAC/Core/Utilities/Subprocess.py,sha256=_lNp7ncg6BYC2TLvkcqACelZSsb0_5oyM2Jq-T5YbFc,23315
 DIRAC/Core/Utilities/ThreadPool.py,sha256=tETtgUdW1dlq_7xp_nQUTxNC87jAkrNnwvuDpGS3ffE,11484
 DIRAC/Core/Utilities/ThreadSafe.py,sha256=EAxEMqEuYlzDvAaupW3fi2MY-3kGZJ0zbK9i7JhEmt4,1117
 DIRAC/Core/Utilities/ThreadScheduler.py,sha256=IzyS59NF0ZhQwGYLpZ-yMX8_6kXFSGL9FQapylnoPLE,6212
@@ -304,7 +304,7 @@ DIRAC/Core/Utilities/test/Test_Pfn.py,sha256=XWTXejQf_TnicaPbQ4ZyDsl1KXDpq3IMNFl
 DIRAC/Core/Utilities/test/Test_ProcessPool.py,sha256=NnfHNkhTOgADk-P9ds_ADuvcSNlK_XdoibRkk13r_NE,10890
 DIRAC/Core/Utilities/test/Test_Profiler.py,sha256=8QRRXm-PwJpjLGwAOJxnrpnnMbmVzXhJ1ZcQ9gDBX5c,4215
 DIRAC/Core/Utilities/test/Test_ReturnValues.py,sha256=w6Jz-Vblgu8RDXPzVi6BZjuFXcSmW-Zb0mcBgW1RWOw,1926
-DIRAC/Core/Utilities/test/Test_Subprocess.py,sha256=nAiVF5oMnle-4E8ijlWF-ilBfgN2VGkkFQQSKhIKN2o,2141
+DIRAC/Core/Utilities/test/Test_Subprocess.py,sha256=f9THTKnVdGcNwpZ0PtfCQz0UAtNmF06126-F0mUAxWQ,3605
 DIRAC/Core/Utilities/test/Test_entrypoints.py,sha256=z_3f7m59v3W6ZsqgeCFbJoBnMY-cKR_blKbPqcV7528,413
 DIRAC/Core/Utilities/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 DIRAC/Core/Workflow/Module.py,sha256=XF_iDyJ1wIwUesHLfDGz9BywHgEuOwqzC2f7fY3E7r4,12631
@@ -361,7 +361,7 @@ DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_ReplicateAndRegiste
 DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py,sha256=3XZCrFdp7Yk5oY9xlaL2itErA9xmNnCJgIIZ38FOiK0,8292
 DIRAC/DataManagementSystem/Client/ConsistencyInspector.py,sha256=dtKNfsVNud9x7aVzr-RI1jQZCqbEqgmaHmE5lxm7Qwo,31227
 DIRAC/DataManagementSystem/Client/DataIntegrityClient.py,sha256=2ug49tvNzEp1GO6nzcW42E43qPXnaJF-mpIVp1TkWFQ,9130
-DIRAC/DataManagementSystem/Client/DataManager.py,sha256=edLmRUHJtZEYAMyen3y9qwdG9XDl9-wc08qZh5pTPDA,82313
+DIRAC/DataManagementSystem/Client/DataManager.py,sha256=ME5a_0csev0X07PVXQV14BrDrvCfOhPWCWAwlcbrK60,82313
 DIRAC/DataManagementSystem/Client/DirectoryListing.py,sha256=CvQZ5Da5WhgwdDL8iLk0xVBjKZrBHTdwMKWwSJwzwEI,7582
 DIRAC/DataManagementSystem/Client/FTS3Client.py,sha256=Kk1wy4YnkYuOI6IltyDYZqkzxV7Zsf7ZuoKT12_VbTw,3531
 DIRAC/DataManagementSystem/Client/FTS3File.py,sha256=gZFN_Uxc2tblUCETb4qWPJ2M1tSmKUvTMiZUL-yJ6M0,3122
@@ -604,7 +604,7 @@ DIRAC/Interfaces/API/test/testWF.jdl,sha256=byby75rpCSYh2FuSqqMP1YALbEcpo7ix6U4x
 DIRAC/Interfaces/API/test/testWF.xml,sha256=PIUoxu00j3OAB8Wcx4Uf_vPpxniO6jMUJgEib9IkeIQ,4452
 DIRAC/Interfaces/API/test/testWFSIO.jdl,sha256=dNZbBaYcz7wnzxrSML7DvpQYxO7kqUd5FmmIj6ijV5U,420
 DIRAC/Interfaces/Utilities/DCommands.py,sha256=qcto59gNjD_Mg2bWYqorxgv3S2K91S5f0dLGxxkEZXc,21265
-DIRAC/Interfaces/Utilities/DConfigCache.py,sha256=nUPaybZgVOBSWwjrW2vYWG-PfHtm0sEi46zFqIGnz8A,2566
+DIRAC/Interfaces/Utilities/DConfigCache.py,sha256=tDiRio3wyvGRRKAJj-4BL8ZbYBhoW5q8SKJ928Uu3uE,2684
 DIRAC/Interfaces/scripts/__init__.py,sha256=bqdOR5-0t1MOV62P-oehaZd_d9YjGSWVTfW-gRXMqLg,44
 DIRAC/Interfaces/scripts/dcd.py,sha256=b5Yoc2amdnv-Aotujir4XlFrVVY7ASaxLzRAU02j63Y,1317
 DIRAC/Interfaces/scripts/dchgrp.py,sha256=0pFrWpppC9ktPlXRqcSSN_23tQR1gdB9JrTKBdQw2_Y,1692
@@ -1047,15 +1047,15 @@ DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py,sha256=zgOaJ6P
 DIRAC/TransformationSystem/ConfigTemplate.cfg,sha256=qm3MaoyX_zjhDGFG4NcAFBKLniWBaPcSkEZCKebYeh4,6730
 DIRAC/TransformationSystem/__init__.py,sha256=KTlQaDBvpqTf0WxwbYIOSppNJ34WZkbaTG1XBWEBvN0,43
 DIRAC/TransformationSystem/Agent/DataRecoveryAgent.py,sha256=JUfOBQd0U2uz9WvvArBRurgxyoVlBbg3rL5ZgdYs0Vw,26982
-DIRAC/TransformationSystem/Agent/InputDataAgent.py,sha256=YdxfB1wb55MBgZAuWNuqx_zXHaxTAEGCKYikr0eZaD8,8085
-DIRAC/TransformationSystem/Agent/MCExtensionAgent.py,sha256=0pmfiy8FTH670Vv3Md6TPo5iAXsu9y1e-1ybIOTQJPY,5266
+DIRAC/TransformationSystem/Agent/InputDataAgent.py,sha256=XytU-0aq0RJZhNNXxzvvn-AXsRBjt20Q3miFl5LBbYo,8177
+DIRAC/TransformationSystem/Agent/MCExtensionAgent.py,sha256=KXBgn_5r7oVIOX0d7gx5ok__sCxk4esJ8dX7hivS7C4,5338
 DIRAC/TransformationSystem/Agent/RequestTaskAgent.py,sha256=0FxuRVONDkC6DtDqfSC4LeZmheaevBsrh4yvan0DtVc,2689
-DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py,sha256=v61j-e_qMQXACENK0JYWa7lMgL0-xNT9irzEdXDHs-8,30652
+DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py,sha256=I1DGxItHZ_nYs1_cGZTTrsts01rQl2_JuBHtYTljTGw,30737
 DIRAC/TransformationSystem/Agent/TransformationAgent.py,sha256=rUPyfX_Ip3ZUyBrmVBrYiddVmgeTx6gKiYPEzwUDOcY,33813
 DIRAC/TransformationSystem/Agent/TransformationAgentsUtilities.py,sha256=XrgrjQWSDQm0uxG-wyEY-W0A_MBJyIE-22TafqrLP3A,2805
-DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py,sha256=uAu_aQxko4GF_cTxDxY4KmARqjuytBO89e4XVy5PPUM,32898
+DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py,sha256=FyEOWtp8_ZIMCEh1RYqLOENnB--unmKy6gmZkgLb7cM,33251
 DIRAC/TransformationSystem/Agent/TransformationPlugin.py,sha256=EpWil1Bla9xdrJkhj61oUFDpGzyWfC8acW3tjBh5JpQ,11322
-DIRAC/TransformationSystem/Agent/ValidateOutputDataAgent.py,sha256=tsE0yE2bqSKwdjJml3Mros-tMOfpSDr9Bt8R3MTeqRo,10108
+DIRAC/TransformationSystem/Agent/ValidateOutputDataAgent.py,sha256=YzjUVECnWYvJ2zQyqMqRa4BZLqq2FUulXnI4tO6HlpU,10190
 DIRAC/TransformationSystem/Agent/WorkflowTaskAgent.py,sha256=UBNVVx0ZgiFnzAPCNC1dSuJXKkt2pPfpSqhy0CREFlg,1741
 DIRAC/TransformationSystem/Agent/__init__.py,sha256=g5D8rYLJboNfnLrH_MZmfyvrXhp0pZBjBTtmr9ZJBQw,49
 DIRAC/TransformationSystem/Agent/RequestOperations/SetFileStatus.py,sha256=asFCK8oxR-p4AMEz6XdFBqft72xUzpqypBN57l_zOEc,1857
@@ -1069,7 +1069,7 @@ DIRAC/TransformationSystem/Client/TaskManager.py,sha256=lh3AsNh7Q-YXf7WCoywNAo1X
 DIRAC/TransformationSystem/Client/TaskManagerPlugin.py,sha256=mCTiFmqs7IpbDcF9wzfoRSMeuWMsxI4LcZr_9u2vr-c,5528
 DIRAC/TransformationSystem/Client/Transformation.py,sha256=Iw8Q4X8ZAUe4HlXAb8TKRvg4vVVnqoaINTAP_b2v7pI,26539
 DIRAC/TransformationSystem/Client/TransformationCLI.py,sha256=EFAbksFDLjX92KcT6VZlriXwFBLjkkxWd0qVWEJPfjU,23972
-DIRAC/TransformationSystem/Client/TransformationClient.py,sha256=6NtwGALJIe7XCrk1XC1HtGCN0IcqeEAL6fiMv1S2rt8,26405
+DIRAC/TransformationSystem/Client/TransformationClient.py,sha256=9fTV7qvhyaiWCeR-faTpjPy4Vqw4wPuDtr7oFUECJmg,26667
 DIRAC/TransformationSystem/Client/TransformationFilesStatus.py,sha256=3O89wYkPdvYPXnwxyM9x7QLcN6uODEV5yH76LFpmQ80,822
 DIRAC/TransformationSystem/Client/TransformationStatus.py,sha256=f7lzca1m6AfGG09OkA9_bQUthUyojeurS_p02r2oXyQ,1411
 DIRAC/TransformationSystem/Client/Utilities.py,sha256=OO2XBMYbFCpEs961k676WXDtn-tPUm7lQSxilUQhdiA,20369
@@ -1083,7 +1083,7 @@ DIRAC/TransformationSystem/Client/test/Test_Client_TaskManagerPlugin.py,sha256=0
 DIRAC/TransformationSystem/Client/test/Test_Client_TransformationClient.py,sha256=e0O2BZ7xA7uMQDf-68MXxQBRvICiL4KTCcSpSZtpHZc,2577
 DIRAC/TransformationSystem/Client/test/Test_Client_TransformationSystem.py,sha256=tADYLmibPoagAbhM_MToRjy8HHDBcR_6tHU7c3hquJY,13663
 DIRAC/TransformationSystem/Client/test/Test_Client_WorkflowTasks.py,sha256=0BNNF17NduQIiQIbamuQuilY7bklS7trtKzGfMjc_Tk,6128
-DIRAC/TransformationSystem/DB/TransformationDB.py,sha256=QAvO_ZPAV_UySqk7hGmgUGu-faLEm4VjMf_McXazUo0,76520
+DIRAC/TransformationSystem/DB/TransformationDB.py,sha256=i3cVrpi19nUTD0wZ6JT_S7AUIsUQ3SuutEUYFnhM1Fg,79576
 DIRAC/TransformationSystem/DB/TransformationDB.sql,sha256=JfOfEj0s8q-qh6Nyg6_HrrWsvj3MVFd8zY9O2hf9YEg,6935
 DIRAC/TransformationSystem/DB/__init__.py,sha256=EZ7harwr41I_3b8a_D7j4tHGFrHQYUWYfZqfD6tAEb4,46
 DIRAC/TransformationSystem/Service/TornadoTransformationManagerHandler.py,sha256=yzNjl3kzFJ20zwI-QYtD0e-rila1Nu45IRp_BVVgols,342
@@ -1209,9 +1209,9 @@ DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py,sha256=s_YHg_PsTGVHBLrS1
 DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperOfflineTemplate.py,sha256=wem5VDN9XiC7szAzdsbgHUxpIOQB2Hj36DIVMoV9px8,2490
 DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperTemplate.py,sha256=4QgcFPMLRaTagP9e_Vvsla8pFH8HdewklHfS-gyS4-g,3313
 DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperUtilities.py,sha256=5w_4PMnaHhuexChADDvt1L9Ih1PstdUuYWObnlv9Dto,10072
-DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py,sha256=wGpIdnyVzI5T9agxNp94gmPZPceXWREaJiEtZg1lAzk,39997
+DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py,sha256=UWVDC_tWzlE9i5PsLycrRda4j3hWjaK1RnQVOvydg6U,38857
 DIRAC/WorkloadManagementSystem/JobWrapper/__init__.py,sha256=e9Oa_ddNLweR3Lp_HOMK6WqqCWWj2SLPxF5UH4F19ic,61
-DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py,sha256=R9onrrnfyc1v4USt2nnvrYHFo5tKbpLjhMkXl4n0y2Y,39177
+DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py,sha256=AhevZTkZfFQV9XN5UTWFmoMe9GvwN_Caorto1o_YoU8,39119
 DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py,sha256=dC_SvC5Rlchlj2NvBfN7FH1ioXgC8bf9U8BQnEL5GYg,21982
 DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_Watchdog.py,sha256=a-QJ1E1ZcWObhOVgxZYD_nYjseCWsbjT0KxjZDNWyAQ,882
 DIRAC/WorkloadManagementSystem/JobWrapper/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -1242,7 +1242,7 @@ DIRAC/WorkloadManagementSystem/Utilities/JobModel.py,sha256=jN9sFbzMZo9tab6Kp7Oe
 DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py,sha256=JW3AAEtBJn1gIO_rm2Ft5qqjfLteIo3HpQtGNZBfhxE,8365
 DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py,sha256=WtGJzC7fHvydANh8JH6e1Kk_jebrCMPr2c5cw3ufjm8,7826
 DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py,sha256=FNUsGhvsFVrtmA7r8G-sd4QTMeBkqG1sdtwiBUKQyd0,605
-DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py,sha256=Ezpfd90dV_j6fOn25v5gr-wNWh1nMpCOQZfPmGDHJD4,12448
+DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py,sha256=ZQk2tD40986awO9pae1zmdPEWlnMJt4m61Z_RU3LWl8,12476
 DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py,sha256=VcvQTpeyTbVYqSsPQDyAt37N2CaEAnIuvbR6yk4kYk8,15465
 DIRAC/WorkloadManagementSystem/Utilities/QueueUtilities.py,sha256=J5-n_lvWbW_TRjrlqp8hx1SHEaXDW2Dxp3R1hBBrWnE,12082
 DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py,sha256=7FcEtlYSJMzdbLIFBUKD-j_wqRHya-ISqk8w-JRy3kw,12159
@@ -1296,9 +1296,9 @@ DIRAC/tests/Workflow/Integration/exe-script.py,sha256=B_slYdTocEzqfQLRhwuPiLyYUn
 DIRAC/tests/Workflow/Integration/helloWorld.py,sha256=tBgEHH3ZF7ZiTS57gtmm3DW-Qxgm_57HWHpM-Y8XSws,205
 DIRAC/tests/Workflow/Regression/helloWorld.py,sha256=69eCgFuVSYo-mK3Dj2dw1c6g86sF5FksKCf8V2aGVoM,509
 DIRAC/tests/Workflow/Regression/helloWorld.xml,sha256=xwydIcFTAHIX-YPfQfyxuQ7hzvIO3IhR3UAF7ORgkGg,5310
-dirac-9.0.8.dist-info/licenses/LICENSE,sha256=uyr4oV6jmjUeepXZPPjkJRwa5q5MrI7jqJz5sVXNblQ,32452
-dirac-9.0.8.dist-info/METADATA,sha256=gpKRR8YAjDBxrxogcUUGPNXojPku4FhNUVqBVSPRNWg,10016
-dirac-9.0.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dirac-9.0.8.dist-info/entry_points.txt,sha256=hupzIL8aVmjK3nn7RLKdhcaiPmLOiD3Kulh3CSDHKmw,16492
-dirac-9.0.8.dist-info/top_level.txt,sha256=RISrnN9kb_mPqmVu8_o4jF-DSX8-h6AcgfkO9cgfkHA,6
-dirac-9.0.8.dist-info/RECORD,,
+dirac-9.0.10.dist-info/licenses/LICENSE,sha256=uyr4oV6jmjUeepXZPPjkJRwa5q5MrI7jqJz5sVXNblQ,32452
+dirac-9.0.10.dist-info/METADATA,sha256=2Pde9NbM-pbmGOY4PXnSp-C9eZYR-YnqbJ8ZmRqnxzQ,10018
+dirac-9.0.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dirac-9.0.10.dist-info/entry_points.txt,sha256=hupzIL8aVmjK3nn7RLKdhcaiPmLOiD3Kulh3CSDHKmw,16492
+dirac-9.0.10.dist-info/top_level.txt,sha256=RISrnN9kb_mPqmVu8_o4jF-DSX8-h6AcgfkO9cgfkHA,6
+dirac-9.0.10.dist-info/RECORD,,