DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/Resources/Computing/SSHBatchComputingElement.py

@@ -7,7 +7,6 @@ import stat
 from urllib.parse import urlparse
 
 from DIRAC import S_ERROR, S_OK
-from DIRAC.Resources.Computing.PilotBundle import bundleProxy, writeScript
 from DIRAC.Resources.Computing.SSHComputingElement import SSHComputingElement
 from DIRAC.WorkloadManagementSystem.Client import PilotStatus
 
@@ -80,18 +79,6 @@ class SSHBatchComputingElement(SSHComputingElement):
         if not os.access(executableFile, 5):
             os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
 
-        # if no proxy is supplied, the executable can be submitted directly
-        # otherwise a wrapper script is needed to get the proxy to the execution node
-        # The wrapper script makes debugging more complicated and thus it is
-        # recommended to transfer a proxy inside the executable if possible.
-        if proxy:
-            self.log.verbose("Setting up proxy for payload")
-            wrapperContent = bundleProxy(executableFile, proxy)
-            name = writeScript(wrapperContent, os.getcwd())
-            submitFile = name
-        else:  # no proxy
-            submitFile = executableFile
-
         # Submit jobs now
         restJobs = numberOfJobs
         submittedJobs = []
@@ -100,7 +87,7 @@ class SSHBatchComputingElement(SSHComputingElement):
             if slots not in rankHosts:
                 continue
             for host in rankHosts[slots]:
-                result = self._submitJobToHost(submitFile, min(slots, restJobs), host)
+                result = self._submitJobToHost(executableFile, min(slots, restJobs), host)
                 if not result["OK"]:
                     continue
 
@@ -114,9 +101,6 @@ class SSHBatchComputingElement(SSHComputingElement):
             if restJobs <= 0:
                 break
 
-        if proxy:
-            os.remove(submitFile)
-
         result = S_OK(submittedJobs)
         result["PilotStampDict"] = stampDict
         return result

DIRAC/Resources/Computing/SSHComputingElement.py

@@ -78,7 +78,6 @@ from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.Core.Utilities.List import breakListIntoChunks, uniqueElements
 from DIRAC.Resources.Computing.BatchSystems.executeBatch import executeBatchContent
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
-from DIRAC.Resources.Computing.PilotBundle import bundleProxy, writeScript
 
 
 class SSH:
@@ -532,23 +531,7 @@ class SSHComputingElement(ComputingElement):
         if not os.access(executableFile, 5):
             os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
 
-        # if no proxy is supplied, the executable can be submitted directly
-        # otherwise a wrapper script is needed to get the proxy to the execution node
-        # The wrapper script makes debugging more complicated and thus it is
-        # recommended to transfer a proxy inside the executable if possible.
-        if proxy:
-            self.log.verbose("Setting up proxy for payload")
-            wrapperContent = bundleProxy(executableFile, proxy)
-            name = writeScript(wrapperContent, os.getcwd())
-            submitFile = name
-        else:  # no proxy
-            submitFile = executableFile
-
-        result = self._submitJobToHost(submitFile, numberOfJobs)
-        if proxy:
-            os.remove(submitFile)
-
-        return result
+        return self._submitJobToHost(executableFile, numberOfJobs)
 
     def _submitJobToHost(self, executableFile, numberOfJobs, host=None):
         """Submit prepared executable to the given host"""

DIRAC/Resources/Computing/SingularityComputingElement.py

@@ -22,7 +22,7 @@ from pathlib import Path
 import DIRAC
 from DIRAC import S_ERROR, S_OK, gConfig, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers import Operations
-from DIRAC.Core.Utilities.Subprocess import systemCall
+from DIRAC.Core.Utilities.CGroups2 import CG2Manager
 from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
 from DIRAC.Resources.Storage.StorageElement import StorageElement
@@ -381,8 +381,18 @@ class SingularityComputingElement(ComputingElement):
         for seName in localSEs:
             try:
                 # Find the base path if a File protocol is defined
-                mountedPath = StorageElement(seName).getStorageParameters(protocol="file")["Value"]["Path"]
-                bindPaths.append(f"{mountedPath}:{mountedPath}:ro")
+                se = StorageElement(seName)
+                if not se.valid:
+                    self.log.warn(f"Storage Element {seName} not valid")
+                    continue
+
+                mountedPath = se.getStorageParameters(protocol="file")["Value"]["Path"]
+                mountOptions = se.getStorageParameters(protocol="file")["Value"].get("MountOptions", "ro")
+                if mountOptions not in ("rw", "ro"):
+                    self.log.warn(f"Unknown mount mode: {mountOptions}")
+                    continue
+
+                bindPaths.append(f"{mountedPath}:{mountedPath}:{mountOptions}")
             except KeyError:
                 pass
 
@@ -405,7 +415,9 @@ class SingularityComputingElement(ComputingElement):
 
         self.log.debug(f"Execute singularity command: {cmd}")
         self.log.debug(f"Execute singularity env: {self.__getEnv()}")
-        result = systemCall(0, cmd, callbackFunction=self.sendOutput, env=self.__getEnv())
+        result = CG2Manager().systemCall(
+            0, cmd, callbackFunction=self.sendOutput, env=self.__getEnv(), ceParameters=self.ceParameters
+        )
 
         self.__runningJobs -= 1
 
@@ -417,7 +429,9 @@ class SingularityComputingElement(ComputingElement):
             self.log.error(f"Singularity env was: {self.__getEnv()}")
             debugCmd = [outerCmd[0], "--debug"] + outerCmd[1:] + ["echo", "All okay"]
             self.log.error("Running with debug output to facilitate debugging", debugCmd)
-            result = systemCall(0, debugCmd, callbackFunction=self.sendOutput, env=self.__getEnv())
+            result = CG2Manager().systemCall(
+                0, debugCmd, callbackFunction=self.sendOutput, env=self.__getEnv(), ceParameters=self.ceParameters
+            )
             if proxy and renewTask:
                 gThreadScheduler.removeTask(renewTask)
             self.__deleteWorkArea(baseDir)

DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py

@@ -2,6 +2,7 @@
 """
 tests for HTCondorCEComputingElement module
 """
+import json
 import uuid
 
 import pytest
@@ -12,18 +13,30 @@ from DIRAC.Resources.Computing.BatchSystems import Condor
 
 MODNAME = "DIRAC.Resources.Computing.HTCondorCEComputingElement"
 
-STATUS_LINES = """
-123.2 5 4 0 undefined
-123.1 3 undefined undefined undefined
-""".strip().split(
-    "\n"
-)
-
-HISTORY_LINES = """
-123.0 4 undefined undefined undefined
-""".strip().split(
-    "\n"
-)
+STATUS_QUEUE = [
+    {
+        "ClusterId": 123,
+        "ProcId": 2,
+        "JobStatus": 5,
+        "HoldReasonCode": 4,
+        "HoldReasonSubCode": 0,
+        "HoldReason": "The credentials for the job are invalid",
+    },
+    {
+        "ClusterId": 123,
+        "ProcId": 1,
+        "JobStatus": 3,
+    },
+]
+
+
+STATUS_HISTORY = [
+    {
+        "ClusterId": 123,
+        "ProcId": 0,
+        "JobStatus": 4,
+    }
+]
 
 
 @pytest.fixture
@@ -32,42 +45,47 @@ def setUp():
 
 
 def test_parseCondorStatus():
-    statusLines = f"""
-  104098.1 1 undefined undefined undefined
-  104098.2 2 undefined undefined undefined
-  104098.3 3 undefined undefined undefined
-  104098.4 4 undefined undefined undefined
-  104098.5 5 16 57 Input data are being spooled
-  104098.6 5 3 {Condor.HOLD_REASON_SUBCODE} Policy
-  104098.7 5 1 0 undefined
-
-  foo bar
-  104096.1 3 16 test test
-  104096.2 3 test
-  104096.3 5 undefined undefined undefined
-  104096.4 7
-  """.strip().split(
-        "\n"
-    )
-    # force there to be an empty line
+    statusOutput = {"ClusterId": 104098, "ProcId": 1, "JobStatus": 1}
+    assert HTCE.getCondorStatus(statusOutput) == ("Waiting", "")
 
-    expectedResults = {
-        "104098.1": "Waiting",
-        "104098.2": "Running",
-        "104098.3": "Aborted",
-        "104098.4": "Done",
-        "104098.5": "Waiting",
-        "104098.6": "Failed",
-        "104098.7": "Aborted",
-        "foo": "Unknown",
-        "104096.1": "Aborted",
-        "104096.2": "Aborted",
-        "104096.3": "Aborted",
-        "104096.4": "Unknown",
-    }
+    statusOutput = {"ClusterId": 104098, "ProcId": 2, "JobStatus": 2}
+    assert HTCE.getCondorStatus(statusOutput) == ("Running", "")
+
+    statusOutput = {"ClusterId": 104098, "ProcId": 3, "JobStatus": 3}
+    assert HTCE.getCondorStatus(statusOutput) == ("Aborted", "")
+
+    statusOutput = {"ClusterId": 104098, "ProcId": 4, "JobStatus": 4}
+    assert HTCE.getCondorStatus(statusOutput) == ("Done", "")
 
-    for jobID, expected in expectedResults.items():
-        assert HTCE.parseCondorStatus(statusLines, jobID)[0] == expected
+    statusOutput = {
+        "ClusterId": 104098,
+        "ProcId": 5,
+        "JobStatus": 5,
+        "HoldReasonCode": 16,
+        "HoldReasonSubCode": 57,
+        "HoldReason": "Input data are being spooled",
+    }
+    assert HTCE.getCondorStatus(statusOutput) == ("Waiting", "Input data are being spooled")
+
+    statusOutput = {
+        "ClusterId": 104098,
+        "ProcId": 6,
+        "JobStatus": 5,
+        "HoldReasonCode": 3,
+        "HoldReasonSubCode": HTCE.HOLD_REASON_SUBCODE,
+        "HoldReason": "Policy",
+    }
+    assert HTCE.getCondorStatus(statusOutput) == ("Failed", "Policy")
+
+    statusOutput = {
+        "ClusterId": 104098,
+        "ProcId": 7,
+        "JobStatus": 5,
+        "HoldReasonCode": 1,
+        "HoldReasonSubCode": 0,
+        "HoldReason": "Aborted by user",
+    }
+    assert HTCE.getCondorStatus(statusOutput) == ("Aborted", "Aborted by user")
 
 
 def test_getJobStatus(mocker):
@@ -75,8 +93,8 @@ def test_getJobStatus(mocker):
     mocker.patch(
         MODNAME + ".systemCall",
         side_effect=[
-            S_OK((0, "\n".join(STATUS_LINES), "")),
-            S_OK((0, "\n".join(HISTORY_LINES), "")),
+            S_OK((0, json.dumps(STATUS_QUEUE), "")),
+            S_OK((0, json.dumps(STATUS_HISTORY), "")),
             S_OK((0, "", "")),
             S_OK((0, "", "")),
         ],
@@ -110,7 +128,7 @@ def test_getJobStatus(mocker):
 def test_getJobStatusBatchSystem(mocker):
     """Test Condor Batch System plugin getJobStatus"""
     patchPopen = mocker.patch("DIRAC.Resources.Computing.BatchSystems.Condor.subprocess.Popen")
-    patchPopen.return_value.communicate.side_effect = [("\n".join(STATUS_LINES), ""), ("\n".join(HISTORY_LINES), "")]
+    patchPopen.return_value.communicate.side_effect = [(json.dumps(STATUS_QUEUE), ""), (json.dumps(STATUS_HISTORY), "")]
     patchPopen.return_value.returncode = 0
 
     ret = Condor.Condor().getJobStatus(JobIDList=["123.0", "123.1", "123.2", "333.3"])

DIRAC/Resources/Computing/test/Test_PoolComputingElement.py

@@ -25,7 +25,7 @@ while True:
   if os.path.isfile(stopFile):
     os.remove(stopFile)
     break
-  if (time.time() - start) > 30:
+  if (time.time() - start) > 5:
     break
 print("End job", jobNumber, time.time())
 """
@@ -262,6 +262,7 @@ def test_executeJob_wholeNode8(createAndDelete):
             assert "Not enough processors" in submissionResult["Message"]
 
 
+@pytest.mark.slow
 def test_executeJob_submitAndStop(createAndDelete):
     time.sleep(0.5)
 

DIRAC/Resources/IdProvider/CheckInIdProvider.py

@@ -26,3 +26,16 @@ class CheckInIdProvider(OAuth2IdProvider):
 
             idPScope = f"eduperson_entitlement?value=urn:mace:egi.eu:group:{vo}:role={groupElements[1]}#aai.egi.eu"
         return scope_to_list(idPScope)
+
+    def fetchToken(self, **kwargs):
+        """Fetch token
+
+        :param kwargs:
+        :return: dict
+        """
+
+        if "audience" in kwargs:
+            kwargs["resource"] = kwargs["audience"]
+            kwargs.pop("audience")
+
+        return super().fetchToken(**kwargs)

DIRAC/Resources/IdProvider/IdProviderFactory.py

@@ -1,6 +1,7 @@
-"""  The Identity Provider Factory instantiates IdProvider objects
-     according to their configuration
+"""The Identity Provider Factory instantiates IdProvider objects
+according to their configuration
 """
+
 import jwt
 
 from DIRAC import S_OK, S_ERROR, gLogger, gConfig
@@ -40,11 +41,12 @@ class IdProviderFactory:
             return result
         return self.getIdProvider(result["Value"])
 
-    def getIdProvider(self, name, **kwargs):
+    def getIdProvider(self, name, client_name_prefix="", **kwargs):
         """This method returns a IdProvider instance corresponding to the supplied
         name.
 
         :param str name: the name of the Identity Provider client
+        :param str client_name_prefix: name of the client of the IdP
 
         :return: S_OK(IdProvider)/S_ERROR()
         """
@@ -68,8 +70,14 @@ class IdProviderFactory:
             if not result["OK"]:
                 self.log.error("Failed to read configuration", f"{name}: {result['Message']}")
                 return result
+
             pDict = result["Value"]
 
+            if client_name_prefix:
+                client_name_prefix = client_name_prefix + "_"
+            pDict["client_id"] = pDict[f"{client_name_prefix}client_id"]
+            pDict["client_secret"] = pDict[f"{client_name_prefix}client_secret"]
+
         pDict.update(kwargs)
         pDict["ProviderName"] = name
 

DIRAC/Resources/MessageQueue/StompMQConnector.py

@@ -172,7 +172,7 @@ class StompMQConnector(MQConnector):
         try:
             try:
                 self.connection.send(body=json.dumps(message), destination=destination)
-            except stomp.exception.StompException:
+            except (stomp.exception.StompException, ConnectionError):
                 self.connect()
                 self.connection.send(body=json.dumps(message), destination=destination)
         except Exception as e:

DIRAC/Resources/Storage/GFAL2_StorageBase.py

@@ -142,9 +142,11 @@ class GFAL2_StorageBase(StorageBase):
         # Workaround for https://github.com/xrootd/xrootd/issues/2396
         # xrootd internaly sets nproc, so we save the limit and reset it
         # just after creating the context
-        saved_limits = getrlimit(RLIMIT_NPROC)
+        saved_limit = getrlimit(RLIMIT_NPROC)
         self.ctx = gfal2.creat_context()
-        setrlimit(RLIMIT_NPROC, saved_limits)
+        new_limit = getrlimit(RLIMIT_NPROC)
+        if saved_limit != new_limit:
+            setrlimit(RLIMIT_NPROC, saved_limit)
 
         # by default turn off BDII checks
         self.ctx.set_opt_boolean("BDII", "ENABLE", False)
@@ -163,6 +165,10 @@ class GFAL2_StorageBase(StorageBase):
             os.environ.get("DIRAC_GFAL_GRIDFTP_ENABLE_IPV6", "true").lower() not in ["false", "no"],
         )
 
+        # Disable retrieving the bearer token for every operations.
+        # It is only useful for TPC
+        self.ctx.set_opt_boolean("HTTP PLUGIN", "RETRIEVE_BEARER_TOKEN", False)
+
         # spaceToken used for copying from and to the storage element
         self.spaceToken = parameters.get("SpaceToken", "")
         # stageTimeout, default timeout to try and stage/pin a file
@@ -303,20 +309,23 @@ class GFAL2_StorageBase(StorageBase):
         failed = {}
         successful = {}
 
-        for dest_url, src_file in urls.items():
-            if not src_file:
-                errStr = "GFAL2_StorageBase.putFile: Source file not set. Argument must be a dictionary \
-                                             (or a list of a dictionary) {url : local path}"
-                self.log.debug(errStr)
-                failed[dest_url] = errStr
-                continue
+        # In principle we only need the bearer token when doing TPC, however it's a
+        # bit cumbersome to test, so we always re-enable it when uploading
+        with setGfalSetting(self.ctx, "HTTP PLUGIN", "RETRIEVE_BEARER_TOKEN", True):
+            for dest_url, src_file in urls.items():
+                if not src_file:
+                    errStr = "GFAL2_StorageBase.putFile: Source file not set. Argument must be a dictionary \
+                                                (or a list of a dictionary) {url : local path}"
+                    self.log.debug(errStr)
+                    failed[dest_url] = errStr
+                    continue
 
-            try:
-                successful[dest_url] = self._putSingleFile(src_file, dest_url, sourceSize)
-            except (gfal2.GError, ValueError, RuntimeError) as e:
-                detailMsg = f"Failed to copy {src_file} to {dest_url}: {repr(e)}"
-                self.log.debug("Exception while copying", detailMsg)
-                failed[dest_url] = detailMsg
+                try:
+                    successful[dest_url] = self._putSingleFile(src_file, dest_url, sourceSize)
+                except (gfal2.GError, ValueError, RuntimeError) as e:
+                    detailMsg = f"Failed to copy {src_file} to {dest_url}: {repr(e)}"
+                    self.log.debug("Exception while copying", detailMsg)
+                    failed[dest_url] = detailMsg
 
         return {"Failed": failed, "Successful": successful}
 

DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingHTTPJson.py

@@ -41,9 +41,7 @@ class WLCGAccountingHTTPJson(WLCGAccountingJson):
         """
         try:
             with open(filePath, "w") as fd:
-                caPath = getCAsLocation()
-                userProxy = getProxyLocation()
-                res = requests.get(occupancyLFN, cert=userProxy, verify=caPath, timeout=30)
+                res = requests.get(occupancyLFN, cert=getProxyLocation(), verify=getCAsLocation(), timeout=30)
                 res.raise_for_status()
                 fd.write(res.text)
         except Exception as e:

DIRAC/Resources/Storage/StorageBase.py

@@ -1,4 +1,4 @@
-""" Base Storage Class provides the base interface for all storage plug-ins
+"""Base Storage Class provides the base interface for all storage plug-ins
 
       exists()
 
@@ -33,6 +33,7 @@ These are the methods for getting information about the Storage:
       getOccupancy()
 
 """
+
 import errno
 import json
 import os
@@ -113,7 +114,8 @@ class StorageBase:
 
     def getParameters(self):
         """Get the parameters with which the storage was instantiated"""
-        parameterDict = dict(self.protocolParameters)
+        parameterDict = dict(self._allProtocolParameters)
+        parameterDict.update(self.protocolParameters)
         parameterDict["StorageName"] = self.name
         parameterDict["PluginName"] = self.pluginName
         parameterDict["URLBase"] = self.getURLBase().get("Value", "")

DIRAC/Resources/Storage/StorageElement.py

@@ -65,14 +65,13 @@ class StorageElementCache:
         # Because the gfal2 context caches the proxy location,
         # we also use the proxy location as a key.
         # In practice, there should almost always be one, except for the REA
-        # If we see its memory consumtpion exploding, this might be a place to look
-        proxyLoc = getProxyLocation()
+        # If we see its memory consumption exploding, this might be a place to look
 
         # ensure protocolSections is hashable! (tuple)
         if isinstance(protocolSections, list):
             protocolSections = tuple(protocolSections)
 
-        argTuple = (tId, name, protocolSections, vo, proxyLoc)
+        argTuple = (tId, name, protocolSections, vo, getProxyLocation())
         seObj = self.seCache.get(argTuple)
 
         if not seObj:
@@ -439,10 +438,6 @@ class StorageElementItem:
 
             kwargs["occupancyLFN"] = occupancyLFN
 
-        filteredPlugins = self.__filterPlugins("getOccupancy")
-        if not filteredPlugins:
-            return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins to query the occupancy")
-
         # Call occupancy plugin if requested
         occupancyPlugin = self.options.get("OccupancyPlugin")
         if occupancyPlugin:
@@ -460,6 +455,10 @@ class StorageElementItem:
             except Exception as e:
                 return S_ERROR(f"Occupancy plugin failed: {str(e)}")
 
+        filteredPlugins = self.__filterPlugins("getOccupancy")
+        if not filteredPlugins:
+            return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins to query the occupancy")
+
         # Try all of the storages one by one
         for storage in filteredPlugins:
             # The result of the plugin is always in B

DIRAC/StorageManagementSystem/DB/StorageManagementDB.sql

@@ -23,7 +23,7 @@ CREATE TABLE Tasks(
   `CompleteTime` DATETIME,
   `CallBackMethod` VARCHAR(255),
   `SourceTaskID` VARCHAR(32),
-  PRIMARY KEY(`TaskID`,`Status`),
+  PRIMARY KEY(`TaskID`),
   INDEX(`TaskID`,`Status`)
 )ENGINE=INNODB;
 
@@ -41,7 +41,7 @@ CREATE TABLE CacheReplicas(
   `LastUpdate` DATETIME,
   `Reason` VARCHAR(255),
   `Links` INTEGER DEFAULT 0,
-  PRIMARY KEY (`ReplicaID`,`LFN`,`SE`),
+  PRIMARY KEY (`ReplicaID`),
   INDEX(`ReplicaID`,`Status`,`SE`)
 )ENGINE=INNODB;
 

DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py

@@ -7,23 +7,21 @@
     In case you want to further extend it you are required to follow the note on the
     initialize method and on the _getClients method.
 """
-import time
-import datetime
 import concurrent.futures
+import datetime
+import time
 
-from DIRAC import S_OK
-
+from DIRAC import S_OK, gConfig
+from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
-from DIRAC.Core.Utilities.List import breakListIntoChunks
 from DIRAC.Core.Utilities.Dictionaries import breakDictionaryIntoChunks
-from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
+from DIRAC.Core.Utilities.List import breakListIntoChunks
+from DIRAC.TransformationSystem.Agent.TransformationAgentsUtilities import TransformationAgentsUtilities
 from DIRAC.TransformationSystem.Client.FileReport import FileReport
-from DIRAC.TransformationSystem.Client.WorkflowTasks import WorkflowTasks
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
-from DIRAC.TransformationSystem.Agent.TransformationAgentsUtilities import TransformationAgentsUtilities
+from DIRAC.TransformationSystem.Client.WorkflowTasks import WorkflowTasks
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
-from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient
 
 AGENT_NAME = "Transformation/TaskManagerAgentBase"
 
@@ -40,7 +38,6 @@ class TaskManagerAgentBase(AgentModule, TransformationAgentsUtilities):
         TransformationAgentsUtilities.__init__(self)
 
         self.transClient = None
-        self.jobManagerClient = None
         self.transType = []
 
         self.tasksPerLoop = 50
@@ -69,7 +66,6 @@ class TaskManagerAgentBase(AgentModule, TransformationAgentsUtilities):
 
         # Default clients
         self.transClient = TransformationClient()
-        self.jobManagerClient = JobManagerClient()
 
         # Bulk submission flag
         self.bulkSubmissionFlag = self.am_getOption("BulkSubmission", self.bulkSubmissionFlag)
@@ -193,11 +189,9 @@ class TaskManagerAgentBase(AgentModule, TransformationAgentsUtilities):
             else:
                 # Get the transformations which should be submitted
                 self.tasksPerLoop = self.am_getOption("TasksPerLoop", self.tasksPerLoop)
-                res = self.jobManagerClient.getMaxParametricJobs()
-                if not res["OK"]:
-                    self.log.warn("Could not get the maxParametricJobs from JobManager", res["Message"])
-                else:
-                    self.maxParametricJobs = res["Value"]
+                self.maxParametricJobs = gConfig.getValue(
+                    "/Systems/WorkloadManagement/Services/JobManager/MaxParametricJobs", self.maxParametricJobs
+                )
 
                 self._addOperationForTransformations(
                     self.operationsOnTransformationDict,

DIRAC/TransformationSystem/Agent/TransformationAgent.py

@@ -1,4 +1,4 @@
-"""  TransformationAgent processes transformations found in the transformation database.
+"""TransformationAgent processes transformations found in the transformation database.
 
 The following options can be set for the TransformationAgent.
 
@@ -8,6 +8,7 @@ The following options can be set for the TransformationAgent.
   :dedent: 2
   :caption: TransformationAgent options
 """
+
 from importlib import import_module
 
 import time
@@ -15,6 +16,7 @@ import os
 import datetime
 import pickle
 import concurrent.futures
+from pathlib import Path
 
 from DIRAC import S_OK, S_ERROR
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
@@ -127,6 +129,9 @@ class TransformationAgent(AgentModule, TransformationAgentsUtilities):
         if not res["OK"]:
             self._logError("Failed to obtain transformations:", res["Message"])
             return S_OK()
+
+        active_trans_ids = [t["TransformationID"] for t in res["Value"]]
+        self.cleanOldTransformationCache(active_trans_ids)
         # Process the transformations
         count = 0
         future_to_transID = {}
@@ -164,6 +169,22 @@ class TransformationAgent(AgentModule, TransformationAgentsUtilities):
 
         return S_OK()
 
+    def cleanOldTransformationCache(self, active_trans_ids: list[int]):
+        cache_filenames = {Path(self.__cacheFile(tid)) for tid in active_trans_ids}
+        existing_caches = set(Path(self.workDirectory).glob("*.pkl"))
+        useless_cache_files = existing_caches - cache_filenames
+
+        if useless_cache_files:
+            self._logInfo(f"Found potentially {len(useless_cache_files)} useless cache files")
+
+        # Since idle transformations aren't in active_trans_ids, let's filter it more
+        # and take only files that haven't been touched for 2 month
+        last_update_threshold = (datetime.datetime.utcnow() - datetime.timedelta(days=60)).timestamp()
+
+        for cache_file in useless_cache_files:
+            if Path(cache_file).stat().st_mtime < last_update_threshold:
+                cache_file.unlink()
+
     def getTransformations(self):
         """Obtain the transformations to be executed - this is executed at the start of every loop (it's really the
         only real thing in the execute()