DIRAC 9.0.14__py3-none-any.whl → 9.0.16__py3-none-any.whl

DIRAC/Resources/Computing/test/Test_PoolComputingElement.py

@@ -2,6 +2,7 @@
 """
 tests for PoolComputingElement module
 """
+
 import os
 import time
 
@@ -50,7 +51,7 @@ def _stopJob(nJob):
 
 @pytest.fixture
 def createAndDelete():
-    for i in range(6):
+    for i in range(9):
         with open(f"testPoolCEJob_{i}.py", "w") as execFile:
             execFile.write(jobScript % i)
         os.chmod(f"testPoolCEJob_{i}.py", 0o755)
@@ -66,24 +67,28 @@ def createAndDelete():
     time.sleep(0.5)
 
     # stopping the jobs
-    for i in range(6):
+    for i in range(9):
         _stopJob(i)
 
     # removing testPoolCEJob files
     # this will also stop the futures unless they are already stopped!
-    for i in range(6):
+    for i in range(9):
         try:
             os.remove(f"testPoolCEJob_{i}.py")
-            os.remove("testBadPoolCEJob.py")
         except OSError:
             pass
 
+    try:
+        os.remove("testBadPoolCEJob.py")
+    except OSError:
+        pass
+
 
 @pytest.mark.slow
 def test_submit_and_shutdown(createAndDelete):
     time.sleep(0.5)
 
-    ceParameters = {"WholeNode": True, "NumberOfProcessors": 4}
+    ceParameters = {"WholeNode": True, "NumberOfProcessors": 4, "MaxRAM": 3800}
     ce = PoolComputingElement("TestPoolCE")
     ce.setParameters(ceParameters)
 
@@ -145,7 +150,7 @@ def test_executeJob_wholeNode4(createAndDelete):
     time.sleep(0.5)
     taskIDs = {}
 
-    ceParameters = {"WholeNode": True, "NumberOfProcessors": 4}
+    ceParameters = {"WholeNode": True, "NumberOfProcessors": 4, "MaxRAM": 16000}
     ce = PoolComputingElement("TestPoolCE")
     ce.setParameters(ceParameters)
 
@@ -159,9 +164,11 @@ def test_executeJob_wholeNode4(createAndDelete):
     result = ce.getCEStatus()
     assert result["UsedProcessors"] == 1
     assert result["AvailableProcessors"] == 3
+    assert result["UsedRAM"] == 0
+    assert result["AvailableRAM"] == 16000
     assert result["RunningJobs"] == 1
 
-    jobParams = {"mpTag": True, "numberOfProcessors": 2}
+    jobParams = {"mpTag": True, "numberOfProcessors": 2, "MaxRAM": 4000}
     result = ce.submitJob("testPoolCEJob_1.py", None, **jobParams)
     assert result["OK"] is True
     taskID = result["Value"]
@@ -171,6 +178,9 @@ def test_executeJob_wholeNode4(createAndDelete):
     result = ce.getCEStatus()
     assert result["UsedProcessors"] == 3
     assert result["AvailableProcessors"] == 1
+    assert result["UsedRAM"] == 4000
+    assert result["AvailableRAM"] == 12000
+
     assert result["RunningJobs"] == 2
 
     # now trying again would fail
@@ -190,13 +200,20 @@ def test_executeJob_wholeNode4(createAndDelete):
 
 
 @pytest.mark.slow
-def test_executeJob_wholeNode8(createAndDelete):
+@pytest.mark.parametrize(
+    "ce_parameters",
+    [
+        ({"NumberOfProcessors": 8}),
+        ({"NumberOfProcessors": 8, "MaxRAM": 32000}),
+        ({"WholeNode": True, "NumberOfProcessors": 8, "MaxRAM": 32000}),
+    ],
+)
+def test_executeJob_wholeNode8(createAndDelete, ce_parameters):
     time.sleep(0.5)
     taskIDs = {}
 
-    ceParameters = {"WholeNode": True, "NumberOfProcessors": 8}
     ce = PoolComputingElement("TestPoolCE")
-    ce.setParameters(ceParameters)
+    ce.setParameters(ce_parameters)
 
     jobParams = {"mpTag": True, "numberOfProcessors": 2, "maxNumberOfProcessors": 2}
     result = ce.submitJob("testPoolCEJob_2.py", None, **jobParams)
@@ -207,6 +224,8 @@ def test_executeJob_wholeNode8(createAndDelete):
 
     result = ce.getCEStatus()
     assert result["UsedProcessors"] == 2
+    assert result["UsedRAM"] == 0
+    assert result["AvailableRAM"] == ce_parameters.get("MaxRAM", 0)
 
     jobParams = {"mpTag": True, "numberOfProcessors": 1, "maxNumberOfProcessors": 3}
     result = ce.submitJob("testPoolCEJob_3.py", None, **jobParams)
@@ -217,8 +236,10 @@ def test_executeJob_wholeNode8(createAndDelete):
 
     result = ce.getCEStatus()
     assert result["UsedProcessors"] == 5
+    assert result["UsedRAM"] == 0
+    assert result["AvailableRAM"] == ce_parameters.get("MaxRAM", 0)
 
-    jobParams = {"numberOfProcessors": 2}  # This is same as asking for SP
+    jobParams = {"numberOfProcessors": 2, "MinRAM": 4000, "MaxRAM": 8000}  # This is same as asking for SP
     result = ce.submitJob("testPoolCEJob_4.py", None, **jobParams)
     assert result["OK"] is True
     taskID = result["Value"]
@@ -227,39 +248,72 @@ def test_executeJob_wholeNode8(createAndDelete):
 
     result = ce.getCEStatus()
     assert result["UsedProcessors"] == 6
+    assert result["UsedRAM"] == 8000
+    assert result["AvailableRAM"] == (
+        ce_parameters.get("MaxRAM") - result["UsedRAM"] if ce_parameters.get("MaxRAM") else 0
+    )
 
-    # now trying again would fail
-    jobParams = {"mpTag": True, "numberOfProcessors": 3}
+    jobParams = {"MinRAM": 8000, "MaxRAM": 8000}  # This is same as asking for SP
     result = ce.submitJob("testPoolCEJob_5.py", None, **jobParams)
     assert result["OK"] is True
     taskID = result["Value"]
     assert taskID == 3
+    taskIDs[taskID] = True
+
+    result = ce.getCEStatus()
+    assert result["UsedProcessors"] == 7
+    assert result["UsedRAM"] == 16000
+    assert result["AvailableRAM"] == (
+        ce_parameters.get("MaxRAM") - result["UsedRAM"] if ce_parameters.get("MaxRAM") else 0
+    )
+
+    jobParams = {"MaxRAM": 24000}  # This will fail for the case when the ce have set a RAM
+    result = ce.submitJob("testPoolCEJob_6.py", None, **jobParams)
+    assert result["OK"] is True
+    taskID = result["Value"]
+    assert taskID == 4
+    if ce_parameters.get("MaxRAM"):
+        assert ce.taskResults[taskID]["OK"] is False
+
+    result = ce.getCEStatus()
+    assert result["UsedProcessors"] == 7 if ce_parameters.get("MaxRAM") else 8
+    assert result["UsedRAM"] == 16000 if ce_parameters.get("MaxRAM") else 40000
+    assert result["AvailableRAM"] == (
+        ce_parameters.get("MaxRAM") - result["UsedRAM"] if ce_parameters.get("MaxRAM") else 0
+    )
+
+    # now trying again would fail
+    jobParams = {"mpTag": True, "numberOfProcessors": 3}
+    result = ce.submitJob("testPoolCEJob_7.py", None, **jobParams)
+    assert result["OK"] is True
+    taskID = result["Value"]
+    assert taskID == 5
     taskIDs[taskID] = False
 
     # waiting and submit again
     while len(ce.taskResults) < 2:
         time.sleep(0.1)
 
-    jobParams = {"mpTag": True, "numberOfProcessors": 3}
-    result = ce.submitJob("testPoolCEJob_5.py", None, **jobParams)
+    jobParams = {"mpTag": True, "numberOfProcessors": 1}
+    result = ce.submitJob("testPoolCEJob_8.py", None, **jobParams)
     assert result["OK"] is True
     taskID = result["Value"]
-    assert taskID == 4
+    assert taskID == 6
     taskIDs[taskID] = True
 
     result = ce.shutdown()
     assert result["OK"] is True
     assert isinstance(result["Value"], dict)
-    assert len(result["Value"]) == 5
+    assert len(result["Value"]) == 7
 
-    while len(ce.taskResults) < 5:
+    while len(ce.taskResults) < 7:
         time.sleep(0.1)
 
     for taskID, expectedResult in taskIDs.items():
         submissionResult = ce.taskResults[taskID]
         assert submissionResult["OK"] is expectedResult
         if not submissionResult["OK"]:
-            assert "Not enough processors" in submissionResult["Message"]
+            assert submissionResult["Message"] in ["Not enough processors for the job", "Not enough memory for the job"]
 
 
 @pytest.mark.slow
@@ -372,28 +426,41 @@ def test_executeJob_WholeNodeJobs(createAndDelete):
 
 
 @pytest.mark.parametrize(
-    "processorsPerTask, kwargs, expected",
+    "processorsPerTask, ramPerTask, kwargs, expected_processors, expected_memory",
     [
-        (None, {}, 1),
-        (None, {"mpTag": False}, 1),
-        (None, {"mpTag": True}, 1),
-        (None, {"mpTag": True, "wholeNode": True}, 16),
-        (None, {"mpTag": True, "wholeNode": False}, 1),
-        (None, {"mpTag": True, "numberOfProcessors": 4}, 4),
-        (None, {"mpTag": True, "numberOfProcessors": 4, "maxNumberOfProcessors": 8}, 8),
-        (None, {"mpTag": True, "numberOfProcessors": 4, "maxNumberOfProcessors": 32}, 16),
-        ({1: 4}, {"mpTag": True, "wholeNode": True}, 0),
-        ({1: 4}, {"mpTag": True, "wholeNode": False}, 1),
-        ({1: 4}, {"mpTag": True, "numberOfProcessors": 2}, 2),
-        ({1: 4}, {"mpTag": True, "maxNumberOfProcessors": 2}, 2),
-        ({1: 4}, {"mpTag": True, "maxNumberOfProcessors": 16}, 12),
+        (None, None, {}, 1, 0),
+        (None, None, {"mpTag": False}, 1, 0),
+        (None, None, {"mpTag": True, "MaxRAM": 8000}, 1, 8000),
+        (None, None, {"mpTag": True, "wholeNode": True}, 16, 0),
+        (None, None, {"mpTag": True, "wholeNode": False}, 1, 0),
+        (None, None, {"mpTag": True, "numberOfProcessors": 4, "MinRAM": 2000}, 4, 2000),
+        (None, None, {"mpTag": True, "numberOfProcessors": 4, "MaxRAM": 4000}, 4, 4000),
+        (None, None, {"mpTag": True, "numberOfProcessors": 4, "MaxRAM": 36000}, 4, None),
+        (None, None, {"mpTag": True, "numberOfProcessors": 4, "MinRAM": 2000, "MaxRAM": 4000}, 4, 4000),
+        (None, None, {"mpTag": True, "numberOfProcessors": 4, "maxNumberOfProcessors": 8}, 8, 0),
+        (None, None, {"mpTag": True, "numberOfProcessors": 4, "maxNumberOfProcessors": 32}, 16, 0),
+        ({1: 4}, {1: 4000}, {"mpTag": True, "wholeNode": True}, 0, 0),
+        ({1: 4}, {1: 4000}, {"mpTag": True, "wholeNode": False}, 1, 0),
+        ({1: 4}, {1: 4000}, {"mpTag": True, "numberOfProcessors": 2, "MinRAM": 8000}, 2, 8000),
+        ({1: 4}, {1: 4000}, {"mpTag": True, "numberOfProcessors": 16, "MinRAM": 8000, "MaxRAM": 12000}, 0, 12000),
+        ({1: 4}, {1: 4000}, {"mpTag": True, "maxNumberOfProcessors": 2, "MaxRAM": 16000}, 2, 16000),
+        ({1: 4}, {1: 4000}, {"mpTag": True, "numberOfProcessors": 2, "MaxRAM": 8000}, 2, 8000),
+        ({1: 4}, {1: 4000}, {"mpTag": True, "maxNumberOfProcessors": 16, "MaxRAM": 32000}, 12, None),
+        ({1: 4, 2: 8}, {1: 4000}, {"mpTag": True, "numberOfProcessors": 2}, 2, 0),
+        ({1: 4, 2: 8}, {1: 4000}, {"mpTag": True, "numberOfProcessors": 4}, 4, 0),
+        ({1: 4, 2: 8, 3: 8}, {1: 4000}, {"mpTag": True, "numberOfProcessors": 4}, 0, 0),
     ],
 )
-def test__getProcessorsForJobs(processorsPerTask, kwargs, expected):
+def test__getLimitsForJobs(processorsPerTask, ramPerTask, kwargs, expected_processors, expected_memory):
     ce = PoolComputingElement("TestPoolCE")
     ce.processors = 16
+    ce.ram = 32000
 
     if processorsPerTask:
         ce.processorsPerTask = processorsPerTask
+    if ramPerTask:
+        ce.ramPerTask = ramPerTask
     res = ce._getProcessorsForJobs(kwargs)
-    assert res == expected
+    assert res == expected_processors
+    res = ce._getMemoryForJobs(kwargs)
+    assert res == expected_memory

DIRAC/Resources/Storage/GFAL2_StorageBase.py

@@ -53,6 +53,9 @@ except AttributeError:
 MAX_SINGLE_STREAM_SIZE = 1024 * 1024 * 10  # 10MB
 MIN_BANDWIDTH = 0.5 * (1024 * 1024)  # 0.5 MB/s
 
+# Default timeout for any stat like call
+DEFAULT_OPERATION_TIMEOUT = 10
+
 
 @contextmanager
 def setGfalSetting(
@@ -169,6 +172,12 @@ class GFAL2_StorageBase(StorageBase):
         # It is only useful for TPC
         self.ctx.set_opt_boolean("HTTP PLUGIN", "RETRIEVE_BEARER_TOKEN", False)
 
+        # Set a global timeout for the operations
+        self.ctx.set_opt_integer("CORE", "NAMESPACE_TIMEOUT", DEFAULT_OPERATION_TIMEOUT)
+        # Because HTTP Plugin does not read the CORE:NAMESPACE_TIMEOUT as it should
+        # I also specify it here
+        self.ctx.set_opt_integer("HTTP PLUGIN", "OPERATION_TIMEOUT", DEFAULT_OPERATION_TIMEOUT)
+
         # spaceToken used for copying from and to the storage element
         self.spaceToken = parameters.get("SpaceToken", "")
         # stageTimeout, default timeout to try and stage/pin a file

DIRAC/TransformationSystem/Agent/TransformationAgent.py

@@ -500,19 +500,18 @@ class TransformationAgent(AgentModule, TransformationAgentsUtilities):
             startTime = time.time()
             self._logInfo(f"Getting replicas for {len(newLFNs)} files from catalog", method=method, transID=transID)
             newReplicas = {}
-            for chunk in breakListIntoChunks(newLFNs, 10000):
-                res = self._getDataReplicasDM(transID, chunk, clients, forJobs=forJobs)
-                if res["OK"]:
-                    reps = {lfn: ses for lfn, ses in res["Value"].items() if ses}
-                    newReplicas.update(reps)
-                    self.__updateCache(transID, reps)
-                else:
-                    self._logWarn(
-                        f"Failed to get replicas for {len(chunk)} files",
-                        res["Message"],
-                        method=method,
-                        transID=transID,
-                    )
+            res = self._getDataReplicasDM(transID, newLFNs, clients, forJobs=forJobs)
+            if res["OK"]:
+                newReplicas = {lfn: ses for lfn, ses in res["Value"].items() if ses}
+
+                self.__updateCache(transID, newReplicas)
+            else:
+                self._logWarn(
+                    f"Failed to get replicas for {len(newLFNs)} files",
+                    res["Message"],
+                    method=method,
+                    transID=transID,
+                )
 
             self._logInfo(
                 f"Obtained {len(newReplicas)} replicas from catalog in {time.time() - startTime:.1f} seconds",

DIRAC/WorkloadManagementSystem/Client/JobReport.py

@@ -115,16 +115,20 @@ class JobReport:
 
     def commit(self):
         """Send all the accumulated information"""
+        messages = []
 
-        success = True
         result = self.sendStoredStatusInfo()
-        success &= result["OK"]
+        if not result["OK"]:
+            messages.append(result["Message"])
         result = self.sendStoredJobParameters()
-        success &= result["OK"]
+        if not result["OK"]:
+            messages.append(result["Message"])
 
-        if success:
-            return S_OK()
-        return S_ERROR("Information upload to JobStateUpdate service failed")
+        if messages:
+            gLogger.warn("Some information could not be uploaded to JobStateUpdate service:", "; ".join(messages))
+            return S_ERROR("Information upload to JobStateUpdate service failed")
+
+        return S_OK()
 
     def dump(self):
         """Print out the contents of the internal cached information"""

DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py

@@ -1,11 +1,16 @@
-""" This object is a wrapper for setting and getting jobs states
-"""
+"""This object is a wrapper for setting and getting jobs states"""
+
 from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
 from DIRAC.WorkloadManagementSystem.Client.JobState.JobManifest import JobManifest
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
-from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB, multiValueDefFields, singleValueDefFields
+from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import (
+    TaskQueueDB,
+    multiValueDefFields,
+    singleValueDefFields,
+    rangeValueDefFields,
+)
 from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
     RIGHT_CHANGE_STATUS,
     RIGHT_GET_INFO,
@@ -351,6 +356,10 @@ class JobState:
             if name in reqCfg:
                 jobReqDict[name] = reqCfg.getOption(name, [])
 
+        for name in rangeValueDefFields:
+            if name in reqCfg:
+                jobReqDict[name] = int(reqCfg[name])
+
         jobPriority = reqCfg.getOption("UserPriority", 1)
 
         result = self.__retryFunction(2, JobState.__db.tqDB.insertJob, (self.__jid, jobReqDict, jobPriority))

DIRAC/WorkloadManagementSystem/Client/Matcher.py

@@ -1,7 +1,8 @@
-""" Encapsulate here the logic for matching jobs
+"""Encapsulate here the logic for matching jobs
 
-    Utilities and classes here are used by MatcherHandler
+Utilities and classes here are used by MatcherHandler
 """
+
 import time
 
 from DIRAC import convertToPy3VersionNumber, gLogger
@@ -16,7 +17,11 @@ from DIRAC.WorkloadManagementSystem.Client.Limiter import Limiter
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
 from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
-from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB, multiValueMatchFields, singleValueDefFields
+from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import (
+    TaskQueueDB,
+    multiValueMatchFields,
+    singleValueDefFields,
+)
 
 
 class PilotVersionError(Exception):
@@ -68,14 +73,14 @@ class Matcher:
 
             # Make a nice print of the resource matching parameters
             toPrintDict = dict(resourceDict)
-            if "MaxRAM" in resourceDescription:
-                toPrintDict["MaxRAM"] = resourceDescription["MaxRAM"]
+            if "MaxRAM" in resourceDict:
+                toPrintDict["MaxRAM"] = resourceDict["MaxRAM"]
             if "NumberOfProcessors" in resourceDescription:
                 toPrintDict["NumberOfProcessors"] = resourceDescription["NumberOfProcessors"]
             toPrintDict["Tag"] = []
             if "Tag" in resourceDict:
                 for tag in resourceDict["Tag"]:
-                    if not tag.endswith("GB") and not tag.endswith("Processors"):
+                    if not tag.endswith("MB") and not tag.endswith("Processors"):
                         toPrintDict["Tag"].append(tag)
             if not toPrintDict["Tag"]:
                 toPrintDict.pop("Tag")
@@ -166,11 +171,7 @@ class Matcher:
         """
 
         resourceDict = {}
-        for name in singleValueDefFields:
-            if name in resourceDescription:
-                resourceDict[name] = resourceDescription[name]
-
-        for name in multiValueMatchFields:
+        for name in singleValueDefFields + multiValueMatchFields + ("MaxRAM",):
             if name in resourceDescription:
                 resourceDict[name] = resourceDescription[name]
 
@@ -191,25 +192,18 @@ class Matcher:
         if "JobID" in resourceDescription:
             resourceDict["JobID"] = resourceDescription["JobID"]
 
-        # Convert MaxRAM and NumberOfProcessors parameters into a list of tags
-        maxRAM = resourceDescription.get("MaxRAM")
-        if maxRAM:
-            try:
-                maxRAM = int(maxRAM / 1000)
-            except ValueError:
-                maxRAM = None
+        # Convert NumberOfProcessors parameters into a list of tags
         nProcessors = resourceDescription.get("NumberOfProcessors")
         if nProcessors:
             try:
                 nProcessors = int(nProcessors)
             except ValueError:
                 nProcessors = None
-        for param, key in [(maxRAM, "GB"), (nProcessors, "Processors")]:
-            if param and param <= 1024:
-                paramList = list(range(2, param + 1))
-                paramTags = ["%d%s" % (par, key) for par in paramList]
-                if paramTags:
-                    resourceDict.setdefault("Tag", []).extend(paramTags)
+        if nProcessors and nProcessors <= 1024:
+            paramList = list(range(1, nProcessors + 1, 1))
+            paramTags = ["%d%s" % (par, "Processors") for par in paramList]
+            if paramTags:
+                resourceDict.setdefault("Tag", []).extend(paramTags)
 
         # Add 'MultiProcessor' to the list of tags
         if nProcessors and nProcessors > 1: