DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Executor/JobSanity.py

@@ -1,6 +1,6 @@
 """ The Job Sanity executor assigns sandboxes to the job """
 from DIRAC import S_OK
-from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient
+from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
 from DIRAC.WorkloadManagementSystem.Executor.Base.OptimizerExecutor import OptimizerExecutor
 
 
@@ -15,7 +15,6 @@ class JobSanity(OptimizerExecutor):
     @classmethod
     def initializeOptimizer(cls):
         """Initialize specific parameters for JobSanityAgent."""
-        cls.sandboxClient = SandboxStoreClient(useCertificates=True, smdb=True)
         return S_OK()
 
     def optimizeJob(self, jid, jobState):
@@ -57,7 +56,8 @@ class JobSanity(OptimizerExecutor):
         if not numSBsToAssign:
             return S_OK(0)
         self.jobLog.info("Assigning sandboxes", f"({numSBsToAssign} on behalf of {ownerName}@{ownerGroup}@{vo})")
-        result = self.sandboxClient.assignSandboxesToJob(jobState.jid, sbsToAssign, ownerName, ownerGroup)
+        eId = f"Job:{jobState.jid}"
+        result = SandboxMetadataDB().assignSandboxesToEntities({eId: sbsToAssign}, ownerName, ownerGroup)
         if not result["OK"]:
             self.jobLog.error("Could not assign sandboxes in the SandboxStore")
             return result

DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py

@@ -77,17 +77,11 @@ class JobStateUpdateClient(FutureClient):
     def setJobAttribute(self, jobID: str | int, attribute: str, value: str):
         with DiracXClient() as api:
             if attribute == "Status":
-                api.jobs.set_job_statuses(
+                return api.jobs.set_job_statuses(
                     {jobID: {datetime.now(tz=timezone.utc): {"Status": value}}},
                 )
             else:
-                api.jobs.patch_metadata({jobID: {attribute: value}})
-
-    @stripValueIfOK
-    @convertToReturnValue
-    def setJobFlag(self, jobID: str | int, flag: str):
-        with DiracXClient() as api:
-            api.jobs.patch_metadata({jobID: {flag: True}})
+                return api.jobs.patch_metadata({jobID: {attribute: value}})
 
     @stripValueIfOK
     @convertToReturnValue
@@ -151,12 +145,6 @@ class JobStateUpdateClient(FutureClient):
             updates = {job_id: {k: v} for job_id, (k, v) in jobsParameterDict.items()}
             api.jobs.patch_metadata(updates)
 
-    @stripValueIfOK
-    @convertToReturnValue
-    def unsetJobFlag(self, jobID: str | int, flag: str):
-        with DiracXClient() as api:
-            api.jobs.patch_metadata({jobID: {flag: False}})
-
     @stripValueIfOK
     @convertToReturnValue
     def updateJobFromStager(self, jobID: str | int, status: str):

DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py

@@ -10,6 +10,7 @@ and a Watchdog Agent that can monitor its progress.
   :caption: JobWrapper options
 
 """
+
 import contextlib
 import datetime
 import glob
@@ -54,6 +55,8 @@ from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateU
 from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient
 from DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog import Watchdog
 
+CHILD_PID_POLL_INTERVALS = list(range(5, 40, 5))
+
 
 class JobWrapper:
     """The only user of the JobWrapper is the JobWrapperTemplate"""
@@ -119,14 +122,16 @@ class JobWrapper:
         self.pilotRef = gConfig.getValue("/LocalSite/PilotReference", "Unknown")
         self.cpuNormalizationFactor = gConfig.getValue("/LocalSite/CPUNormalizationFactor", 0.0)
         self.bufferLimit = gConfig.getValue(self.section + "/BufferLimit", 10485760)
-        self.defaultOutputSE = getDestinationSEList(
-            gConfig.getValue("/Resources/StorageElementGroups/SE-USER", []), self.siteName
-        )
+        try:
+            self.defaultOutputSE = getDestinationSEList("SE-USER", self.siteName)
+        except RuntimeError:
+            self.defaultOutputSE = []
         self.defaultCatalog = gConfig.getValue(self.section + "/DefaultCatalog", [])
         self.masterCatalogOnlyFlag = gConfig.getValue(self.section + "/MasterCatalogOnlyFlag", True)
-        self.defaultFailoverSE = getDestinationSEList(
-            gConfig.getValue("/Resources/StorageElementGroups/Tier1-Failover", []), self.siteName
-        )
+        try:
+            self.defaultFailoverSE = getDestinationSEList("Tier1-Failover", self.siteName)
+        except RuntimeError:
+            self.defaultFailoverSE = []
         self.defaultOutputPath = ""
         self.retryUpload = gConfig.getValue(self.section + "/RetryUpload", False)
         self.dm = DataManager()
@@ -427,14 +432,14 @@ class JobWrapper:
             )
             exeThread.start()
             payloadPID = None
-            for seconds in range(5, 40, 5):
+            for seconds in CHILD_PID_POLL_INTERVALS:
                 time.sleep(seconds)
                 payloadPID = spObject.getChildPID()
                 if payloadPID:
                     self.__setJobParam("PayloadPID", payloadPID)
                     break
             if not payloadPID:
-                return S_ERROR("Payload process could not start after 140 seconds")
+                return S_ERROR(f"Payload process could not start after {sum(CHILD_PID_POLL_INTERVALS)} seconds")
 
             watchdog = Watchdog(
                 pid=self.currentPID,
@@ -1210,8 +1215,8 @@ class JobWrapper:
             lfn = str(basePath / outputPath / os.path.basename(localfile))
         else:
             # if LFN is given, take it as it is
-            localfile = str(self.jobIDPath / outputFile.replace("LFN:", ""))
             lfn = outputFile.replace("LFN:", "")
+            localfile = str(self.jobIDPath / os.path.basename(lfn))
 
         return (lfn, localfile)
 

DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py

@@ -344,24 +344,40 @@ def test_processQuickExecutionNoWatchdog(mocker):
 
 
 @pytest.mark.slow
-def test_processSubprocessFailureNoPid(mocker):
-    """Test the process method of the JobWrapper class: the subprocess fails and no PID is returned."""
+@pytest.mark.parametrize("expect_failure", [True, False])
+def test_processSubprocessFailureNoPid(mocker, monkeypatch, expect_failure):
+    """Test the process method of the JobWrapper class: the subprocess fails and no PID is returned.
+
+    expect_failure is used to ensure that the JobWrapper is functioning correctly even with the other patching
+    that is applied in the test (e.g. CHILD_PID_POLL_INTERVALS).
+    """
     # Test failure in starting the payload process
     jw = JobWrapper()
     jw.jobArgs = {}
 
     mocker.patch.object(jw, "_JobWrapper__report")
     mocker.patch.object(jw, "_JobWrapper__setJobParam")
+    monkeypatch.setattr(
+        "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.CHILD_PID_POLL_INTERVALS", [0.1, 0.2, 0.3, 0.4, 0.5]
+    )
+
     mock_exeThread = mocker.Mock()
     mock_exeThread.start.side_effect = lambda: time.sleep(0.1)
-    mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ExecutionThread", return_value=mock_exeThread)
+    if expect_failure:
+        mocker.patch(
+            "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ExecutionThread", return_value=mock_exeThread
+        )
 
     with tempfile.NamedTemporaryFile(delete=True) as std_out, tempfile.NamedTemporaryFile(delete=True) as std_err:
         jw.outputFile = std_out.name
         jw.errorFile = std_err.name
         result = jw.process(command="mock_command", env={})
-    assert not result["OK"]
-    assert "Payload process could not start after 140 seconds" in result["Message"]
+
+    if expect_failure:
+        assert not result["OK"]
+        assert "Payload process could not start after 1.5 seconds" in result["Message"]
+    else:
+        assert result["OK"]
 
 
 # -------------------------------------------------------------------------------------------------
@@ -648,6 +664,7 @@ def jobIDPath():
     # Output data files
     (p / "00232454_00000244_1.sim").touch()
     (p / "1720442808testFileUpload.txt").touch()
+    (p / "testFileUploadFullLFN.txt").touch()
 
     with open(p / "pool_xml_catalog.xml", "w") as f:
         f.write(
@@ -847,7 +864,11 @@ def test_processJobOutputs_output_data_upload(mocker, setup_another_job_wrapper)
     # BTW, isn't the concept of pool_xml_catalog.xml from lhcbdirac?
     jw.jobArgs = {
         "OutputSandbox": [],
-        "OutputData": ["1720442808testFileUpload.txt", "LFN:00232454_00000244_1.sim"],
+        "OutputData": [
+            "1720442808testFileUpload.txt",
+            "LFN:00232454_00000244_1.sim",
+            "LFN:/dirac/user/u/unknown/testFileUploadFullLFN.txt",
+        ],
         "Owner": "Jane Doe",
     }
 
@@ -863,10 +884,15 @@ def test_processJobOutputs_output_data_upload(mocker, setup_another_job_wrapper)
     assert jw.jobReport.jobStatusInfo[1][:-1] == ("", JobMinorStatus.UPLOADING_OUTPUT_DATA)
     assert jw.jobReport.jobStatusInfo[2][:-1] == (JobStatus.COMPLETING, JobMinorStatus.OUTPUT_DATA_UPLOADED)
     assert len(jw.jobReport.jobParameters) == 1
-    assert jw.jobReport.jobParameters[0] == (
-        "UploadedOutputData",
-        "00232454_00000244_1.sim, /dirac/user/u/unknown/0/123/1720442808testFileUpload.txt",
-    )
+
+    expected_files = {
+        "00232454_00000244_1.sim",
+        "/dirac/user/u/unknown/0/123/1720442808testFileUpload.txt",
+        "/dirac/user/u/unknown/testFileUploadFullLFN.txt",
+    }
+    assert jw.jobReport.jobParameters[0][0] == "UploadedOutputData"
+    uploaded_files = set(jw.jobReport.jobParameters[0][1].split(", "))
+    assert uploaded_files == expected_files
 
 
 # -------------------------------------------------------------------------------------------------

DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py

@@ -72,6 +72,7 @@ def extraOptions():
     os.remove(extraOptions)
 
 
+@pytest.mark.slow
 def test_createAndExecuteJobWrapperTemplate_success(extraOptions):
     """Test the creation of a classical job wrapper and its execution:
     There is an extra option cfg file to be passed to the job wrapper.
@@ -144,6 +145,7 @@ def test_createAndExecuteJobWrapperTemplate_success(extraOptions):
     shutil.rmtree(os.path.join(os.getcwd(), "job"))
 
 
+@pytest.mark.slow
 def test_createAndExecuteJobWrapperTemplate_missingExtraOptions():
     """Test the creation of a classical job wrapper and its execution:
     There is no extra options to be passed to the job wrapper.
@@ -205,6 +207,7 @@ def test_createAndExecuteJobWrapperTemplate_missingExtraOptions():
     shutil.rmtree(os.path.join(os.getcwd(), "job"))
 
 
+@pytest.mark.slow
 def test_createAndExecuteRelocatedJobWrapperTemplate_success(extraOptions):
     """Test the creation of a relocated job wrapper and its execution:
     This is generally used when containers are involved (SingularityCE).
@@ -325,6 +328,7 @@ def test_createAndExecuteRelocatedJobWrapperTemplate_success(extraOptions):
     shutil.rmtree(wrapperPath)
 
 
+@pytest.mark.slow
 def test_createAndExecuteJobWrapperOfflineTemplate_success(extraOptions):
     """Test the creation of an offline job wrapper and its execution:
     This is generally used when pre/post processing operations are executed locally,

DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py

@@ -21,9 +21,7 @@ from DIRAC.Core.Utilities.JDL import jdlToBaseJobDescriptionModel
 from DIRAC.Core.Utilities.JEncode import strToIntDict
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
-from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
-from DIRAC.WorkloadManagementSystem.Client.JobStatus import filterJobStateTransition
 from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
     RIGHT_DELETE,
     RIGHT_KILL,
@@ -32,8 +30,10 @@ from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
     RIGHT_SUBMIT,
     JobPolicy,
 )
+from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
 from DIRAC.WorkloadManagementSystem.Utilities.JobModel import JobDescriptionModel
 from DIRAC.WorkloadManagementSystem.Utilities.ParametricJob import generateParametricJobs, getParameterVectorLength
+from DIRAC.WorkloadManagementSystem.Utilities.Utils import rescheduleJobs
 
 MAX_PARAMETRIC_JOBS = 20
 
@@ -104,16 +104,6 @@ class JobManagerHandlerMixin:
             return
         self.log.info("Optimize msg sent", f"for {len(jids)} jobs")
 
-    ###########################################################################
-    types_getMaxParametricJobs = []
-
-    def export_getMaxParametricJobs(self):
-        """Get the maximum number of parametric jobs
-
-        :return: S_OK()/S_ERROR()
-        """
-        return S_OK(self.maxParametricJobs)
-
     types_submitJob = [str]
 
     def export_submitJob(self, jobDesc):
@@ -345,8 +335,7 @@ class JobManagerHandlerMixin:
     types_rescheduleJob = []
 
     def export_rescheduleJob(self, jobIDs):
-        """Reschedule a single job. If the optional proxy parameter is given
-        it will be used to refresh the proxy in the Proxy Repository
+        """Reschedule a list of jobs.
 
         :param list jobIDs: list of job IDs
 
@@ -360,22 +349,12 @@ class JobManagerHandlerMixin:
         validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights(
             jobList, RIGHT_RESCHEDULE
         )
-        for jobID in validJobList:
-            self.taskQueueDB.deleteJob(jobID)
-            result = self.jobDB.rescheduleJob(jobID)
-            self.log.debug(str(result))
-            if not result["OK"]:
-                return result
-            self.jobLoggingDB.addLoggingRecord(
-                result["JobID"],
-                status=result["Status"],
-                minorStatus=result["MinorStatus"],
-                applicationStatus="Unknown",
-                source="JobManager",
-            )
+        res = rescheduleJobs(validJobList, source="JobManager")
+        if not res["OK"]:
+            self.log.error(res["Message"])
 
         if invalidJobList or nonauthJobList:
-            result = S_ERROR("Some jobs failed reschedule")
+            result = S_ERROR("Some jobs can not be rescheduled")
             if invalidJobList:
                 result["InvalidJobIDs"] = invalidJobList
             if nonauthJobList:
@@ -450,131 +429,28 @@ class JobManagerHandlerMixin:
 
         return S_OK(validJobList)
 
-    def __deleteJob(self, jobID, force=False):
-        """Set the job status to "Deleted"
-        and remove the pilot that ran and its logging info if the pilot is finished.
-
-        :param int jobID: job ID
-        :return: S_OK()/S_ERROR()
-        """
-        if not (result := self.jobDB.setJobStatus(jobID, JobStatus.DELETED, "Checking accounting", force=force))["OK"]:
-            return result
-
-        if not (result := self.taskQueueDB.deleteJob(jobID))["OK"]:
-            self.log.warn("Failed to delete job from the TaskQueue")
-
-        # if it was the last job for the pilot
-        result = self.pilotAgentsDB.getPilotsForJobID(jobID)
-        if not result["OK"]:
-            self.log.error("Failed to get Pilots for JobID", result["Message"])
-            return result
-        for pilot in result["Value"]:
-            res = self.pilotAgentsDB.getJobsForPilot(pilot)
-            if not res["OK"]:
-                self.log.error("Failed to get jobs for pilot", res["Message"])
-                return res
-            if not res["Value"]:  # if list of jobs for pilot is empty, delete pilot
-                result = self.pilotAgentsDB.getPilotInfo(pilotID=pilot)
-                if not result["OK"]:
-                    self.log.error("Failed to get pilot info", result["Message"])
-                    return result
-                ret = self.pilotAgentsDB.deletePilot(result["Value"]["PilotJobReference"])
-                if not ret["OK"]:
-                    self.log.error("Failed to delete pilot from PilotAgentsDB", ret["Message"])
-                    return ret
-
-        return S_OK()
+    ###########################################################################
+    types_deleteJob = []
 
-    def __killJob(self, jobID, sendKillCommand=True, force=False):
-        """Kill one job
+    def export_deleteJob(self, jobIDs, force=False):
+        """Delete jobs specified in the jobIDs list
 
-        :param int jobID: job ID
-        :param bool sendKillCommand: send kill command
+        :param list jobIDs: list of job IDs
 
-        :return: S_OK()/S_ERROR()
+        :return: S_OK/S_ERROR
         """
-        if sendKillCommand:
-            if not (result := self.jobDB.setJobCommand(jobID, "Kill"))["OK"]:
-                return result
-
-        self.log.info("Job marked for termination", jobID)
-        if not (result := self.jobDB.setJobStatus(jobID, JobStatus.KILLED, "Marked for termination", force=force))[
-            "OK"
-        ]:
-            self.log.warn("Failed to set job Killed status", result["Message"])
-        if not (result := self.taskQueueDB.deleteJob(jobID))["OK"]:
-            self.log.warn("Failed to delete job from the TaskQueue", result["Message"])
-
-        return S_OK()
-
-    def _kill_delete_jobs(self, jobIDList, right, force=False):
-        """Kill (== set the status to "KILLED") or delete (== set the status to "DELETED") jobs as necessary
-
-        :param list jobIDList: job IDs
-        :param str right: RIGHT_KILL or RIGHT_DELETE
 
-        :return: S_OK()/S_ERROR()
-        """
-        jobList = self.__getJobList(jobIDList)
+        jobList = self.__getJobList(jobIDs)
         if not jobList:
             self.log.warn("No jobs specified")
             return S_OK([])
 
-        validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights(jobList, right)
-
-        badIDs = []
-
-        killJobList = []
-        deleteJobList = []
-        if validJobList:
-            # Get the jobs allowed to transition to the Killed state
-            filterRes = filterJobStateTransition(validJobList, JobStatus.KILLED)
-            if not filterRes["OK"]:
-                return filterRes
-            killJobList.extend(filterRes["Value"])
-
-            if not right == RIGHT_KILL:
-                # Get the jobs allowed to transition to the Deleted state
-                filterRes = filterJobStateTransition(validJobList, JobStatus.DELETED)
-                if not filterRes["OK"]:
-                    return filterRes
-                deleteJobList.extend(filterRes["Value"])
-
-            # Look for jobs that are in the Staging state to send kill signal to the stager
-            result = self.jobDB.getJobsAttributes(killJobList, ["Status"])
-            if not result["OK"]:
-                return result
-            stagingJobList = [jobID for jobID, sDict in result["Value"].items() if sDict["Status"] == JobStatus.STAGING]
-
-            for jobID in killJobList:
-                result = self.__killJob(jobID, force=force)
-                if not result["OK"]:
-                    badIDs.append(jobID)
-
-            for jobID in deleteJobList:
-                result = self.__deleteJob(jobID, force=force)
-                if not result["OK"]:
-                    badIDs.append(jobID)
-
-            if stagingJobList:
-                stagerClient = StorageManagerClient()
-                self.log.info("Going to send killing signal to stager as well!")
-                result = stagerClient.killTasksBySourceTaskID(stagingJobList)
-                if not result["OK"]:
-                    self.log.warn("Failed to kill some Stager tasks", result["Message"])
+        validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights(
+            jobList, RIGHT_DELETE
+        )
 
-        if nonauthJobList or badIDs:
-            result = S_ERROR("Some jobs failed deletion")
-            if nonauthJobList:
-                self.log.warn("Non-authorized JobIDs won't be deleted", str(nonauthJobList))
-                result["NonauthorizedJobIDs"] = nonauthJobList
-            if badIDs:
-                self.log.warn("JobIDs failed to be deleted", str(badIDs))
-                result["FailedJobIDs"] = badIDs
-            return result
+        result = kill_delete_jobs(RIGHT_DELETE, validJobList, nonauthJobList, force=force)
 
-        jobsList = killJobList if right == RIGHT_KILL else deleteJobList
-        result = S_OK(jobsList)
         result["requireProxyUpload"] = len(ownerJobList) > 0 and self.__checkIfProxyUploadIsRequired()
 
         if invalidJobList:
@@ -583,30 +459,33 @@ class JobManagerHandlerMixin:
         return result
 
     ###########################################################################
-    types_deleteJob = []
+    types_killJob = []
 
-    def export_deleteJob(self, jobIDs, force=False):
-        """Delete jobs specified in the jobIDs list
+    def export_killJob(self, jobIDs, force=False):
+        """Kill jobs specified in the jobIDs list
 
         :param list jobIDs: list of job IDs
 
         :return: S_OK/S_ERROR
         """
 
-        return self._kill_delete_jobs(jobIDs, RIGHT_DELETE, force=force)
+        jobList = self.__getJobList(jobIDs)
+        if not jobList:
+            self.log.warn("No jobs specified")
+            return S_OK([])
 
-    ###########################################################################
-    types_killJob = []
+        validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights(
+            jobList, RIGHT_KILL
+        )
 
-    def export_killJob(self, jobIDs, force=False):
-        """Kill jobs specified in the jobIDs list
+        result = kill_delete_jobs(RIGHT_KILL, validJobList, nonauthJobList, force=force)
 
-        :param list jobIDs: list of job IDs
+        result["requireProxyUpload"] = len(ownerJobList) > 0 and self.__checkIfProxyUploadIsRequired()
 
-        :return: S_OK/S_ERROR
-        """
+        if invalidJobList:
+            result["InvalidJobIDs"] = invalidJobList
 
-        return self._kill_delete_jobs(jobIDs, RIGHT_KILL, force=force)
+        return result
 
     ###########################################################################
     types_resetJob = []