DIRAC 9.0.0a68__py3-none-any.whl → 9.0.0a70__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -1,4 +1,4 @@
-"""  The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.
+"""The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.
 
 .. literalinclude:: ../ConfigTemplate.cfg
   :start-after: ##BEGIN SiteDirector
@@ -7,6 +7,7 @@
   :caption: SiteDirector options
 
 """
+
 import datetime
 import os
 from collections import defaultdict
@@ -147,10 +148,10 @@ class SiteDirector(AgentModule):
             self.sendSubmissionAccounting = True
 
         # Get the site description dictionary
-        siteNames = self.am_getOption("Site", [])
-        ceTypes = self.am_getOption("CETypes", [])
-        ces = self.am_getOption("CEs", [])
-        tags = self.am_getOption("Tags", [])
+        siteNames = self.am_getOption("Site", []) or None
+        ceTypes = self.am_getOption("CETypes", []) or None
+        ces = self.am_getOption("CEs", []) or None
+        tags = self.am_getOption("Tags", []) or None
 
         # Display options used
         self.log.always("VO:", self.vo)
@@ -229,12 +230,8 @@ class SiteDirector(AgentModule):
             site = self.queueDict[queueName]["Site"]
             ce = self.queueDict[queueName]["CEName"]
 
-            # Check the status of the Site
-            if site in siteMaskList:
-                continue
-
-            # Check the status of the CE (only for RSS=Active)
-            if ce not in ceMaskList:
+            # Check the status of the Site and CE
+            if site in siteMaskList and ce in ceMaskList:
                 continue
 
             self.log.warn("Queue not considered because not usable:", queueName)

DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py

@@ -17,11 +17,9 @@ from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Utilities import DErrno
 from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
+from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.Core.Utilities.TimeUtilities import fromString, second, toEpoch
 from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus, JobStatus
-from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
-from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
-from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
 from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_KILL
 from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
 from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
@@ -40,6 +38,9 @@ class StalledJobAgent(AgentModule):
 
         self.jobDB = None
         self.logDB = None
+        self.taskQueueDB = None
+        self.pilotAgentsDB = None
+        self.storageManagementDB = None
         self.matchedTime = 7200
         self.rescheduledTime = 600
         self.submittingTime = 300
@@ -51,8 +52,30 @@ class StalledJobAgent(AgentModule):
     #############################################################################
     def initialize(self):
         """Sets default parameters."""
-        self.jobDB = JobDB()
-        self.logDB = JobLoggingDB()
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobDB", "JobDB")
+        if not result["OK"]:
+            return result
+        self.jobDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
+        if not result["OK"]:
+            return result
+        self.logDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
+        if not result["OK"]:
+            return result
+        self.taskQueueDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
+        if not result["OK"]:
+            return result
+        self.pilotAgentsDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("StorageManagementSystem.DB.StorageManagementDB", "StorageManagementDB")
+        if not result["OK"]:
+            return result
+        self.storageManagementDB = result["Value"]()
 
         # getting parameters
 
@@ -235,7 +258,16 @@ class StalledJobAgent(AgentModule):
         # Set the jobs Failed, send them a kill signal in case they are not really dead
         # and send accounting info
         if setFailed:
-            res = kill_delete_jobs(RIGHT_KILL, [jobID], nonauthJobList=[], force=True)
+            res = kill_delete_jobs(
+                RIGHT_KILL,
+                [jobID],
+                nonauthJobList=[],
+                force=True,
+                jobdb=self.jobDB,
+                taskqueuedb=self.taskQueueDB,
+                pilotagentsdb=self.pilotAgentsDB,
+                storagemanagementdb=self.storageManagementDB,
+            )
             if not res["OK"]:
                 self.log.error("Failed to kill job", jobID)
 
@@ -262,7 +294,7 @@ class StalledJobAgent(AgentModule):
             # There is no pilot reference, hence its status is unknown
             return S_OK("NoPilot")
 
-        result = PilotAgentsDB().getPilotInfo(pilotReference)
+        result = self.pilotAgentsDB.getPilotInfo(pilotReference)
         if not result["OK"]:
             if DErrno.cmpError(result, DErrno.EWMSNOPILOT):
                 self.log.warn("No pilot found", f"for job {jobID}: {result['Message']}")

DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py

@@ -169,10 +169,16 @@ def sd(mocker, config):
         gConfig.getSections("Resources/Sites/LCG")["Value"] + gConfig.getSections("Resources/Sites/DIRAC")["Value"]
     )
     mocker.patch(
-        "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.SiteStatus.getUsableSites", return_values=usableSites
+        "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.SiteStatus.getUsableSites", return_value=S_OK(usableSites)
     )
+
+    # Mock getElementStatus to return a properly formatted dictionary
+    def mock_getElementStatus(ceNamesList, *args, **kwargs):
+        return S_OK({ceName: {"all": "Active"} for ceName in ceNamesList})
+
     mocker.patch(
-        "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.ResourceStatus.getElementStatus", return_values=usableSites
+        "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.ResourceStatus.getElementStatus",
+        side_effect=mock_getElementStatus,
     )
     mocker.patch(
         "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.gProxyManager.downloadProxy", side_effect=mockPMProxyReply

DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py

@@ -23,10 +23,31 @@ def sja(mocker):
         side_effect=lambda x, y=None: y,
         create=True,
     )
-    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobDB")
-    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobLoggingDB")
+
+    # Mock ObjectLoader to return mock DB instances
+    mockJobDB = MagicMock()
+    mockJobDB.log = gLogger
+    mockJobLoggingDB = MagicMock()
+    mockTaskQueueDB = MagicMock()
+    mockPilotAgentsDB = MagicMock()
+    mockStorageManagementDB = MagicMock()
+
+    def mock_load_object(module_path, class_name):
+        mocks = {
+            "JobDB": mockJobDB,
+            "JobLoggingDB": mockJobLoggingDB,
+            "TaskQueueDB": mockTaskQueueDB,
+            "PilotAgentsDB": mockPilotAgentsDB,
+            "StorageManagementDB": mockStorageManagementDB,
+        }
+        return {"OK": True, "Value": lambda: mocks[class_name]}
+
+    mocker.patch(
+        "DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.ObjectLoader.loadObject",
+        side_effect=mock_load_object,
+    )
+
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.rescheduleJobs", return_value=MagicMock())
-    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.PilotAgentsDB", return_value=MagicMock())
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getJobParameters", return_value=MagicMock())
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.kill_delete_jobs", return_value=MagicMock())
 
@@ -34,7 +55,6 @@ def sja(mocker):
     stalledJobAgent._AgentModule__configDefaults = mockAM
     stalledJobAgent.log = gLogger
     stalledJobAgent.initialize()
-    stalledJobAgent.jobDB.log = gLogger
     stalledJobAgent.log.setLevel("DEBUG")
     stalledJobAgent.stalledTime = 120
 

DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py

@@ -1,6 +1,7 @@
-""" The Download Input Data module wraps around the Replica Management
-    components to provide access to datasets by downloading locally
+"""The Download Input Data module wraps around the Replica Management
+components to provide access to datasets by downloading locally
 """
+
 import os
 import random
 import tempfile
@@ -226,7 +227,7 @@ class DownloadInputData:
         diskSpace = getDiskSpace(self.__getDownloadDir(False))  # MB
         availableBytes = diskSpace * 1024 * 1024  # bytes
         bufferGBs = gConfig.getValue(
-            os.path.join("/Systems/WorkloadManagement/JobWrapper", "JobWrapper", "MinOutputDataBufferGB"), 5.0
+            os.path.join("/Systems/WorkloadManagement/JobWrapper", "MinOutputDataBufferGB"), 5.0
         )
         data = bufferGBs * 1024 * 1024 * 1024  # bufferGBs in bytes
         if (data + totalSize) < availableBytes:

DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg

@@ -263,11 +263,11 @@ Agents
     # the DN of the certificate proxy used to submit pilots. If not found here, what is in Operations/Pilot section of the CS will be used
     PilotDN =
 
-    # List of sites that will be treated by this SiteDirector (No value can refer to any Site defined in the CS)
+    # List of Sites that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
     Site =
-    # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
+    # List of CEs that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
     CEs =
-    # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
+    # List of CETypes that are required to be present in the CE/Queue definition
     CETypes =
     # List of Tags that are required to be present in the CE/Queue definition
     Tags =

DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py

@@ -1,10 +1,10 @@
-""" Module containing a front-end to the OpenSearch-based JobParametersDB.
-    This is a drop-in replacement for MySQL-based table JobDB.JobParameters.
+"""Module containing a front-end to the OpenSearch-based JobParametersDB.
+This is a drop-in replacement for MySQL-based table JobDB.JobParameters.
 
-    The following class methods are provided for public usage
-      - getJobParameters()
-      - setJobParameter()
-      - deleteJobParameters()
+The following class methods are provided for public usage
+  - getJobParameters()
+  - setJobParameter()
+  - deleteJobParameters()
 """
 
 from DIRAC import S_ERROR, S_OK
@@ -37,11 +37,11 @@ class JobParametersDB(ElasticDB):
     def __init__(self, parentLogger=None):
         """Standard Constructor"""
 
-        self.fullname = "WorkloadManagement/ElasticJobParametersDB"
+        self.fullname = "WorkloadManagement/JobParametersDB"
         self.index_name = self.getCSOption("index_name", "job_parameters")
 
         try:
-            # Connecting to the ES cluster
+            # Connecting to the OpenSearch cluster
             super().__init__(self.fullname, self.index_name, parentLogger=parentLogger)
         except Exception:
             RuntimeError("Can't connect to JobParameters index")

DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py

@@ -64,7 +64,7 @@ class SandboxMetadataDB(DB):
                 "Type": "VARCHAR(64) NOT NULL",
             },
             "Indexes": {"Entity": ["EntityId"], "SBIndex": ["SBId"]},
-            "UniqueIndexes": {"Mapping": ["SBId", "EntityId", "Type"]},
+            "PrimaryKey": ["SBId", "EntityId", "Type"],
         }
 
         for tableName in self.__tablesDesc:

DIRAC/WorkloadManagementSystem/DB/StatusUtils.py

@@ -1,43 +1,40 @@
 from DIRAC import S_ERROR, S_OK, gLogger
-from DIRAC.StorageManagementSystem.DB.StorageManagementDB import StorageManagementDB
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
-from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
-from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
-from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB
+from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_DELETE, RIGHT_KILL
 from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import _filterJobStateTransition
 
 
-def _deleteJob(jobID, force=False):
+def _deleteJob(jobID, force=False, *, jobdb, taskqueuedb, pilotagentsdb):
     """Set the job status to "Deleted"
     and remove the pilot that ran and its logging info if the pilot is finished.
 
     :param int jobID: job ID
     :return: S_OK()/S_ERROR()
     """
-    if not (result := JobDB().setJobStatus(jobID, JobStatus.DELETED, "Checking accounting", force=force))["OK"]:
+    if not (result := jobdb.setJobStatus(jobID, JobStatus.DELETED, "Checking accounting", force=force))["OK"]:
         gLogger.warn("Failed to set job Deleted status", result["Message"])
         return result
 
-    if not (result := TaskQueueDB().deleteJob(jobID))["OK"]:
+    if not (result := taskqueuedb.deleteJob(jobID))["OK"]:
         gLogger.warn("Failed to delete job from the TaskQueue")
 
     # if it was the last job for the pilot
-    result = PilotAgentsDB().getPilotsForJobID(jobID)
+    result = pilotagentsdb.getPilotsForJobID(jobID)
     if not result["OK"]:
         gLogger.error("Failed to get Pilots for JobID", result["Message"])
         return result
     for pilot in result["Value"]:
-        res = PilotAgentsDB().getJobsForPilot(pilot)
+        res = pilotagentsdb.getJobsForPilot(pilot)
         if not res["OK"]:
             gLogger.error("Failed to get jobs for pilot", res["Message"])
             return res
         if not res["Value"]:  # if list of jobs for pilot is empty, delete pilot
-            result = PilotAgentsDB().getPilotInfo(pilotID=pilot)
+            result = pilotagentsdb.getPilotInfo(pilotID=pilot)
             if not result["OK"]:
                 gLogger.error("Failed to get pilot info", result["Message"])
                 return result
-            ret = PilotAgentsDB().deletePilot(result["Value"]["PilotJobReference"])
+            ret = pilotagentsdb.deletePilot(result["Value"]["PilotJobReference"])
             if not ret["OK"]:
                 gLogger.error("Failed to delete pilot from PilotAgentsDB", ret["Message"])
                 return ret
@@ -45,7 +42,7 @@ def _deleteJob(jobID, force=False):
     return S_OK()
 
 
-def _killJob(jobID, sendKillCommand=True, force=False):
+def _killJob(jobID, sendKillCommand=True, force=False, *, jobdb, taskqueuedb):
     """Kill one job
 
     :param int jobID: job ID
@@ -54,32 +51,63 @@ def _killJob(jobID, sendKillCommand=True, force=False):
     :return: S_OK()/S_ERROR()
     """
     if sendKillCommand:
-        if not (result := JobDB().setJobCommand(jobID, "Kill"))["OK"]:
+        if not (result := jobdb.setJobCommand(jobID, "Kill"))["OK"]:
             gLogger.warn("Failed to set job Kill command", result["Message"])
             return result
 
     gLogger.info("Job marked for termination", jobID)
-    if not (result := JobDB().setJobStatus(jobID, JobStatus.KILLED, "Marked for termination", force=force))["OK"]:
+    if not (result := jobdb.setJobStatus(jobID, JobStatus.KILLED, "Marked for termination", force=force))["OK"]:
         gLogger.warn("Failed to set job Killed status", result["Message"])
-    if not (result := TaskQueueDB().deleteJob(jobID))["OK"]:
+    if not (result := taskqueuedb.deleteJob(jobID))["OK"]:
         gLogger.warn("Failed to delete job from the TaskQueue", result["Message"])
 
     return S_OK()
 
 
-def kill_delete_jobs(right, validJobList, nonauthJobList=[], force=False):
+def kill_delete_jobs(
+    right,
+    validJobList,
+    nonauthJobList=[],
+    force=False,
+    *,
+    jobdb=None,
+    taskqueuedb=None,
+    pilotagentsdb=None,
+    storagemanagementdb=None,
+):
     """Kill (== set the status to "KILLED") or delete (== set the status to "DELETED") jobs as necessary
 
     :param str right: RIGHT_KILL or RIGHT_DELETE
 
     :return: S_OK()/S_ERROR()
     """
+    if jobdb is None:
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobDB", "JobDB")
+        if not result["OK"]:
+            return result
+        jobdb = result["Value"]()
+    if taskqueuedb is None:
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
+        if not result["OK"]:
+            return result
+        taskqueuedb = result["Value"]()
+    if pilotagentsdb is None:
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
+        if not result["OK"]:
+            return result
+        pilotagentsdb = result["Value"]()
+    if storagemanagementdb is None:
+        result = ObjectLoader().loadObject("StorageManagementSystem.DB.StorageManagementDB", "StorageManagementDB")
+        if not result["OK"]:
+            return result
+        storagemanagementdb = result["Value"]()
+
     badIDs = []
 
     killJobList = []
     deleteJobList = []
     if validJobList:
-        result = JobDB().getJobsAttributes(killJobList, ["Status"])
+        result = jobdb.getJobsAttributes(validJobList, ["Status"])
         if not result["OK"]:
             return result
         jobStates = result["Value"]
@@ -92,12 +120,12 @@ def kill_delete_jobs(right, validJobList, nonauthJobList=[], force=False):
             deleteJobList.extend(_filterJobStateTransition(jobStates, JobStatus.DELETED))
 
         for jobID in killJobList:
-            result = _killJob(jobID, force=force)
+            result = _killJob(jobID, force=force, jobdb=jobdb, taskqueuedb=taskqueuedb)
             if not result["OK"]:
                 badIDs.append(jobID)
 
         for jobID in deleteJobList:
-            result = _deleteJob(jobID, force=force)
+            result = _deleteJob(jobID, force=force, jobdb=jobdb, taskqueuedb=taskqueuedb, pilotagentsdb=pilotagentsdb)
             if not result["OK"]:
                 badIDs.append(jobID)
 
@@ -105,9 +133,8 @@ def kill_delete_jobs(right, validJobList, nonauthJobList=[], force=False):
         stagingJobList = [jobID for jobID, sDict in jobStates.items() if sDict["Status"] == JobStatus.STAGING]
 
         if stagingJobList:
-            stagerDB = StorageManagementDB()
             gLogger.info("Going to send killing signal to stager as well!")
-            result = stagerDB.killTasksBySourceTaskID(stagingJobList)
+            result = storagemanagementdb.killTasksBySourceTaskID(stagingJobList)
             if not result["OK"]:
                 gLogger.warn("Failed to kill some Stager tasks", result["Message"])
 

DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py

@@ -19,10 +19,25 @@ from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
     ],
 )
 def test___kill_delete_jobs(mocker, jobIDs_list, right):
-    mocker.patch("DIRAC.WorkloadManagementSystem.DB.StatusUtils.JobDB", MagicMock())
-    mocker.patch("DIRAC.WorkloadManagementSystem.DB.StatusUtils.TaskQueueDB", MagicMock())
-    mocker.patch("DIRAC.WorkloadManagementSystem.DB.StatusUtils.PilotAgentsDB", MagicMock())
-    mocker.patch("DIRAC.WorkloadManagementSystem.DB.StatusUtils.StorageManagementDB", MagicMock())
+    # Mock ObjectLoader to return mock DB instances
+    mockJobDB = MagicMock()
+    mockTaskQueueDB = MagicMock()
+    mockPilotAgentsDB = MagicMock()
+    mockStorageManagementDB = MagicMock()
+
+    def mock_load_object(module_path, class_name):
+        mocks = {
+            "JobDB": mockJobDB,
+            "TaskQueueDB": mockTaskQueueDB,
+            "PilotAgentsDB": mockPilotAgentsDB,
+            "StorageManagementDB": mockStorageManagementDB,
+        }
+        return {"OK": True, "Value": lambda: mocks[class_name]}
+
+    mocker.patch(
+        "DIRAC.WorkloadManagementSystem.DB.StatusUtils.ObjectLoader.loadObject",
+        side_effect=mock_load_object,
+    )
 
     res = kill_delete_jobs(right, jobIDs_list)
     assert res["OK"]

DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py

@@ -16,7 +16,6 @@ import datetime
 import glob
 import json
 import os
-from pathlib import Path
 import re
 import shutil
 import stat
@@ -24,12 +23,12 @@ import sys
 import tarfile
 import threading
 import time
+from pathlib import Path
 from urllib.parse import unquote
 
 import DIRAC
 from DIRAC import S_ERROR, S_OK, gConfig, gLogger
 from DIRAC.AccountingSystem.Client.Types.Job import Job as AccountingJob
-
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
 from DIRAC.Core.Utilities import DEncode, DErrno, List
@@ -1021,7 +1020,7 @@ class JobWrapper:
 
         for i in outputSandbox:
             if i not in okFiles:
-                if not f"{i}.tar" in okFiles:
+                if f"{i}.tar" not in okFiles:
                     if not re.search(r"\*", i):
                         if i not in missing:
                             missing.append(i)
@@ -1215,8 +1214,8 @@ class JobWrapper:
             lfn = str(basePath / outputPath / os.path.basename(localfile))
         else:
             # if LFN is given, take it as it is
-            localfile = str(self.jobIDPath / outputFile.replace("LFN:", ""))
             lfn = outputFile.replace("LFN:", "")
+            localfile = str(self.jobIDPath / os.path.basename(lfn))
 
         return (lfn, localfile)
 

DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py

@@ -1,22 +1,22 @@
-"""  The Watchdog class is used by the Job Wrapper to resolve and monitor
-     the system resource consumption.  The Watchdog can determine if
-     a running job is stalled and indicate this to the Job Wrapper.
-     Furthermore, the Watchdog will identify when the Job CPU limit has been
-     exceeded and fail jobs meaningfully.
-
-     Information is returned to the WMS via the heart-beat mechanism.  This
-     also interprets control signals from the WMS e.g. to kill a running
-     job.
-
-     - Still to implement:
-          - CPU normalization for correct comparison with job limit
+"""The Watchdog class is used by the Job Wrapper to resolve and monitor
+the system resource consumption.  The Watchdog can determine if
+a running job is stalled and indicate this to the Job Wrapper.
+Furthermore, the Watchdog will identify when the Job CPU limit has been
+exceeded and fail jobs meaningfully.
+
+Information is returned to the WMS via the heart-beat mechanism.  This
+also interprets control signals from the WMS e.g. to kill a running
+job.
+
+- Still to implement:
+     - CPU normalization for correct comparison with job limit
 """
+
 import datetime
 import errno
 import getpass
 import math
 import os
-import signal
 import socket
 import time
 from pathlib import Path
@@ -32,28 +32,6 @@ from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus
 from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
 
 
-def kill_proc_tree(pid, sig=signal.SIGTERM, includeParent=True):
-    """Kill a process tree (including grandchildren) with signal
-    "sig" and return a (gone, still_alive) tuple.
-    called as soon as a child terminates.
-
-    Taken from https://psutil.readthedocs.io/en/latest/index.html#kill-process-tree
-    """
-    assert pid != os.getpid(), "won't kill myself"
-    parent = psutil.Process(pid)
-    children = parent.children(recursive=True)
-    if includeParent:
-        children.append(parent)
-    for p in children:
-        try:
-            p.send_signal(sig)
-        except psutil.NoSuchProcess:
-            pass
-    _gone, alive = psutil.wait_procs(children, timeout=10)
-    for p in alive:
-        p.kill()
-
-
 class Watchdog:
     #############################################################################
     def __init__(self, pid, exeThread, spObject, jobCPUTime, memoryLimit=0, processors=1, jobArgs={}):
@@ -212,7 +190,7 @@ class Watchdog:
             if self.littleTimeLeftCount == 0 and self.__timeLeft() == -1:
                 self.checkError = JobMinorStatus.JOB_EXCEEDED_CPU
                 self.log.error(self.checkError, self.timeLeft)
-                self.__killRunningThread()
+                self.spObject.killChild()
                 return S_OK()
 
             self.littleTimeLeftCount -= 1
@@ -321,7 +299,7 @@ class Watchdog:
 
                     self.log.info("=================END=================")
 
-            self.__killRunningThread()
+            self.spObject.killChild()
             return S_OK()
 
         recentStdOut = "None"
@@ -408,7 +386,7 @@ class Watchdog:
             if "Kill" in signalDict:
                 self.log.info("Received Kill signal, stopping job via control signal")
                 self.checkError = JobMinorStatus.RECEIVED_KILL_SIGNAL
-                self.__killRunningThread()
+                self.spObject.killChild()
             else:
                 self.log.info("The following control signal was sent but not understood by the watchdog:")
                 self.log.info(signalDict)
@@ -862,13 +840,6 @@ class Watchdog:
 
         return result
 
-    #############################################################################
-    def __killRunningThread(self):
-        """Will kill the running thread process and any child processes."""
-        self.log.info("Sending kill signal to application PID", self.spObject.getChildPID())
-        self.spObject.killChild()
-        return S_OK("Thread killed")
-
     #############################################################################
     def __sendSignOfLife(self, jobID, heartBeatDict, staticParamDict):
         """Sends sign of life 'heartbeat' signal and triggers control signal

DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py

@@ -1,5 +1,5 @@
-""" Test class for JobWrapper
-"""
+"""Test class for JobWrapper"""
+
 import os
 import shutil
 import tempfile
@@ -314,8 +314,7 @@ def test_processKilledSubprocess(mocker):
         result = jw.process("sleep 20", {})
 
     assert result["OK"]
-    assert result["Value"]["payloadStatus"] == 15  # SIGTERM
-    assert not result["Value"]["payloadOutput"]
+    assert result["Value"]["payloadStatus"] is None
     assert not result["Value"]["payloadExecutorError"]
     assert result["Value"]["watchdogError"] == "Job is stalled!"  # Error message from the watchdog
 
@@ -664,6 +663,7 @@ def jobIDPath():
     # Output data files
     (p / "00232454_00000244_1.sim").touch()
     (p / "1720442808testFileUpload.txt").touch()
+    (p / "testFileUploadFullLFN.txt").touch()
 
     with open(p / "pool_xml_catalog.xml", "w") as f:
         f.write(
@@ -863,7 +863,11 @@ def test_processJobOutputs_output_data_upload(mocker, setup_another_job_wrapper)
     # BTW, isn't the concept of pool_xml_catalog.xml from lhcbdirac?
     jw.jobArgs = {
         "OutputSandbox": [],
-        "OutputData": ["1720442808testFileUpload.txt", "LFN:00232454_00000244_1.sim"],
+        "OutputData": [
+            "1720442808testFileUpload.txt",
+            "LFN:00232454_00000244_1.sim",
+            "LFN:/dirac/user/u/unknown/testFileUploadFullLFN.txt",
+        ],
         "Owner": "Jane Doe",
     }
 
@@ -879,10 +883,15 @@ def test_processJobOutputs_output_data_upload(mocker, setup_another_job_wrapper)
     assert jw.jobReport.jobStatusInfo[1][:-1] == ("", JobMinorStatus.UPLOADING_OUTPUT_DATA)
     assert jw.jobReport.jobStatusInfo[2][:-1] == (JobStatus.COMPLETING, JobMinorStatus.OUTPUT_DATA_UPLOADED)
     assert len(jw.jobReport.jobParameters) == 1
-    assert jw.jobReport.jobParameters[0] == (
-        "UploadedOutputData",
-        "00232454_00000244_1.sim, /dirac/user/u/unknown/0/123/1720442808testFileUpload.txt",
-    )
+
+    expected_files = {
+        "00232454_00000244_1.sim",
+        "/dirac/user/u/unknown/0/123/1720442808testFileUpload.txt",
+        "/dirac/user/u/unknown/testFileUploadFullLFN.txt",
+    }
+    assert jw.jobReport.jobParameters[0][0] == "UploadedOutputData"
+    uploaded_files = set(jw.jobReport.jobParameters[0][1].split(", "))
+    assert uploaded_files == expected_files
 
 
 # -------------------------------------------------------------------------------------------------