DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Agent/JobAgent.py

@@ -5,34 +5,36 @@
   and the current resource status that is used for matching.
 """
 import os
-import sys
 import re
+import sys
 import time
+from pathlib import Path
 
 from diraccfg import CFG
 
-from DIRAC import S_OK, S_ERROR, gConfig, rootPath, siteName
+from DIRAC import S_ERROR, S_OK, gConfig, rootPath, siteName
 from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername
-from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 from DIRAC.Core.Base.AgentModule import AgentModule
-from DIRAC.Core.Security.ProxyInfo import getProxyInfo
 from DIRAC.Core.Security import Properties
+from DIRAC.Core.Security.ProxyFile import writeChainToTemporaryFile
+from DIRAC.Core.Security.ProxyInfo import getProxyInfo
 from DIRAC.Core.Utilities import DErrno
+from DIRAC.Core.Utilities.CGroups2 import CG2Manager
+from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
-from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import TimeLeft
-from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory
-from DIRAC.RequestManagementSystem.Client.Request import Request
 from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
+from DIRAC.RequestManagementSystem.Client.Request import Request
 from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator
-from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient
-from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
+from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import TimeLeft
+from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory
+from DIRAC.WorkloadManagementSystem.Client import JobStatus, PilotStatus
 from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient
 from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
 from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
-from DIRAC.WorkloadManagementSystem.Client import JobStatus
+from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient
+from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
 from DIRAC.WorkloadManagementSystem.Utilities.Utils import createJobWrapper
-from DIRAC.WorkloadManagementSystem.Client import PilotStatus
 
 
 class JobAgent(AgentModule):
@@ -134,6 +136,14 @@ class JobAgent(AgentModule):
 
         # Utilities
         self.timeLeftUtil = TimeLeft()
+
+        # Some innerCEs may want to make use of CGroup2 support, so we prepare it globally here
+        res = CG2Manager().setUp()
+        if res["OK"]:
+            self.log.info("CGroup2 support configured successfully.")
+        else:
+            self.log.info("CGroup2 support unavailable:", res["Message"])
+
         return S_OK()
 
     def _initializeComputingElement(self, localCE):
@@ -226,7 +236,6 @@ class JobAgent(AgentModule):
         jobGroup = matcherInfo["Group"]
         owner = matcherInfo["Owner"]
         ceDict = matcherInfo["CEDict"]
-        matchTime = matcherInfo["matchTime"]
 
         optimizerParams = {}
         for key in matcherInfo:
@@ -253,9 +262,6 @@ class JobAgent(AgentModule):
         self.log.verbose("Job request successful: \n", jobRequest["Value"])
         self.log.info("Received", f"JobID={jobID}, JobType={jobType}, Owner={owner}, JobGroup={jobGroup}")
         self.jobCount += 1
-        self.jobs[jobID]["JobReport"].setJobParameter(
-            par_name="MatcherServiceTime", par_value=str(matchTime), sendFlag=False
-        )
 
         self.jobs[jobID]["JobReport"].setJobStatus(minorStatus="Job Received by Agent", sendFlag=False)
         result_setupProxy = self._setupProxy(owner, jobGroup)
@@ -476,8 +482,6 @@ class JobAgent(AgentModule):
 
         proxyChain = ret["Value"]["chain"]
         if "groupProperties" not in ret["Value"]:
-            print(ret["Value"])
-            print(proxyChain.dumpAllToString())
             self.log.error("Invalid Proxy", "Group has no properties defined")
             return S_ERROR("Proxy has no group properties defined")
 
@@ -539,7 +543,7 @@ class JobAgent(AgentModule):
             jobRequest = MatcherClient().requestJob(ceDict)
             matchTime = time.time() - start
 
-            self.log.info("MatcherTime", f"= {matchTime:.2f} (s)")
+            self.log.verbose("MatcherTime", f"= {matchTime:.2f} (s)")
             if jobRequest["OK"]:
                 jobRequest["Value"]["matchTime"] = matchTime
                 jobRequest["Value"]["CEDict"] = ceDict
@@ -626,13 +630,15 @@ class JobAgent(AgentModule):
 
         self.log.info("Submitting JobWrapper", f"{os.path.basename(wrapperFile)} to {self.ceName}CE")
 
-        # Pass proxy to the CE
-        proxy = proxyChain.dumpAllToString()
-        if not proxy["OK"]:
-            self.log.error("Invalid proxy", proxy)
-            return S_ERROR("Payload Proxy Not Found")
+        # Pass proxy to the CE, writing it to a temporary file to ensure the DiracX token is included
+        retVal = writeChainToTemporaryFile(proxyChain)
+        if not retVal["OK"]:
+            self.log.error("Invalid proxy", retVal["Message"])
+            return S_ERROR("Failed to write proxy to temporary file")
+        proxyLocation = Path(retVal["Value"])
+        payloadProxy = proxyLocation.read_text()
+        proxyLocation.unlink()
 
-        payloadProxy = proxy["Value"]
         try:
             result = self.computingElement.submitJob(
                 wrapperFile,
@@ -651,7 +657,7 @@ class JobAgent(AgentModule):
             self.log.exception("Exception occurred when submitting", f"JobID: {jobID}")
             taskID = 0
             # We create a S_ERROR from the exception to compute it as a normal error
-            self.computingElement.taskResults[taskID] = S_ERROR(unexpectedSubmitException)
+            self.computingElement.taskResults[taskID] = S_ERROR(str(unexpectedSubmitException))
             self.jobs[jobID]["TaskID"] = taskID
             return S_OK()
 
@@ -683,7 +689,13 @@ class JobAgent(AgentModule):
         # Here we iterate over a copy of the keys because we are modifying the dictionary within the loop
         for jobID in list(self.jobs.keys()):
             taskID = self.jobs[jobID].get("TaskID")
-            if taskID is None or taskID not in self.computingElement.taskResults:
+            if taskID is None:
+                # This generally means that there was an error before the submission
+                # and the TaskID was not set and will never be.
+                self.log.info("No taskID found for job", jobID)
+                del self.jobs[jobID]
+                continue
+            if taskID not in self.computingElement.taskResults:
                 continue
 
             result = self.computingElement.taskResults[taskID]

DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py

@@ -35,10 +35,12 @@ from DIRAC.RequestManagementSystem.Client.Operation import Operation
 from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
 from DIRAC.RequestManagementSystem.Client.Request import Request
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
-from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
-from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient
 from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
+from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
+from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_DELETE
+from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
+from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
 
 
 class JobCleaningAgent(AgentModule):
@@ -152,8 +154,9 @@ class JobCleaningAgent(AgentModule):
             return S_OK()
 
         self.log.info("Unassigning sandboxes from soon to be deleted jobs", f"({len(jobList)})")
-        result = SandboxStoreClient(useCertificates=True).unassignJobs(jobList)
-        if not result["OK"]:
+
+        entitiesList = [f"Job:{jobId}" for jobId in jobList]
+        if not (result := SandboxMetadataDB().unassignEntities(entitiesList))["OK"]:
             self.log.error("Cannot unassign jobs to sandboxes", result["Message"])
             return result
 
@@ -229,14 +232,14 @@ class JobCleaningAgent(AgentModule):
             if not res["OK"]:
                 self.log.error("No DN found", f"for {user}")
                 return res
-            wmsClient = WMSClient(useCertificates=True, delegatedDN=res["Value"][0], delegatedGroup=ownerGroup)
             if remove:
+                wmsClient = WMSClient(useCertificates=True, delegatedDN=res["Value"][0], delegatedGroup=ownerGroup)
                 result = wmsClient.removeJob(jobsList)
             else:
-                result = wmsClient.deleteJob(jobsList)
+                result = kill_delete_jobs(RIGHT_DELETE, jobsList)
             if not result["OK"]:
                 self.log.error(
-                    "Could not {'remove' if remove else 'delete'} jobs",
+                    f"Could not {'remove' if remove else 'delete'} jobs",
                     f"for {user} : {ownerGroup} (n={len(jobsList)}) : {result['Message']}",
                 )
                 fail = True
@@ -293,7 +296,8 @@ class JobCleaningAgent(AgentModule):
         failed = {}
         successful = {}
 
-        result = JobMonitoringClient().getJobParameters(jobIDList, ["OutputSandboxLFN"])
+        jobIDs = [int(jobID) for jobID in jobIDList]
+        result = getJobParameters(jobIDs, "OutputSandboxLFN")
         if not result["OK"]:
             return result
         osLFNDict = result["Value"]

DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py

@@ -1,4 +1,4 @@
-""" This agent syncs CS and pilot files to a web server of your choice
+"""This agent syncs CS and pilot files to a web server of your choice
 
 .. literalinclude:: ../ConfigTemplate.cfg
   :start-after: ##BEGIN PilotSyncAgent
@@ -7,6 +7,7 @@
   :caption: PilotsSyncAgent options
 
 """
+
 import os
 import json
 import shutil
@@ -38,8 +39,8 @@ class PilotSyncAgent(AgentModule):
         self.workingDirectory = self.am_getOption("WorkDirectory")
         self.saveDir = self.am_getOption("SaveDirectory", self.saveDir)
         self.uploadLocations = self.am_getOption("UploadLocations", self.uploadLocations)
-        includeMasterCS = self.am_getOption("IncludeMasterCS", self.includeMasterCS)
-        if isinstance(includeMasterCS, str) and includeMasterCS.lower() in ["n", "no", "false"]:
+        self.includeMasterCS = self.am_getOption("IncludeMasterCS", self.includeMasterCS)
+        if isinstance(self.includeMasterCS, str) and self.includeMasterCS.lower() in ["n", "no", "false"]:
             self.includeMasterCS = False
 
         self.certAndKeyLocation = getHostCertificateAndKeyLocation()

DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py

@@ -12,16 +12,14 @@
 import hashlib
 import json
 import os
-from pathlib import Path
 import random
 import shutil
 import sys
-from collections import defaultdict
 import time
+from collections import defaultdict
+from pathlib import Path
 
-from diraccfg import CFG
-
-from DIRAC import gConfig, S_OK, S_ERROR
+from DIRAC import S_ERROR, S_OK, gConfig
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueues
 from DIRAC.Core.Utilities import DErrno
@@ -30,6 +28,7 @@ from DIRAC.Core.Utilities.Proxy import executeWithUserProxy
 from DIRAC.Core.Utilities.Version import getVersion
 from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
 from DIRAC.Resources.Computing import ComputingElement
+from DIRAC.WorkloadManagementSystem.Agent.JobAgent import JobAgent
 from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus, JobStatus, PilotStatus
 from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
 from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
@@ -41,11 +40,10 @@ from DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapperUtilities import (
     resolveInputData,
     transferInputSandbox,
 )
-from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved
-from DIRAC.WorkloadManagementSystem.Agent.JobAgent import JobAgent
-from DIRAC.WorkloadManagementSystem.Utilities.Utils import createJobWrapper
 from DIRAC.WorkloadManagementSystem.private.ConfigHelper import findGenericPilotCredentials
+from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
 from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved
+from DIRAC.WorkloadManagementSystem.Utilities.Utils import createJobWrapper
 
 MAX_JOBS_MANAGED = 100
 
@@ -230,6 +228,12 @@ class PushJobAgent(JobAgent):
             return result
         pilotProxy = result["Value"]
 
+        # Dump the proxy to a file to get DiracX token (it's later used by DiracX)
+        result = gProxyManager.dumpProxyToFile(pilotProxy)
+        if not result["OK"]:
+            return result
+        os.environ["X509_USER_PROXY"] = result["Value"]
+
         for queueName, queueDictionary in queueDictItems:
             # Make sure there is no problem with the queue before trying to submit
             if not self._allowedToSubmit(queueName):
@@ -281,7 +285,6 @@ class PushJobAgent(JobAgent):
                 jobGroup = matcherInfo["Group"]
                 owner = matcherInfo["Owner"]
                 ceDict = matcherInfo["CEDict"]
-                matchTime = matcherInfo["matchTime"]
 
                 optimizerParams = {}
                 for key in matcherInfo:
@@ -309,9 +312,6 @@ class PushJobAgent(JobAgent):
                 self.log.verbose("Job request successful: \n", jobRequest["Value"])
                 self.log.info("Received", f"JobID={jobID}, JobType={jobType}, Owner={owner}, JobGroup={jobGroup}")
 
-                self.jobs[jobID]["JobReport"].setJobParameter(
-                    par_name="MatcherServiceTime", par_value=str(matchTime), sendFlag=False
-                )
                 self.jobs[jobID]["JobReport"].setJobStatus(
                     status=JobStatus.MATCHED, minorStatus="Job Received by Agent", sendFlag=False
                 )
@@ -734,7 +734,7 @@ class PushJobAgent(JobAgent):
             return S_OK()
 
         # Get their parameters
-        if not (result := self.jobMonitoring.getJobParameters(jobs, ["GridCE", "TaskID", "Stamp"]))["OK"]:
+        if not (result := getJobParameters(jobs, ["GridCE", "TaskID", "Stamp"]))["OK"]:
             self.log.error("Failed to get the list of taskIDs", result["Message"])
             return result
 

DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -1,4 +1,4 @@
-"""  The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.
+"""The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.
 
 .. literalinclude:: ../ConfigTemplate.cfg
   :start-after: ##BEGIN SiteDirector
@@ -7,6 +7,7 @@
   :caption: SiteDirector options
 
 """
+
 import datetime
 import os
 from collections import defaultdict
@@ -14,7 +15,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any
 
 import DIRAC
-from DIRAC import S_ERROR, S_OK, gConfig
+from DIRAC import S_ERROR, S_OK
 from DIRAC.AccountingSystem.Client.DataStoreClient import gDataStoreClient
 from DIRAC.AccountingSystem.Client.Types.Pilot import Pilot as PilotAccounting
 from DIRAC.AccountingSystem.Client.Types.PilotSubmission import (
@@ -147,10 +148,10 @@ class SiteDirector(AgentModule):
             self.sendSubmissionAccounting = True
 
         # Get the site description dictionary
-        siteNames = self.am_getOption("Site", [])
-        ceTypes = self.am_getOption("CETypes", [])
-        ces = self.am_getOption("CEs", [])
-        tags = self.am_getOption("Tags", [])
+        siteNames = self.am_getOption("Site")
+        ceTypes = self.am_getOption("CETypes")
+        ces = self.am_getOption("CEs")
+        tags = self.am_getOption("Tags")
 
         # Display options used
         self.log.always("VO:", self.vo)
@@ -168,7 +169,7 @@ class SiteDirector(AgentModule):
         self.log.always("MaxPilotsToSubmit:", self.maxPilotsToSubmit)
 
         # Build the dictionary of queues that are going to be used: self.queueDict
-        if not (result := self._buildQueueDict(siteNames, ceTypes, ces, tags))["OK"]:
+        if not (result := self._buildQueueDict(siteNames, ces, ceTypes, tags))["OK"]:
             return result
 
         # Stop the execution if there is no usable queue
@@ -229,12 +230,8 @@ class SiteDirector(AgentModule):
             site = self.queueDict[queueName]["Site"]
             ce = self.queueDict[queueName]["CEName"]
 
-            # Check the status of the Site
-            if site in siteMaskList:
-                continue
-
-            # Check the status of the CE (only for RSS=Active)
-            if ce not in ceMaskList:
+            # Check the status of the Site and CE
+            if site in siteMaskList and ce in ceMaskList:
                 continue
 
             self.log.warn("Queue not considered because not usable:", queueName)
@@ -580,7 +577,7 @@ class SiteDirector(AgentModule):
             pilotOptions = []
 
         pilotOptions = " ".join(pilotOptions)
-        self.log.verbose(f"pilotOptions: {pilotOptions}")
+        self.log.verbose(f"{pilotOptions=}")
 
         # if a global workingDirectory is defined for the CEType (like HTCondor)
         # use it (otherwise the __cleanup done by HTCondor will be in the wrong folder !)
@@ -624,6 +621,11 @@ class SiteDirector(AgentModule):
         else:
             self.log.info("DIRAC project will be installed by pilots")
 
+        # Architecture script to use
+        architectureScript = opsHelper.getValue("Pilot/ArchitectureScript", "")
+        if architectureScript:
+            pilotOptions.append(f"--architectureScript={architectureScript}")
+
         # Preinstalled environment or list of CVMFS locations defined ?
         preinstalledEnv = opsHelper.getValue("Pilot/PreinstalledEnv", "")
         preinstalledEnvPrefix = opsHelper.getValue("Pilot/PreinstalledEnvPrefix", "")
@@ -695,6 +697,8 @@ class SiteDirector(AgentModule):
         :returns: file name of the pilot wrapper created
         """
 
+        pilotFilesCompressedEncodedDict = None
+
         try:
             pilotFilesCompressedEncodedDict = getPilotFilesCompressedEncodedDict([], proxy)
         except Exception as be:

DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py

@@ -14,18 +14,18 @@ import datetime
 from DIRAC import S_ERROR, S_OK, gConfig
 from DIRAC.AccountingSystem.Client.Types.Job import Job
 from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath
-from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Utilities import DErrno
 from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 from DIRAC.Core.Utilities.TimeUtilities import fromString, second, toEpoch
 from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus, JobStatus
-from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient
-from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
-from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
-from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
+from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
+from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_KILL
+from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
+from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
+from DIRAC.WorkloadManagementSystem.Utilities.Utils import rescheduleJobs
 
 
 class StalledJobAgent(AgentModule):
@@ -235,7 +235,7 @@ class StalledJobAgent(AgentModule):
         # Set the jobs Failed, send them a kill signal in case they are not really dead
         # and send accounting info
         if setFailed:
-            res = self._sendKillCommand(jobID)
+            res = kill_delete_jobs(RIGHT_KILL, [jobID], nonauthJobList=[], force=True)
             if not res["OK"]:
                 self.log.error("Failed to kill job", jobID)
 
@@ -254,15 +254,15 @@ class StalledJobAgent(AgentModule):
 
     def _getJobPilotStatus(self, jobID):
         """Get the job pilot status."""
-        result = JobMonitoringClient().getJobParameter(jobID, "Pilot_Reference")
+        result = getJobParameters([jobID], "Pilot_Reference")
         if not result["OK"]:
             return result
-        pilotReference = result["Value"].get("Pilot_Reference", "Unknown")
-        if pilotReference == "Unknown":
+        pilotReference = result["Value"].get("Pilot_Reference")
+        if not pilotReference:
             # There is no pilot reference, hence its status is unknown
             return S_OK("NoPilot")
 
-        result = PilotManagerClient().getPilotInfo(pilotReference)
+        result = PilotAgentsDB().getPilotInfo(pilotReference)
         if not result["OK"]:
             if DErrno.cmpError(result, DErrno.EWMSNOPILOT):
                 self.log.warn("No pilot found", f"for job {jobID}: {result['Message']}")
@@ -389,11 +389,11 @@ class StalledJobAgent(AgentModule):
             if lastHeartBeatTime is not None and lastHeartBeatTime > endTime:
                 endTime = lastHeartBeatTime
 
-            result = JobMonitoringClient().getJobParameter(jobID, "CPUNormalizationFactor")
-            if not result["OK"] or not result["Value"]:
+            result = getJobParameters([jobID], "CPUNormalizationFactor")
+            if not result["OK"] or not result["Value"] or not result["Value"].get("CPUNormalizationFactor"):
                 self.log.error(
                     "Error getting Job Parameter CPUNormalizationFactor, setting 0",
-                    result.get("Message", "No such value"),
+                    result.get("Message"),
                 )
                 cpuNormalization = 0.0
             else:
@@ -518,8 +518,7 @@ class StalledJobAgent(AgentModule):
         return startTime, endTime
 
     def _kickStuckJobs(self):
-        """Reschedule jobs stuck in initialization status Rescheduled,
-        Matched."""
+        """Reschedule jobs stuck in initialization status Rescheduled, Matched."""
 
         message = ""
 
@@ -530,17 +529,12 @@ class StalledJobAgent(AgentModule):
             return result
 
         jobIDs = result["Value"]
-        jobManagerClient = JobManagerClient()
         if jobIDs:
             self.log.info(f"Rescheduling {len(jobIDs)} jobs stuck in {JobStatus.MATCHED} status")
-            result = jobManagerClient.rescheduleJob(jobIDs)
+            result = rescheduleJobs(jobIDs)
             if not result["OK"]:
                 message = f"Failed to reschedule jobs stuck in {JobStatus.MATCHED} status"
                 message += "\n" + result["Message"]
-                if "InvalidJobIDs" in result:
-                    message += "\n" + "\tInvalid job IDs: " + str(result["InvalidJobIDs"])
-                if "NonauthorizedJobIDs" in result:
-                    message += "\n" + "\tNon authorized job IDs: " + str(result["NonauthorizedJobIDs"])
 
         checkTime = datetime.datetime.utcnow() - self.rescheduledTime * second
         result = self.jobDB.selectJobs({"Status": JobStatus.RESCHEDULED}, older=checkTime)
@@ -550,18 +544,14 @@ class StalledJobAgent(AgentModule):
 
         jobIDs = result["Value"]
         if jobIDs:
-            self.log.info(f"Rescheduling {len(jobIDs)} jobs stuck in Rescheduled status")
-            result = jobManagerClient.rescheduleJob(jobIDs)
+            self.log.info(f"Rescheduling {len(jobIDs)} jobs stuck in {JobStatus.RESCHEDULED} status")
+            result = rescheduleJobs(jobIDs)
             if not result["OK"]:
                 message = f"Failed to reschedule jobs stuck in {JobStatus.RESCHEDULED} status"
                 message += "\n" + result["Message"]
-                if "InvalidJobIDs" in result:
-                    message += "\n" + "\tInvalid job IDs: " + str(result["InvalidJobIDs"])
-                if "NonauthorizedJobIDs" in result:
-                    message += "\n" + "\tNon authorized job IDs: " + str(result["NonauthorizedJobIDs"])
 
         if message:
-            return S_ERROR(message)
+            self.log.error(message)
         return S_OK()
 
     def _failSubmittingJobs(self):
@@ -584,26 +574,3 @@ class StalledJobAgent(AgentModule):
                 continue
 
         return S_OK()
-
-    def _sendKillCommand(self, job):
-        """Send a kill signal to the job such that it cannot continue running.
-
-        :param int job: ID of job to send kill command
-        """
-
-        res = self.jobDB.getJobAttribute(job, "Owner")
-        if not res["OK"]:
-            return res
-        owner = res["Value"]
-
-        res = self.jobDB.getJobAttribute(job, "OwnerGroup")
-        if not res["OK"]:
-            return res
-        ownerGroup = res["Value"]
-
-        wmsClient = WMSClient(
-            useCertificates=True,
-            delegatedDN=getDNForUsername(owner)["Value"][0] if owner else None,
-            delegatedGroup=ownerGroup,
-        )
-        return wmsClient.killJob(job)

DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py

@@ -9,10 +9,11 @@
 """
 import datetime
 
-from DIRAC import S_ERROR, S_OK
+from DIRAC import S_ERROR, S_OK, gConfig
 from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient
 from DIRAC.AccountingSystem.Client.Types.WMSHistory import WMSHistory
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
+from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Utilities import TimeUtilities
 from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
@@ -77,6 +78,8 @@ class StatesAccountingAgent(AgentModule):
     def execute(self):
         """Main execution method"""
 
+        site_metadata = self._getSitesMetadata()
+
         # on the first iteration of the agent, do nothing in order to avoid double committing after a restart
         if self.am_getModuleParam("cyclesDone") == 0:
             self.log.notice("Skipping the first iteration of the agent")
@@ -131,6 +134,16 @@ class StatesAccountingAgent(AgentModule):
 
             for backend in self.datastores:
                 if backend.lower() == "monitoring":
+                    site_name = rD["Site"]
+                    if site_name not in site_metadata:
+                        self.log.warn(
+                            f"Site {site_name} not found in site metadata, using default values",
+                        )
+                        rD["Tier"] = "4"
+                        rD["Type"] = site_name.split(".")[0]
+                    else:
+                        rD["Tier"] = site_metadata[site_name]["Tier"]
+                        rD["Type"] = site_metadata[site_name]["Type"]
                     rD["timestamp"] = int(TimeUtilities.toEpochMilliSeconds(now))
                     self.datastores["Monitoring"].addRecord(rD)
 
@@ -154,3 +167,30 @@ class StatesAccountingAgent(AgentModule):
             self.log.verbose(f"Done committing WMSHistory to {backend} backend")
 
         return S_OK()
+
+    def _getSitesMetadata(self):
+        """Get the metadata for the sites"""
+        res = getSites()
+        if not res["OK"]:
+            return res
+        sites = res["Value"]
+        site_metadata = {}
+
+        for site in sites:
+            site_metadata[site] = {}
+
+            # Get the site metadata from the Configuration System
+            grid = site.split(".")[0]
+            res = gConfig.getOptionsDict(f"Resources/Sites/{grid}/{site}")
+            if not res["OK"]:
+                self.log.error("Failure getting options dict for site", f"{site}: {res['Message']}")
+                continue
+            siteInfoCS = res["Value"]
+
+            # The site tier is normally 1 or 2. Few VOs may define tier 3.
+            # If the tier is not defined, we assume it is 4, with 4 meaning "not pledged" (opportunistic).
+            site_metadata[site]["Tier"] = siteInfoCS.get("MoUTierLevel", "4")
+            # The site type is defined by the first part of the site name.
+            # It needs to be interpreted at the Monitoring side (e.g. in Grafana).
+            site_metadata[site]["Type"] = site.split(".")[0]
+        return site_metadata