DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py

@@ -4,17 +4,13 @@
     The following methods are available in the Service interface
 """
 
-import DIRAC.Core.Utilities.TimeUtilities as TimeUtilities
 from DIRAC import S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
-from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.DISET.RequestHandler import RequestHandler
 from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning
 from DIRAC.Core.Utilities.JEncode import strToIntDict
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
-from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
-from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
-from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_GET_INFO, JobPolicy
+from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
 
 
 class JobMonitoringHandlerMixin:
@@ -40,51 +36,12 @@ class JobMonitoringHandlerMixin:
             return result
         cls.elasticJobParametersDB = result["Value"](parentLogger=cls.log)
 
-        cls.pilotManager = PilotManagerClient()
         return S_OK()
 
     def initializeRequest(self):
         credDict = self.getRemoteCredentials()
         self.vo = credDict.get("VO", Registry.getVOForGroup(credDict["group"]))
 
-    @classmethod
-    def parseSelectors(cls, selectDict=None):
-        """Parse selectors before DB query
-
-        :param dict selectDict: selectors
-
-        :return: str, str, dict -- start/end date, selectors
-        """
-        selectDict = selectDict or {}
-
-        # Get time period
-        startDate = selectDict.get("FromDate", None)
-        if startDate:
-            del selectDict["FromDate"]
-        # For backward compatibility
-        if startDate is None:
-            startDate = selectDict.get("LastUpdate", None)
-            if startDate:
-                del selectDict["LastUpdate"]
-        endDate = selectDict.get("ToDate", None)
-        if endDate:
-            del selectDict["ToDate"]
-
-        # Provide JobID bound to a specific PilotJobReference
-        # There is no reason to have both PilotJobReference and JobID in selectDict
-        # If that occurs, use the JobID instead of the PilotJobReference
-        pilotJobRefs = selectDict.get("PilotJobReference")
-        if pilotJobRefs:
-            del selectDict["PilotJobReference"]
-            if not selectDict.get("JobID"):
-                for pilotJobRef in [pilotJobRefs] if isinstance(pilotJobRefs, str) else pilotJobRefs:
-                    res = cls.pilotManager.getPilotInfo(pilotJobRef)
-                    if res["OK"] and "Jobs" in res["Value"][pilotJobRef]:
-                        selectDict["JobID"] = selectDict.get("JobID", [])
-                        selectDict["JobID"].extend(res["Value"][pilotJobRef]["Jobs"])
-
-        return startDate, endDate, selectDict
-
     @classmethod
     def getJobsAttributes(cls, *args, **kwargs):
         """Utility function for unpacking"""
@@ -93,82 +50,6 @@ class JobMonitoringHandlerMixin:
             return res
         return S_OK(strToIntDict(res["Value"]))
 
-    ##############################################################################
-    types_getApplicationStates = []
-
-    @classmethod
-    def export_getApplicationStates(cls, condDict=None, older=None, newer=None):
-        """Return Distinct Values of ApplicationStatus job Attribute in WMS"""
-        return cls.jobDB.getDistinctJobAttributes("ApplicationStatus", condDict, older, newer)
-
-    ##############################################################################
-    types_getJobTypes = []
-
-    @classmethod
-    def export_getJobTypes(cls, condDict=None, older=None, newer=None):
-        """Return Distinct Values of JobType job Attribute in WMS"""
-        return cls.jobDB.getDistinctJobAttributes("JobType", condDict, older, newer)
-
-    ##############################################################################
-    types_getOwners = []
-
-    @classmethod
-    def export_getOwners(cls, condDict=None, older=None, newer=None):
-        """
-        Return Distinct Values of Owner job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("Owner", condDict, older, newer)
-
-    ##############################################################################
-    types_getOwnerGroup = []
-
-    @classmethod
-    def export_getOwnerGroup(cls):
-        """
-        Return Distinct Values of OwnerGroup from the JobDB
-        """
-        return cls.jobDB.getDistinctJobAttributes("OwnerGroup")
-
-    ##############################################################################
-    types_getJobGroups = []
-
-    @classmethod
-    def export_getJobGroups(cls, condDict=None, older=None, cutDate=None):
-        """
-        Return Distinct Values of ProductionId job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("JobGroup", condDict, older, newer=cutDate)
-
-    ##############################################################################
-    types_getSites = []
-
-    @classmethod
-    def export_getSites(cls, condDict=None, older=None, newer=None):
-        """
-        Return Distinct Values of Site job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("Site", condDict, older, newer)
-
-    ##############################################################################
-    types_getStates = []
-
-    @classmethod
-    def export_getStates(cls, condDict=None, older=None, newer=None):
-        """
-        Return Distinct Values of Status job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("Status", condDict, older, newer)
-
-    ##############################################################################
-    types_getMinorStates = []
-
-    @classmethod
-    def export_getMinorStates(cls, condDict=None, older=None, newer=None):
-        """
-        Return Distinct Values of Minor Status job Attribute in WMS
-        """
-        return cls.jobDB.getDistinctJobAttributes("MinorStatus", condDict, older, newer)
-
     ##############################################################################
     types_getJobs = []
 
@@ -189,36 +70,6 @@ class JobMonitoringHandlerMixin:
 
         return cls.jobDB.selectJobs(attrDict, newer=cutDate)
 
-    ##############################################################################
-    types_getCounters = [list]
-
-    @classmethod
-    def export_getCounters(cls, attrList, attrDict=None, cutDate=""):
-        """
-        Retrieve list of distinct attributes values from attrList
-        with attrDict as condition.
-        For each set of distinct values, count number of occurences.
-        Return a list. Each item is a list with 2 items, the list of distinct
-        attribute values and the counter
-        """
-
-        _, _, attrDict = cls.parseSelectors(attrDict)
-        return cls.jobDB.getCounters("Jobs", attrList, attrDict, newer=str(cutDate), timeStamp="LastUpdateTime")
-
-    ##############################################################################
-    types_getJobOwner = [int]
-
-    @classmethod
-    def export_getJobOwner(cls, jobID):
-        return cls.jobDB.getJobAttribute(jobID, "Owner")
-
-    ##############################################################################
-    types_getJobSite = [int]
-
-    @classmethod
-    def export_getJobSite(cls, jobID):
-        return cls.jobDB.getJobAttribute(jobID, "Site")
-
     ##############################################################################
     types_getJobJDL = [int, bool]
 
@@ -233,14 +84,6 @@ class JobMonitoringHandlerMixin:
     def export_getJobLoggingInfo(cls, jobID):
         return cls.jobLoggingDB.getJobLoggingInfo(jobID)
 
-    ##############################################################################
-    types_getJobsParameters = [[str, int, list], list]
-
-    @classmethod
-    @ignoreEncodeWarning
-    def export_getJobsParameters(cls, jobIDs, parameters):
-        return cls.getJobsAttributes(jobIDs, parameters)
-
     ##############################################################################
     types_getJobsStates = [[str, int, list]]
 
@@ -295,122 +138,6 @@ class JobMonitoringHandlerMixin:
     def export_getJobsSummary(cls, jobIDs):
         return cls.getJobsAttributes(jobIDs)
 
-    ##############################################################################
-    types_getJobPageSummaryWeb = [dict, list, int, int]
-
-    def export_getJobPageSummaryWeb(self, selectDict, sortList, startItem, maxItems, selectJobs=True):
-        """Get the summary of the job information for a given page in the
-        job monitor in a generic format
-        """
-
-        resultDict = {}
-
-        startDate, endDate, selectDict = self.parseSelectors(selectDict)
-
-        # initialize jobPolicy
-        credDict = self.getRemoteCredentials()
-        owner = credDict["username"]
-        ownerGroup = credDict["group"]
-        operations = Operations(group=ownerGroup)
-        globalJobsInfo = operations.getValue("/Services/JobMonitoring/GlobalJobsInfo", True)
-        jobPolicy = JobPolicy(owner, ownerGroup, globalJobsInfo)
-        jobPolicy.jobDB = self.jobDB
-        result = jobPolicy.getControlledUsers(RIGHT_GET_INFO)
-        if not result["OK"]:
-            return result
-        if not result["Value"]:
-            return S_ERROR(f"User and group combination has no job rights ({owner!r}, {ownerGroup!r})")
-        if result["Value"] != "ALL":
-            selectDict[("Owner", "OwnerGroup")] = result["Value"]
-
-        # Sorting instructions. Only one for the moment.
-        if sortList:
-            orderAttribute = sortList[0][0] + ":" + sortList[0][1]
-        else:
-            orderAttribute = None
-
-        result = self.jobDB.getCounters(
-            "Jobs", ["Status"], selectDict, newer=startDate, older=endDate, timeStamp="LastUpdateTime"
-        )
-        if not result["OK"]:
-            return result
-
-        statusDict = {}
-        nJobs = 0
-        for stDict, count in result["Value"]:
-            nJobs += count
-            statusDict[stDict["Status"]] = count
-
-        resultDict["TotalRecords"] = nJobs
-        if nJobs == 0:
-            return S_OK(resultDict)
-
-        resultDict["Extras"] = statusDict
-
-        if selectJobs:
-            iniJob = startItem
-            if iniJob >= nJobs:
-                return S_ERROR("Item number out of range")
-
-            result = self.jobDB.selectJobs(
-                selectDict, orderAttribute=orderAttribute, newer=startDate, older=endDate, limit=(maxItems, iniJob)
-            )
-            if not result["OK"]:
-                return result
-
-            summaryJobList = result["Value"]
-            if not globalJobsInfo:
-                validJobs, _invalidJobs, _nonauthJobs, _ownJobs = jobPolicy.evaluateJobRights(
-                    summaryJobList, RIGHT_GET_INFO
-                )
-                summaryJobList = validJobs
-
-            result = self.getJobsAttributes(summaryJobList)
-            if not result["OK"]:
-                return result
-
-            summaryDict = result["Value"]
-            # If no jobs can be selected after the properties check
-            if not summaryDict:
-                return S_OK(resultDict)
-
-            # Evaluate last sign of life time
-            for jobDict in summaryDict.values():
-                if not jobDict.get("HeartBeatTime") or jobDict["HeartBeatTime"] == "None":
-                    jobDict["LastSignOfLife"] = jobDict["LastUpdateTime"]
-                else:
-                    jobDict["LastSignOfLife"] = jobDict["HeartBeatTime"]
-
-            # prepare the standard structure now
-            # This should be faster than making a list of values()
-            for jobDict in summaryDict.values():
-                paramNames = list(jobDict)
-                break
-            records = [list(jobDict.values()) for jobDict in summaryDict.values()]
-
-            resultDict["ParameterNames"] = paramNames
-            resultDict["Records"] = records
-
-        return S_OK(resultDict)
-
-    ##############################################################################
-    types_getJobStats = [str, dict]
-
-    @classmethod
-    def export_getJobStats(cls, attribute, selectDict):
-        """Get job statistics distribution per attribute value with a given selection"""
-        startDate, endDate, selectDict = cls.parseSelectors(selectDict)
-        result = cls.jobDB.getCounters(
-            "Jobs", [attribute], selectDict, newer=startDate, older=endDate, timeStamp="LastUpdateTime"
-        )
-        if not result["OK"]:
-            return result
-        resultDict = {}
-        for cDict, count in result["Value"]:
-            resultDict[cDict[attribute]] = count
-
-        return S_OK(resultDict)
-
     ##############################################################################
     types_getJobParameter = [[str, int], str]
 
@@ -420,18 +147,11 @@ class JobMonitoringHandlerMixin:
         :param str/int jobID: one single Job ID
         :param str parName: one single parameter name
         """
-        res = self.elasticJobParametersDB.getJobParameters(int(jobID), self.vo, [parName])
+        res = getJobParameters([int(jobID)], parName, self.vo or "")
         if not res["OK"]:
             return res
         return S_OK(res["Value"].get(int(jobID), {}))
 
-    ##############################################################################
-    types_getJobOptParameters = [int]
-
-    @classmethod
-    def export_getJobOptParameters(cls, jobID):
-        return cls.jobDB.getJobOptParameters(jobID)
-
     ##############################################################################
     types_getJobParameters = [[str, int, list]]
 
@@ -439,41 +159,13 @@ class JobMonitoringHandlerMixin:
     def export_getJobParameters(self, jobIDs, parName=None):
         """
         :param str/int/list jobIDs: one single job ID or a list of them
-        :param str parName: one single parameter name, a list or None (meaning all of them)
+        :param str parName: one single parameter name, or None (meaning all of them)
         """
         if not isinstance(jobIDs, list):
             jobIDs = [jobIDs]
         jobIDs = [int(jobID) for jobID in jobIDs]
-        res = self.elasticJobParametersDB.getJobParameters(jobIDs, self.vo, parName)
-        if not res["OK"]:
-            return res
-        parameters = res["Value"]
 
-        # Need anyway to get also from JobDB, for those jobs with parameters registered in MySQL or in both backends
-        res = self.jobDB.getJobParameters(jobIDs, parName)
-        if not res["OK"]:
-            return res
-        parametersM = res["Value"]
-
-        # and now combine
-        final = dict(parametersM)
-        # if job in JobDB, update with parameters from ES if any
-        for jobID in final:
-            final[jobID].update(parameters.get(jobID, {}))
-        # if job in ES and not in JobDB, take ES
-        for jobID in parameters:
-            if jobID not in final:
-                final[jobID] = parameters[jobID]
-        return S_OK(final)
-
-    ##############################################################################
-    types_getAtticJobParameters = [int]
-
-    @classmethod
-    def export_getAtticJobParameters(cls, jobID, parameters=None, rescheduleCycle=-1):
-        if not parameters:
-            parameters = []
-        return cls.jobDB.getAtticJobParameters(jobID, parameters, rescheduleCycle)
+        return getJobParameters(jobIDs, parName, self.vo or "")
 
     ##############################################################################
     types_getJobAttributes = [int]
@@ -499,13 +191,6 @@ class JobMonitoringHandlerMixin:
 
         return cls.jobDB.getJobAttribute(jobID, attribute)
 
-    ##############################################################################
-    types_getSiteSummary = []
-
-    @classmethod
-    def export_getSiteSummary(cls):
-        return cls.jobDB.getSiteSummary()
-
     ##############################################################################
     types_getJobHeartBeatData = [int]
 
@@ -514,7 +199,7 @@ class JobMonitoringHandlerMixin:
         return cls.jobDB.getHeartBeatData(jobID)
 
     ##############################################################################
-    types_getInputData = [int]
+    types_getInputData = [(int, list)]
 
     @classmethod
     def export_getInputData(cls, jobID):
@@ -522,9 +207,6 @@ class JobMonitoringHandlerMixin:
         return cls.jobDB.getInputData(jobID)
 
 
-##############################################################################
-
-
 class JobMonitoringHandler(JobMonitoringHandlerMixin, RequestHandler):
     def initialize(self):
         return self.initializeRequest()

DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py

@@ -128,22 +128,6 @@ class JobStateUpdateHandlerMixin:
         """Allows the site attribute to be set for a job specified by its jobID."""
         return cls.jobDB.setJobAttribute(int(jobID), "Site", site)
 
-    ###########################################################################
-    types_setJobFlag = [[str, int], str]
-
-    @classmethod
-    def export_setJobFlag(cls, jobID, flag):
-        """Set job flag for job with jobID"""
-        return cls.jobDB.setJobAttribute(int(jobID), flag, "True")
-
-    ###########################################################################
-    types_unsetJobFlag = [[str, int], str]
-
-    @classmethod
-    def export_unsetJobFlag(cls, jobID, flag):
-        """Unset job flag for job with jobID"""
-        return cls.jobDB.setJobAttribute(int(jobID), flag, "False")
-
     ###########################################################################
     types_setJobApplicationStatus = [[str, int], str, str]
 

DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py

@@ -13,7 +13,6 @@ from DIRAC.WorkloadManagementSystem.Client import PilotStatus
 from DIRAC.WorkloadManagementSystem.Service.WMSUtilities import (
     getPilotCE,
     getPilotRef,
-    killPilotsInQueues,
     setPilotCredentials,
 )
 
@@ -94,10 +93,10 @@ class PilotManagerHandler(RequestHandler):
 
         result = self.pilotAgentsDB.getPilotInfo(pilotReference)
         if not result["OK"]:
-            self.log.error("Failed to get info for pilot", result["Message"])
+            self.log.error("Failed to get info for pilot", f"{pilotReference}: {result['Message']}")
             return S_ERROR("Failed to get info for pilot")
         if not result["Value"]:
-            self.log.warn("The pilot info is empty", pilotReference)
+            self.log.warn("The pilot info is empty for", pilotReference)
             return S_ERROR("Pilot info is empty")
 
         pilotDict = result["Value"][pilotReference]
@@ -106,11 +105,14 @@ class PilotManagerHandler(RequestHandler):
         # classic logs first, by default
         funcs = [self._getPilotOutput, self._getRemotePilotOutput]
         if remote:
+            self.log.info("Trying to retrieve output of pilot", f"{pilotReference} remotely first")
             funcs.reverse()
 
         result = funcs[0](pilotReference, pilotDict)
         if not result["OK"]:
-            self.log.warn("Pilot log retrieval failed (first attempt), remote ?", remote)
+            self.log.warn(
+                "Failed getting output for pilot", f"{pilotReference}. Will try another approach: {result['Message']}"
+            )
             result = funcs[1](pilotReference, pilotDict)
             return result
         else:
@@ -275,37 +277,6 @@ class PilotManagerHandler(RequestHandler):
 
         return cls.pilotAgentsDB.getPilotSummary(startdate, enddate)
 
-    ##############################################################################
-    types_getPilotMonitorWeb = [dict, list, int, int]
-
-    @classmethod
-    def export_getPilotMonitorWeb(cls, selectDict, sortList, startItem, maxItems):
-        """Get the summary of the pilot information for a given page in the
-        pilot monitor in a generic format
-        """
-
-        return cls.pilotAgentsDB.getPilotMonitorWeb(selectDict, sortList, startItem, maxItems)
-
-    ##############################################################################
-    types_getPilotMonitorSelectors = []
-
-    @classmethod
-    def export_getPilotMonitorSelectors(cls):
-        """Get all the distinct selector values for the Pilot Monitor web portal page"""
-
-        return cls.pilotAgentsDB.getPilotMonitorSelectors()
-
-    ##############################################################################
-    types_getPilotSummaryWeb = [dict, list, int, int]
-
-    @classmethod
-    def export_getPilotSummaryWeb(cls, selectDict, sortList, startItem, maxItems):
-        """Get the summary of the pilot information for a given page in the
-        pilot monitor in a generic format
-        """
-
-        return cls.pilotAgentsDB.getPilotSummaryWeb(selectDict, sortList, startItem, maxItems)
-
     ##############################################################################
     types_getGroupedPilotSummary = [list]
 
@@ -332,35 +303,6 @@ class PilotManagerHandler(RequestHandler):
 
         return cls.pilotAgentsDB.getPilotInfo(pilotID=result["Value"])
 
-    ##############################################################################
-    types_killPilot = [[str, list]]
-
-    @classmethod
-    def export_killPilot(cls, pilotRefList):
-        """Kill the specified pilots"""
-        # Make a list if it is not yet
-        if isinstance(pilotRefList, str):
-            pilotRefList = [pilotRefList]
-
-        # Regroup pilots per site
-        pilotRefDict = {}
-        for pilotReference in pilotRefList:
-            result = cls.pilotAgentsDB.getPilotInfo(pilotReference)
-            if not result["OK"] or not result["Value"]:
-                return S_ERROR(f"Failed to get info for pilot {pilotReference}")
-
-            pilotDict = result["Value"][pilotReference]
-            queue = "@@@".join(
-                [pilotDict["VO"], pilotDict["GridSite"], pilotDict["DestinationSite"], pilotDict["Queue"]]
-            )
-            gridType = pilotDict["GridType"]
-            pilotRefDict.setdefault(queue, {})
-            pilotRefDict[queue].setdefault("PilotList", [])
-            pilotRefDict[queue]["PilotList"].append(pilotReference)
-            pilotRefDict[queue]["GridType"] = gridType
-
-        return killPilotsInQueues(pilotRefDict)
-
     ##############################################################################
     types_setJobForPilot = [[str, int], str]
 
@@ -416,44 +358,6 @@ class PilotManagerHandler(RequestHandler):
         return cls.pilotAgentsDB.countPilots(condDict, older, newer, timeStamp)
 
     ##########################################################################################
-    types_getCounters = [str, list, dict]
-
-    @classmethod
-    def export_getCounters(cls, table, keys, condDict, newer=None, timeStamp="SubmissionTime"):
-        """Set the pilot agent status"""
-
-        return cls.pilotAgentsDB.getCounters(table, keys, condDict, newer=newer, timeStamp=timeStamp)
-
-    ##############################################################################
-    types_getPilotStatistics = [str, dict]
-
-    @classmethod
-    def export_getPilotStatistics(cls, attribute, selectDict):
-        """Get pilot statistics distribution per attribute value with a given selection"""
-
-        startDate = selectDict.get("FromDate", None)
-        if startDate:
-            del selectDict["FromDate"]
-
-        if startDate is None:
-            startDate = selectDict.get("LastUpdate", None)
-            if startDate:
-                del selectDict["LastUpdate"]
-        endDate = selectDict.get("ToDate", None)
-        if endDate:
-            del selectDict["ToDate"]
-
-        result = cls.pilotAgentsDB.getCounters(
-            "PilotAgents", [attribute], selectDict, newer=startDate, older=endDate, timeStamp="LastUpdateTime"
-        )
-        statistics = {}
-        if result["OK"]:
-            for status, count in result["Value"]:
-                statistics[status[attribute]] = count
-
-        return S_OK(statistics)
-
-    ##############################################################################
     types_deletePilots = [[list, str, int]]
 
     @classmethod