panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jedidog/AtlasTaskWithholderWatchDog.py

@@ -0,0 +1,245 @@
+import os
+import socket
+import sys
+import traceback
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jedibrokerage import AtlasBrokerUtils
+from pandajedi.jediconfig import jedi_config
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+from pandaserver.dataservice import DataServiceUtils
+
+from .WatchDogBase import WatchDogBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# task withholder watchdog for ATLAS
+class AtlasTaskWithholderWatchDog(WatchDogBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        WatchDogBase.__init__(self, taskBufferIF, ddmIF)
+        self.pid = f"{socket.getfqdn().split('.')[0]}-{os.getpid()}-dog"
+        # self.cronActions = {'forPrestage': 'atlas_prs'}
+        self.vo = "atlas"
+        self.prodSourceLabelList = ["managed"]
+        # call refresh
+        self.refresh()
+
+    # get process lock
+    def _get_lock(self):
+        return self.taskBufferIF.lockProcess_JEDI(
+            vo=self.vo,
+            prodSourceLabel="managed",
+            cloud=None,
+            workqueue_id=None,
+            resource_name=None,
+            component="AtlasTaskWithholderWatchDog",
+            pid=self.pid,
+            timeLimit=5,
+        )
+
+    # refresh information stored in the instance
+    def refresh(self):
+        # work queue mapper
+        self.workQueueMapper = self.taskBufferIF.getWorkQueueMap()
+        # site mapper
+        self.siteMapper = self.taskBufferIF.get_site_mapper()
+        # all sites
+        allSiteList = []
+        for siteName, tmpSiteSpec in self.siteMapper.siteSpecList.items():
+            # if tmpSiteSpec.type == 'analysis' or tmpSiteSpec.is_grandly_unified():
+            allSiteList.append(siteName)
+        self.allSiteList = allSiteList
+
+    # get map of site to list of RSEs and blacklisted RSEs
+    def get_site_rse_map_and_blacklisted_rse_set(self, prod_source_label):
+        site_rse_map = {}
+        blacklisted_rse_set = set()
+        for tmpPseudoSiteName in self.allSiteList:
+            tmpSiteSpec = self.siteMapper.getSite(tmpPseudoSiteName)
+            tmpSiteName = tmpSiteSpec.get_unified_name()
+            scope_input, scope_output = DataServiceUtils.select_scope(tmpSiteSpec, prod_source_label, prod_source_label)
+            try:
+                tmp_ddm_spec = tmpSiteSpec.ddm_endpoints_input[scope_input]
+                endpoint_name = tmpSiteSpec.ddm_input[scope_input]
+                endpoint_token_map = tmp_ddm_spec.getTokenMap("input")
+                tmp_endpoint = tmp_ddm_spec.getEndPoint(endpoint_name)
+            except KeyError:
+                continue
+            else:
+                # fill site rse map
+                site_rse_map[tmpSiteName] = list(endpoint_token_map.values())
+                # blacklisted rse
+                if tmp_endpoint is not None and tmp_endpoint["blacklisted"] == "Y":
+                    blacklisted_rse_set.add(endpoint_name)
+        # return
+        return site_rse_map, blacklisted_rse_set
+
+    # get busy sites
+    def get_busy_sites(self, gshare, cutoff):
+        busy_sites_list = []
+        # get global share
+        tmpSt, jobStatPrioMap = self.taskBufferIF.getJobStatisticsByGlobalShare(self.vo)
+        if not tmpSt:
+            # got nothing...
+            return busy_sites_list
+        for tmpPseudoSiteName in self.allSiteList:
+            tmpSiteSpec = self.siteMapper.getSite(tmpPseudoSiteName)
+            tmpSiteName = tmpSiteSpec.get_unified_name()
+            # get nQueue and nRunning
+            nRunning = AtlasBrokerUtils.getNumJobs(jobStatPrioMap, tmpSiteName, "running", workQueue_tag=gshare)
+            nQueue = 0
+            for jobStatus in ["defined", "assigned", "activated", "starting"]:
+                nQueue += AtlasBrokerUtils.getNumJobs(jobStatPrioMap, tmpSiteName, jobStatus, workQueue_tag=gshare)
+            # busy sites
+            if nQueue > max(cutoff, nRunning * 2):
+                busy_sites_list.append(tmpSiteName)
+        # return
+        return busy_sites_list
+
+    # # handle waiting jobs
+    # def do_make_tasks_pending(self, task_list):
+    #     tmpLog = MsgWrapper(logger, 'do_make_tasks_pending')
+    #     tmpLog.debug('start')
+    #     # check every x min
+    #     checkInterval = 20
+    #     # make task pending
+    #     for taskID, pending_reason in task_list:
+    #         tmpLog = MsgWrapper(logger, '< #ATM #KV do_make_tasks_pending jediTaskID={0}>'.format(taskID))
+    #         retVal = self.taskBufferIF.makeTaskPending_JEDI(taskID, reason=pending_reason)
+    #         tmpLog.debug('done with {0}'.format(retVal))
+
+    # set tasks to be pending due to condition of data locality
+    def do_for_data_locality(self):
+        tmp_log = MsgWrapper(logger)
+        # refresh
+        self.refresh()
+        # list of resource type
+        # resource_type_list = [ rt.resource_name for rt in self.taskBufferIF.load_resource_types() ]
+        # loop
+        for prod_source_label in self.prodSourceLabelList:
+            # site-rse map and blacklisted rses
+            site_rse_map, blacklisted_rse_set = self.get_site_rse_map_and_blacklisted_rse_set(prod_source_label)
+            tmp_log.debug(f"Found {len(blacklisted_rse_set)} blacklisted RSEs : {','.join(list(blacklisted_rse_set))}")
+            # parameter from GDP config
+            upplimit_ioIntensity = self.taskBufferIF.getConfigValue("task_withholder", f"LIMIT_IOINTENSITY_{prod_source_label}", "jedi", self.vo)
+            lowlimit_currentPriority = self.taskBufferIF.getConfigValue("task_withholder", f"LIMIT_PRIORITY_{prod_source_label}", "jedi", self.vo)
+            if upplimit_ioIntensity is None:
+                upplimit_ioIntensity = 999999
+            if lowlimit_currentPriority is None:
+                lowlimit_currentPriority = -999999
+            upplimit_ioIntensity = max(upplimit_ioIntensity, 100)
+            # get work queue for gshare
+            work_queue_list = self.workQueueMapper.getAlignedQueueList(self.vo, prod_source_label)
+            # loop over work queue
+            for work_queue in work_queue_list:
+                gshare = work_queue.queue_name
+                # get cutoff
+                cutoff = self.taskBufferIF.getConfigValue("jobbroker", f"NQUEUELIMITSITE_{gshare}", "jedi", self.vo)
+                if not cutoff:
+                    cutoff = 20
+                # busy sites
+                busy_sites_list = self.get_busy_sites(gshare, cutoff)
+                # rses of busy sites
+                busy_rses = set()
+                for site in busy_sites_list:
+                    try:
+                        busy_rses.update(set(site_rse_map[site]))
+                    except KeyError:
+                        continue
+                # make sql parameters of rses
+                to_exclude_rses = list(busy_rses | blacklisted_rse_set)
+                if not to_exclude_rses:
+                    continue
+                rse_params_list = []
+                rse_params_map = {}
+                for j, rse in enumerate(to_exclude_rses):
+                    rse_param = f":rse_{j + 1}"
+                    rse_params_list.append(rse_param)
+                    rse_params_map[rse_param] = rse
+                rse_params_str = ",".join(rse_params_list)
+                # sql
+                sql_query = (
+                    "SELECT t.jediTaskID "
+                    "FROM {jedi_schema}.JEDI_Tasks t "
+                    "WHERE t.status IN ('ready','running','scouting') AND t.lockedBy IS NULL "
+                    "AND t.gshare=:gshare "
+                    "AND t.ioIntensity>=:ioIntensity AND t.currentPriority<:currentPriority "
+                    "AND EXISTS ( "
+                    "SELECT * FROM {jedi_schema}.JEDI_Datasets d "
+                    "WHERE d.jediTaskID=t.jediTaskID "
+                    "AND d.type='input' "
+                    ") "
+                    "AND NOT EXISTS ( "
+                    "SELECT * FROM {jedi_schema}.JEDI_Dataset_Locality dl "
+                    "WHERE dl.jediTaskID=t.jediTaskID "
+                    "AND dl.rse NOT IN ({rse_params_str}) "
+                    ") "
+                ).format(jedi_schema=jedi_config.db.schemaJEDI, rse_params_str=rse_params_str)
+                # params map
+                params_map = {
+                    ":gshare": gshare,
+                    ":ioIntensity": upplimit_ioIntensity,
+                    ":currentPriority": lowlimit_currentPriority,
+                }
+                params_map.update(rse_params_map)
+                # pending reason
+                reason = (
+                    "no local input data, ioIntensity>={ioIntensity}, currentPriority<{currentPriority},"
+                    "nQueue>max({cutOff},nRunning*2) at all sites where the task can run".format(
+                        ioIntensity=upplimit_ioIntensity, currentPriority=lowlimit_currentPriority, cutOff=cutoff
+                    )
+                )
+                # set pending
+                dry_run = False
+                if dry_run:
+                    dry_sql_query = (
+                        "SELECT t.jediTaskID "
+                        "FROM {jedi_schema}.JEDI_Tasks t "
+                        "WHERE t.status IN ('ready','running','scouting') AND t.lockedBy IS NULL "
+                        "AND t.gshare=:gshare "
+                        "AND t.ioIntensity>=:ioIntensity AND t.currentPriority<:currentPriority "
+                        "AND EXISTS ( "
+                        "SELECT * FROM {jedi_schema}.JEDI_Datasets d "
+                        "WHERE d.jediTaskID=t.jediTaskID "
+                        "AND d.type='input' "
+                        ") "
+                        "AND NOT EXISTS ( "
+                        "SELECT * FROM {jedi_schema}.JEDI_Dataset_Locality dl "
+                        "WHERE dl.jediTaskID=t.jediTaskID "
+                        "AND dl.rse NOT IN ({rse_params_str}) "
+                        ") "
+                    ).format(jedi_schema=jedi_config.db.schemaJEDI, rse_params_str=rse_params_str)
+                    res = self.taskBufferIF.querySQL(dry_sql_query, params_map)
+                    n_tasks = 0 if res is None else len(res)
+                    if n_tasks > 0:
+                        result = [x[0] for x in res]
+                        tmp_log.debug(f'[dry run] gshare: {gshare:<16} {n_tasks:>5} tasks would be pending : {result} ; reason="{reason}" ')
+                else:
+                    n_tasks = self.taskBufferIF.queryTasksToBePending_JEDI(sql_query, params_map, reason)
+                    if n_tasks is not None and n_tasks > 0:
+                        tmp_log.info(f'gshare: {gshare:<16} {str(n_tasks):>5} tasks got pending ; reason="{reason}" ')
+
+    # main
+    def doAction(self):
+        try:
+            # get logger
+            origTmpLog = MsgWrapper(logger)
+            origTmpLog.debug("start")
+            # lock
+            got_lock = self._get_lock()
+            if not got_lock:
+                origTmpLog.debug("locked by another process. Skipped")
+                return self.SC_SUCCEEDED
+            origTmpLog.debug("got lock")
+            # make tasks pending under certain conditions
+            self.do_for_data_locality()
+        except Exception:
+            errtype, errvalue = sys.exc_info()[:2]
+            err_str = traceback.format_exc()
+            origTmpLog.error(f"failed with {errtype} {errvalue} ; {err_str}")
+        # return
+        origTmpLog.debug("done")
+        return self.SC_SUCCEEDED

pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py

@@ -0,0 +1,63 @@
+import sys
+import traceback
+
+# logger
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+from pandaserver.workflow.workflow_core import WorkflowInterface
+
+from .WatchDogBase import WatchDogBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+class AtlasWorkflowManagerWatchDog(WatchDogBase):
+    """
+    Workflow manager watchdog for ATLAS
+    """
+
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        WatchDogBase.__init__(self, taskBufferIF, ddmIF)
+        self.vo = "atlas"
+        self.workflow_interface = WorkflowInterface(taskBufferIF)
+
+    def doProcessWorkflows(self):
+        """
+        Action to process active workflows
+        """
+        tmpLog = MsgWrapper(logger, " #ATM #KV doProcessWorkflows")
+        tmpLog.debug("start")
+        try:
+            # watchdog lock
+            got_lock = self.get_process_lock("AtlasWFManagerDog.doProcessWorkflows", timeLimit=5)
+            if not got_lock:
+                tmpLog.debug("locked by another watchdog process. Skipped")
+                return
+            tmpLog.debug("got watchdog lock")
+            # process active workflows
+            stats = self.workflow_interface.process_active_workflows()
+            tmpLog.info(f"processed workflows: {stats}")
+            # done
+            tmpLog.debug("done")
+        except Exception:
+            errtype, errvalue = sys.exc_info()[:2]
+            tmpLog.error(f"failed with {errtype} {errvalue} {traceback.format_exc()}")
+
+    # main
+    def doAction(self):
+        try:
+            # get logger
+            origTmpLog = MsgWrapper(logger)
+            origTmpLog.debug("start")
+            # clean up
+            # check
+            # process workflows
+            self.doProcessWorkflows()
+        except Exception:
+            errtype, errvalue = sys.exc_info()[:2]
+            origTmpLog.error(f"failed with {errtype} {errvalue}")
+        # return
+        origTmpLog.debug("done")
+        return self.SC_SUCCEEDED

pandajedi/jedidog/GenWatchDog.py

@@ -0,0 +1,21 @@
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+
+from .TypicalWatchDogBase import TypicalWatchDogBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# watchdog for general purpose
+class GenWatchDog(TypicalWatchDogBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        TypicalWatchDogBase.__init__(self, taskBufferIF, ddmIF)
+
+    # main
+    def doAction(self):
+        tmpLog = MsgWrapper(logger)
+        tmpLog.debug("start")
+        tmpLog.debug("done")
+        return self.SC_SUCCEEDED

pandajedi/jedidog/JumboWatchDog.py

@@ -0,0 +1,147 @@
+import os
+import socket
+import sys
+import traceback
+
+from pandaserver.taskbuffer.JediTaskSpec import JediTaskSpec
+
+
+# watchdog to take actions for jumbo jobs
+class JumboWatchDog:
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF, log, vo, prodSourceLabel):
+        self.taskBufferIF = taskBufferIF
+        self.ddmIF = ddmIF
+        self.pid = f"{socket.getfqdn().split('.')[0]}-{os.getpid()}_{os.getpgrp()}-jumbo"
+        self.log = log
+        self.vo = vo
+        self.prodSourceLabel = prodSourceLabel
+        self.component = "JumboWatchDog"
+        self.dryRun = True
+
+    # main
+    def run(self):
+        try:
+            # get process lock
+            locked = self.taskBufferIF.lockProcess_JEDI(
+                vo=self.vo,
+                prodSourceLabel=self.prodSourceLabel,
+                cloud=None,
+                workqueue_id=None,
+                resource_name=None,
+                component=self.component,
+                pid=self.pid,
+                timeLimit=10,
+            )
+            if not locked:
+                self.log.debug(f"component={self.component} skipped since locked by another")
+                return
+            # get parameters for conversion
+            self.log.debug(f"component={self.component} start")
+            maxTasks = self.taskBufferIF.getConfigValue(self.component, "JUMBO_MAX_TASKS", "jedi", self.vo)
+            if maxTasks is None:
+                maxTasks = 1
+            nEventsToDisable = self.taskBufferIF.getConfigValue(self.component, "JUMBO_MIN_EVENTS_DISABLE", "jedi", self.vo)
+            if nEventsToDisable is None:
+                nEventsToDisable = 100000
+            nEventsToEnable = self.taskBufferIF.getConfigValue(self.component, "JUMBO_MIN_EVENTS_ENABLE", "jedi", self.vo)
+            if nEventsToEnable is None:
+                nEventsToEnable = nEventsToDisable * 10
+            maxEvents = self.taskBufferIF.getConfigValue(self.component, "JUMBO_MAX_EVENTS", "jedi", self.vo)
+            if maxEvents is None:
+                maxEvents = maxTasks * nEventsToEnable // 2
+            nJumboPerTask = self.taskBufferIF.getConfigValue(self.component, "JUMBO_PER_TASK", "jedi", self.vo)
+            if nJumboPerTask is None:
+                nJumboPerTask = 1
+            nJumboPerSite = self.taskBufferIF.getConfigValue(self.component, "JUMBO_PER_SITE", "jedi", self.vo)
+            if nJumboPerSite is None:
+                nJumboPerSite = 1
+            maxPrio = self.taskBufferIF.getConfigValue(self.component, "JUMBO_MAX_CURR_PRIO", "jedi", self.vo)
+            if maxPrio is None:
+                maxPrio = 500
+            progressToBoost = self.taskBufferIF.getConfigValue(self.component, "JUMBO_PROG_TO_BOOST", "jedi", self.vo)
+            if progressToBoost is None:
+                progressToBoost = 95
+            maxFilesToBoost = self.taskBufferIF.getConfigValue(self.component, "JUMBO_MAX_FILES_TO_BOOST", "jedi", self.vo)
+            if maxFilesToBoost is None:
+                maxFilesToBoost = 500
+            prioToBoost = 900
+            prioWhenDisabled = self.taskBufferIF.getConfigValue(self.component, "JUMBO_PRIO_DISABLED", "jedi", self.vo)
+            if prioWhenDisabled is None:
+                prioWhenDisabled = 500
+            # get current info
+            tasksWithJumbo = self.taskBufferIF.getTaskWithJumbo_JEDI(self.vo, self.prodSourceLabel)
+            totEvents = 0
+            doneEvents = 0
+            nTasks = 0
+            for jediTaskID, taskData in tasksWithJumbo.items():
+                # disable jumbo
+                if taskData["useJumbo"] != JediTaskSpec.enum_useJumbo["disabled"] and taskData["site"] is None:
+                    if taskData["nEvents"] - taskData["nEventsDone"] < nEventsToDisable:
+                        # disable
+                        self.log.info(
+                            "component={0} disable jumbo in jediTaskID={1} due to n_events_to_process={2} < {3}".format(
+                                self.component, jediTaskID, taskData["nEvents"] - taskData["nEventsDone"], nEventsToDisable
+                            )
+                        )
+                        self.taskBufferIF.enableJumboJobs(jediTaskID, 0, 0)
+                    else:
+                        # wait
+                        nTasks += 1
+                        totEvents += taskData["nEvents"]
+                        doneEvents += taskData["nEventsDone"]
+                        self.log.info(
+                            "component={0} keep jumbo in jediTaskID={1} due to n_events_to_process={2} > {3}".format(
+                                self.component, jediTaskID, taskData["nEvents"] - taskData["nEventsDone"], nEventsToDisable
+                            )
+                        )
+                # increase priority for jumbo disabled
+                if taskData["useJumbo"] == JediTaskSpec.enum_useJumbo["disabled"] and taskData["currentPriority"] < prioWhenDisabled:
+                    self.taskBufferIF.changeTaskPriorityPanda(jediTaskID, prioWhenDisabled)
+                    self.log.info(f"component={self.component} priority boost to {prioWhenDisabled} after disabing jumbo in in jediTaskID={jediTaskID}")
+                # increase priority when close to completion
+                if (
+                    taskData["nEvents"] > 0
+                    and (taskData["nEvents"] - taskData["nEventsDone"]) * 100 // taskData["nEvents"] < progressToBoost
+                    and taskData["currentPriority"] < prioToBoost
+                    and (taskData["nFiles"] - taskData["nFilesDone"]) < maxFilesToBoost
+                ):
+                    # boost
+                    tmpStr = "component={0} priority boost to {5} for jediTaskID={1} due to n_events_done={2} > {3}*{4}% ".format(
+                        self.component, jediTaskID, taskData["nEventsDone"], taskData["nEvents"], progressToBoost, prioToBoost
+                    )
+                    tmpStr += f"n_files_remaining={taskData['nFiles'] - taskData['nFilesDone']} < {maxFilesToBoost}"
+                    self.log.info(tmpStr)
+                    self.taskBufferIF.changeTaskPriorityPanda(jediTaskID, prioToBoost)
+                # kick pending
+                if taskData["taskStatus"] in ["pending", "running"] and taskData["useJumbo"] in [
+                    JediTaskSpec.enum_useJumbo["pending"],
+                    JediTaskSpec.enum_useJumbo["running"],
+                ]:
+                    nActiveJumbo = 0
+                    for computingSite, jobStatusMap in taskData["jumboJobs"].items():
+                        for jobStatus, nJobs in jobStatusMap.items():
+                            if jobStatus in ["defined", "assigned", "activated", "sent", "starting", "running", "transferring", "holding"]:
+                                nActiveJumbo += nJobs
+                    if nActiveJumbo == 0:
+                        self.log.info(f"component={self.component} kick jumbo in {taskData['taskStatus']} jediTaskID={jediTaskID}")
+                        self.taskBufferIF.kickPendingTasksWithJumbo_JEDI(jediTaskID)
+                # reset input to re-generate co-jumbo
+                if taskData["currentPriority"] >= prioToBoost:
+                    nReset = self.taskBufferIF.resetInputToReGenCoJumbo_JEDI(jediTaskID)
+                    if nReset is not None and nReset > 0:
+                        self.log.info(f"component={self.component} reset {nReset} inputs to regenerate co-jumbo for jediTaskID={jediTaskID}")
+                    else:
+                        self.log.debug(f"component={self.component} tried to reset inputs to regenerate co-jumbo with {nReset} for jediTaskID={jediTaskID}")
+            self.log.info(
+                f"component={self.component} total_events={totEvents} n_events_to_process={totEvents - doneEvents} n_tasks={nTasks} available for jumbo"
+            )
+
+            self.log.debug(f"component={self.component} done")
+        except Exception:
+            # error
+            errtype, errvalue = sys.exc_info()[:2]
+            errStr = f": {errtype.__name__} {errvalue}"
+            errStr.strip()
+            errStr += traceback.format_exc()
+            self.log.error(errStr)

pandajedi/jedidog/TypicalWatchDogBase.py

@@ -0,0 +1,122 @@
+import os
+import socket
+
+from pandajedi.jediconfig import jedi_config
+from pandajedi.jedicore import Interaction
+from pandaserver.proxycache.token_cache import TokenCache
+from pandaserver.srvcore import CoreUtils
+
+from .WatchDogBase import WatchDogBase
+
+
+# base class for typical watchdog (for production and analysis, etc.)
+class TypicalWatchDogBase(WatchDogBase):
+    # pre-action
+    def pre_action(self, tmpLog, vo, prodSourceLabel, pid, *args, **kwargs):
+        # rescue picked files
+        tmpLog.info(f"rescue tasks with picked files for vo={vo} label={prodSourceLabel}")
+        tmpRet = self.taskBufferIF.rescuePickedFiles_JEDI(vo, prodSourceLabel, jedi_config.watchdog.waitForPicked)
+        if tmpRet is None:
+            # failed
+            tmpLog.error("failed to rescue")
+        else:
+            tmpLog.info(f"rescued {tmpRet} tasks")
+        # reactivate pending tasks
+        tmpLog.info(f"reactivate pending tasks for vo={vo} label={prodSourceLabel}")
+        timeoutForPending = self.taskBufferIF.getConfigValue("watchdog", f"PENDING_TIMEOUT_{prodSourceLabel}", "jedi", vo)
+        if timeoutForPending is None:
+            if hasattr(jedi_config.watchdog, "timeoutForPendingVoLabel"):
+                timeoutForPending = CoreUtils.getConfigParam(jedi_config.watchdog.timeoutForPendingVoLabel, vo, prodSourceLabel)
+            if timeoutForPending is None:
+                timeoutForPending = jedi_config.watchdog.timeoutForPending
+            timeoutForPending = int(timeoutForPending) * 24
+        tmpRet, msg_driven_taskid_set = self.taskBufferIF.reactivatePendingTasks_JEDI(
+            vo, prodSourceLabel, jedi_config.watchdog.waitForPending, timeoutForPending
+        )
+        if tmpRet is None:
+            # failed
+            tmpLog.error("failed to reactivate")
+        else:
+            tmpLog.info(f"reactivated {tmpRet} tasks")
+            for jeditaskid in msg_driven_taskid_set:
+                push_ret = self.taskBufferIF.push_task_trigger_message("jedi_job_generator", jeditaskid)
+                if push_ret:
+                    tmpLog.debug(f"pushed trigger message to jedi_job_generator for jeditaskid={jeditaskid}")
+                else:
+                    tmpLog.warning(f"failed to push trigger message to jedi_job_generator for jeditaskid={jeditaskid}")
+        # unlock tasks
+        tmpLog.info(f"unlock tasks for vo={vo} label={prodSourceLabel} host={socket.getfqdn().split('.')[0]} pgid={os.getpgrp()}")
+        tmpRet = self.taskBufferIF.unlockTasks_JEDI(vo, prodSourceLabel, 10, socket.getfqdn().split(".")[0], os.getpgrp())
+        if tmpRet is None:
+            # failed
+            tmpLog.error("failed to unlock")
+        else:
+            tmpLog.info(f"unlock {tmpRet} tasks")
+        # unlock tasks
+        tmpLog.info(f"unlock tasks for vo={vo} label={prodSourceLabel}")
+        tmpRet = self.taskBufferIF.unlockTasks_JEDI(vo, prodSourceLabel, jedi_config.watchdog.waitForLocked)
+        if tmpRet is None:
+            # failed
+            tmpLog.error("failed to unlock")
+        else:
+            tmpLog.info(f"unlock {tmpRet} tasks")
+        # restart contents update
+        tmpLog.info(f"restart contents update for vo={vo} label={prodSourceLabel}")
+        tmpRet, msg_driven_taskid_set = self.taskBufferIF.restartTasksForContentsUpdate_JEDI(vo, prodSourceLabel)
+        if tmpRet is None:
+            # failed
+            tmpLog.error("failed to restart")
+        else:
+            tmpLog.info(f"restarted {tmpRet} tasks")
+            for jeditaskid in msg_driven_taskid_set:
+                push_ret = self.taskBufferIF.push_task_trigger_message("jedi_contents_feeder", jeditaskid)
+                if push_ret:
+                    tmpLog.debug(f"pushed trigger message to jedi_contents_feeder for jeditaskid={jeditaskid}")
+                else:
+                    tmpLog.warning(f"failed to push trigger message to jedi_contents_feeder for jeditaskid={jeditaskid}")
+        # kick exhausted tasks
+        tmpLog.info(f"kick exhausted tasks for vo={vo} label={prodSourceLabel}")
+        tmpRet = self.taskBufferIF.kickExhaustedTasks_JEDI(vo, prodSourceLabel, jedi_config.watchdog.waitForExhausted)
+        if tmpRet is None:
+            # failed
+            tmpLog.error("failed to kick")
+        else:
+            tmpLog.info(f"kicked {tmpRet} tasks")
+        # finish tasks when goal is reached
+        tmpLog.info(f"finish achieved tasks for vo={vo} label={prodSourceLabel}")
+        tmpRet = self.taskBufferIF.getAchievedTasks_JEDI(vo, prodSourceLabel, jedi_config.watchdog.waitForAchieved)
+        if tmpRet is None:
+            # failed
+            tmpLog.error("failed to finish")
+        else:
+            for jediTaskID in tmpRet:
+                self.taskBufferIF.sendCommandTaskPanda(jediTaskID, "JEDI. Goal reached", True, "finish", comQualifier="soft")
+            tmpLog.info(f"finished {tmpRet} tasks")
+        # rescue unlocked tasks with picked files
+        tmpLog.info(f"rescue unlocked tasks with picked files for vo={vo} label={prodSourceLabel}")
+        tmpRet = self.taskBufferIF.rescueUnLockedTasksWithPicked_JEDI(vo, prodSourceLabel, 60, pid)
+        if tmpRet is None:
+            # failed
+            tmpLog.error("failed to rescue unlocked tasks")
+        else:
+            tmpLog.info(f"rescue unlocked {tmpRet} tasks")
+        # cache tokens
+        got_lock = self.get_process_lock("TypicalWatchDogBase.cache_tokens", timeLimit=5)
+        if not got_lock:
+            tmpLog.debug("locked by another watchdog process. Skipped to cache tokens")
+        else:
+            tmpLog.info(f"cache tokens")
+            cacher = TokenCache()
+            cacher.run()
+
+    # action to set scout job data w/o scouts
+    def doActionToSetScoutJobData(self, gTmpLog):
+        tmpRet = self.taskBufferIF.setScoutJobDataToTasks_JEDI(self.vo, self.prodSourceLabel)
+        if tmpRet is None:
+            # failed
+            gTmpLog.error("failed to set scout job data")
+        else:
+            gTmpLog.info("set scout job data successfully")
+
+
+Interaction.installSC(TypicalWatchDogBase)

pandajedi/jedidog/WatchDogBase.py

@@ -0,0 +1,53 @@
+import os
+import socket
+
+from pandajedi.jedicore import Interaction
+
+
+# base class for watchdog
+class WatchDogBase(object):
+    """
+    Base class for watchdog
+    """
+
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        self.taskBufferIF = taskBufferIF
+        self.ddmIF = ddmIF
+        self.pid = f"{socket.getfqdn().split('.')[0]}-{os.getpid()}-dog"
+        self.vo = None
+        self.refresh()
+
+    def get_process_lock(self, component, timeLimit=5, **kwargs):
+        """
+        Shortcut of get process lock for watchdog action methods
+
+        Args:
+        component (str): spec of the request
+        timeLimit (int): lifetime of the lock in minutes
+        **kwargs: other arguments for taskBufferIF.lockProcess_JEDI
+
+        Returns:
+            bool : True if got lock, False otherwise
+        """
+        return self.taskBufferIF.lockProcess_JEDI(
+            vo=self.vo,
+            prodSourceLabel=kwargs.get("prodSourceLabel", "default"),
+            cloud=kwargs.get("cloud", None),
+            workqueue_id=kwargs.get("workqueue_id", None),
+            resource_name=kwargs.get("resource_name", None),
+            component=component,
+            pid=self.pid,
+            timeLimit=timeLimit,
+        )
+
+    # refresh
+    def refresh(self):
+        self.siteMapper = self.taskBufferIF.get_site_mapper()
+
+    # pre-action
+    def pre_action(self, tmpLog, vo, prodSourceLabel, pid, *args, **kwargs):
+        pass
+
+
+Interaction.installSC(WatchDogBase)