panda-server-source 0.8.0__py3-none-any.whl

pandaserver/daemons/scripts/task_evaluator.py

@@ -0,0 +1,405 @@
+import datetime
+import functools
+import json
+import os
+import socket
+import sys
+import traceback
+from zlib import adler32
+
+from pandacommon.pandalogger import logger_utils
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+from pandacommon.pandautils.thread_utils import GenericThread
+
+from pandaserver.config import panda_config
+from pandaserver.daemons.scripts.metric_collector import MetricsDB
+
+# logger
+main_logger = PandaLogger().getLogger("task_evaluator")
+
+# dry run
+DRY_RUN = False
+
+# list of metrics in FetchData to fetch data and update to DB. Format: (metric, period_minutes)
+metric_list = [
+    ("analy_task_eval", 10),
+]
+
+# constant maps
+# class_value_rank_map = {1: 'A_sites', 0: 'B_sites', -1: 'C_sites'}
+
+
+class TaskEvaluationDB(object):
+    """
+    Proxy to access the task_evaluation table in DB
+    """
+
+    def __init__(self, tbuf):
+        self.tbuf = tbuf
+
+    def _decor(method):
+        def _decorator(_method, *args, **kwargs):
+            @functools.wraps(_method)
+            def _wrapped_method(self, *args, **kwargs):
+                try:
+                    _method(self, *args, **kwargs)
+                except Exception as exc:
+                    pass
+
+            return _wrapped_method
+
+        return _decorator(method)
+
+    def update(self, metric, entity_dict):
+        tmp_log = logger_utils.make_logger(main_logger, "TaskEvaluationDB.update")
+        tmp_log.debug(f"start metric={metric}")
+        # sql
+        sql_query_taskid = """SELECT jediTaskID """ """FROM ATLAS_PANDA.Task_Evaluation """ """WHERE metric = :metric """
+        sql_update = (
+            """UPDATE ATLAS_PANDA.Task_Evaluation SET """
+            """value_json = :patch_value_json, """
+            """timestamp = :timestamp """
+            """WHERE jediTaskID=:taskID AND metric=:metric """
+        )
+        sql_insert = """INSERT INTO ATLAS_PANDA.Task_Evaluation """ """VALUES ( """ """:taskID, :metric, :patch_value_json, :timestamp """ """) """
+        # now
+        now_time = naive_utcnow()
+        # get existing taskID list
+        res = self.tbuf.querySQL(sql_query_taskid, {":metric": metric})
+        existing_taskID_list = [taskID for (taskID,) in res]
+        # var map template
+        varMap_template = {
+            ":taskID": None,
+            ":metric": metric,
+            ":timestamp": now_time,
+            ":patch_value_json": None,
+        }
+        # make var map lists
+        update_varMap_list = []
+        insert_varMap_list = []
+        for taskID, v in entity_dict.items():
+            # values to json string
+            try:
+                patch_value_json = json.dumps(v)
+            except Exception:
+                tmp_log.error(traceback.format_exc() + " " + str(v))
+                return
+            # update varMap
+            varMap = varMap_template.copy()
+            varMap[":taskID"] = taskID
+            varMap[":patch_value_json"] = patch_value_json
+            # append to the list
+            if taskID in existing_taskID_list:
+                update_varMap_list.append(varMap)
+            else:
+                insert_varMap_list.append(varMap)
+        # update
+        n_row = self.tbuf.executemanySQL(sql_update, update_varMap_list)
+        if n_row < len(update_varMap_list):
+            tmp_log.warning(f"only {n_row}/{len(update_varMap_list)} rows updated for metric={metric}")
+        else:
+            tmp_log.debug(f"updated {len(update_varMap_list)} rows for metric={metric}")
+        # insert
+        n_row = self.tbuf.executemanySQL(sql_insert, insert_varMap_list)
+        if n_row is None:
+            # try to insert one by one
+            n_row = 0
+            for varMap in insert_varMap_list:
+                res = self.tbuf.querySQL(sql_insert, varMap)
+                try:
+                    n_row += res
+                except TypeError:
+                    pass
+        if n_row < len(insert_varMap_list):
+            tmp_log.warning(f"only {n_row}/{len(insert_varMap_list)} rows inserted for metric={metric}")
+        else:
+            tmp_log.debug(f"inserted {len(insert_varMap_list)} rows for metric={metric}")
+        # done
+        tmp_log.debug(f"done metric={metric}")
+
+    def get_metrics(self, metric, fresher_than_minutes_ago=120):
+        tmp_log = logger_utils.make_logger(main_logger, "TaskEvaluationDB.update")
+        tmp_log.debug(f"start metric={metric}")
+        # sql
+        sql_query = (
+            """SELECT jediTaskID, value_json """ """FROM ATLAS_PANDA.Task_Evaluation """ """WHERE metric = :metric """ """AND timestamp >= :min_timestamp """
+        )
+        # now
+        now_time = naive_utcnow()
+        # var map
+        varMap = {
+            ":metric": metric,
+            ":min_timestamp": now_time - datetime.timedelta(minutes=fresher_than_minutes_ago),
+        }
+        # query
+        res = self.tbuf.querySQL(sql_query, varMap)
+        if res is None:
+            tmp_log.warning(f"failed to query metric={metric}")
+            return
+        # return map
+        ret_map = {}
+        for taskID, value_json in res:
+            try:
+                value_dict = json.loads(value_json)
+            except Exception:
+                tmp_log.error(traceback.format_exc() + " " + str(taskID) + str(value_json))
+            else:
+                ret_map[taskID] = value_dict
+        # return
+        return ret_map
+
+    def clean_up(self, metric, fresher_than_minutes_ago=120):
+        tmp_log = logger_utils.make_logger(main_logger, "TaskEvaluationDB.clean_up")
+        tmp_log.debug(f"start metric={metric}")
+        # sql
+        sql_delete_terminated_tasks = (
+            "DELETE "
+            "FROM ATLAS_PANDA.Task_Evaluation te "
+            "WHERE te.jediTaskID IN ( "
+            "SELECT jt.jediTaskID "
+            "FROM ATLAS_PANDA.JEDI_Tasks jt, ATLAS_PANDA.Task_Evaluation tez "
+            "WHERE tez.jediTaskID = jt.jediTaskID "
+            "AND jt.status IN ('done', 'finished', 'failed', 'broken', 'aborted', 'exhausted') "
+            ") "
+            "AND te.metric = :metric "
+            "AND te.timestamp <= :max_timestamp "
+        )
+        # now
+        now_time = naive_utcnow()
+        # var map
+        varMap = {
+            ":metric": metric,
+            ":max_timestamp": now_time - datetime.timedelta(minutes=fresher_than_minutes_ago),
+        }
+        # clean up
+        n_row = self.tbuf.querySQL(sql_delete_terminated_tasks, varMap)
+        tmp_log.debug(f"cleaned up {n_row} rows for metric={metric}")
+
+
+class FetchData(object):
+    """
+    methods to fetch or evaluate data values to store
+    """
+
+    def __init__(self, tbuf):
+        self.tbuf = tbuf
+        # initialize stored data
+        self.gshare_status = None
+
+    def analy_task_eval(self):
+        tmp_log = logger_utils.make_logger(main_logger, "FetchData")
+        # sql
+        sql_get_active_tasks = (
+            "SELECT jt.jediTaskID, jt.userName, jt.gshare "
+            "FROM ATLAS_PANDA.JEDI_Tasks jt, ATLAS_PANDA.JEDI_AUX_Status_MinTaskID asm "
+            "WHERE jt.taskType = 'anal' AND jt.prodSourceLabel = 'user' "
+            "AND jt.status=asm.status AND jt.jediTaskID >= asm.min_jediTaskID "
+            "AND jt.status IN ('scouting', 'scouted', 'running', 'pending', 'throttled') "
+            "AND jt.userName NOT IN ('gangarbt') "
+            "AND jt.modificationTime >= CURRENT_DATE - 30 "
+        )
+        sql_get_task_dsinfo = (
+            "SELECT ds.jediTaskID, SUM(ds.nFiles), SUM(ds.nFilesFinished), SUM(ds.nFilesFailed) "
+            "FROM ATLAS_PANDA.JEDI_Datasets ds "
+            "WHERE ds.jediTaskID = :taskID "
+            "AND ds.type IN ('input', 'pseudo_input') "
+            "AND ds.masterID IS NULL "
+            "GROUP BY ds.jediTaskID "
+        )
+        sql_get_task_n_jobs = (
+            "SELECT COUNT(*) FROM ( "
+            "SELECT DISTINCT c.PandaID "
+            "FROM ATLAS_PANDA.JEDI_Datasets ds, ATLAS_PANDA.JEDI_Dataset_Contents c "
+            "WHERE c.jediTaskID=ds.jediTaskID AND c.datasetID=ds.datasetID "
+            "AND ds.jediTaskID=:taskID AND ds.masterID IS NULL "
+            "AND ds.type IN ('input', 'pseudo_input') "
+            ") "
+        )
+        try:
+            # initialize
+            # tmp_site_dict = dict()
+            task_dict = dict()
+            # now time
+            now_time = naive_utcnow()
+            # MetricsDB
+            mdb = MetricsDB(self.tbuf)
+            # get user evaluation
+            ue_dict = mdb.get_metrics("analy_user_eval", "neither", fresher_than_minutes_ago=20)
+            # get active tasks
+            varMap = {}
+            active_tasks_list = self.tbuf.querySQL(sql_get_active_tasks, varMap)
+            taskID_list = [task[0] for task in active_tasks_list]
+            n_tot_tasks = len(active_tasks_list)
+            tmp_log.debug(f"got total {n_tot_tasks} tasks")
+            # counter
+            cc = 0
+            n_tasks_dict = {
+                2: 0,
+                1: 0,
+                0: 0,
+                -1: 0,
+            }
+            # loop over tasks
+            for taskID, user, gshare in active_tasks_list:
+                # initialize
+                task_class = 1
+                n_files_total = 0
+                n_files_finished = 0
+                n_files_failed = 0
+                n_files_remaining = 0
+                pct_finished = 0
+                pct_failed = 0
+                n_jobs = 0
+                n_jobs_remaining = 0
+                # get dataset info of each task
+                varMap = {":taskID": taskID}
+                dsinfo_list = self.tbuf.querySQL(sql_get_task_dsinfo, varMap)
+                dsinfo_dict = {
+                    tup[0]: {
+                        "nFiles": tup[1],
+                        "nFilesFinished": tup[2],
+                        "nFilesFailed": tup[3],
+                    }
+                    for tup in dsinfo_list
+                }
+                # get n jobs of each task
+                tmp_res = self.tbuf.querySQL(sql_get_task_n_jobs, varMap)
+                for obj in tmp_res:
+                    (n_jobs,) = obj
+                    dsinfo_dict.setdefault(taskID, {})
+                    dsinfo_dict[taskID]["nJobs"] = n_jobs
+                    break
+                # get task proceeding progress
+                ds_info = dsinfo_dict.get(taskID)
+                if ds_info is not None:
+                    n_files_total = ds_info.get("nFiles", 0)
+                    n_files_finished = ds_info.get("nFilesFinished", 0)
+                    n_files_failed = ds_info.get("nFilesFailed", 0)
+                    n_files_remaining = max(n_files_total - n_files_finished - n_files_failed, 0)
+                    n_jobs = ds_info.get("nJobs", 0)
+                    if n_files_total > 0:
+                        pct_finished = n_files_finished * 100 / n_files_total
+                        pct_failed = n_files_failed * 100 / n_files_total
+                        n_jobs_remaining = int(n_jobs * n_files_remaining / n_files_total)
+                # classify
+                if gshare == "Express Analysis":
+                    # Express Analysis tasks always in class S
+                    task_class = 2
+                else:
+                    # parameters
+                    progress_to_boost_A = self.tbuf.getConfigValue("analy_eval", "PROGRESS_TO_BOOST_A", default=90)
+                    progress_to_boost_B = self.tbuf.getConfigValue("analy_eval", "PROGRESS_TO_BOOST_B", default=95)
+                    max_rem_jobs_to_boost_A = self.tbuf.getConfigValue("analy_eval", "MAX_REM_JOBS_TO_BOOST_A", default=500)
+                    max_rem_jobs_to_boost_B = self.tbuf.getConfigValue("analy_eval", "MAX_REM_JOBS_TO_BOOST_B", default=200)
+                    # check usage of the user
+                    usage_dict = ue_dict.get(user)
+                    if usage_dict is None:
+                        continue
+                    if usage_dict["rem_slots_A"] <= 0:
+                        if usage_dict["rem_slots_B"] <= 0:
+                            task_class = -1
+                        else:
+                            task_class = 0
+                    # boost for nearly done tasks
+                    if task_class == 1 and pct_finished >= progress_to_boost_A and n_jobs_remaining > 0 and n_jobs_remaining <= max_rem_jobs_to_boost_A:
+                        # almost done A-tasks, to boost
+                        task_class = 2
+                    elif task_class == 0 and pct_finished >= progress_to_boost_B and n_jobs_remaining > 0 and n_jobs_remaining <= max_rem_jobs_to_boost_B:
+                        # almost done B-tasks, to boost
+                        task_class = 2
+                # fill in task class
+                task_dict[taskID] = {
+                    "task_id": taskID,
+                    "user": user,
+                    "gshare": gshare,
+                    "n_files_total": n_files_total,
+                    "n_files_finished": n_files_finished,
+                    "n_files_failed": n_files_failed,
+                    "pct_finished": pct_finished,
+                    "pct_failed": pct_failed,
+                    "class": task_class,
+                }
+                # counter
+                cc += 1
+                if cc % 5000 == 0:
+                    tmp_log.debug(f"evaluated {cc:6d} tasks")
+                n_tasks_dict[task_class] += 1
+            tmp_log.debug(f"evaluated {cc:6d} tasks in total (S:{n_tasks_dict[2]}, A:{n_tasks_dict[1]}, B:{n_tasks_dict[0]}, C:{n_tasks_dict[-1]})")
+            # return
+            # tmp_log.debug('{}'.format(str([ v for v in task_dict.values() if v['class'] != 1 ])[:3000]))
+            tmp_log.debug("done")
+            return task_dict
+        except Exception:
+            tmp_log.error(traceback.format_exc())
+
+
+# main
+def main(tbuf=None, **kwargs):
+    # instantiate TB
+    if tbuf is None:
+        from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+        requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+        taskBuffer.init(
+            panda_config.dbhost,
+            panda_config.dbpasswd,
+            nDBConnection=1,
+            useTimeout=True,
+            requester=requester_id,
+        )
+    else:
+        taskBuffer = tbuf
+    # pid
+    my_pid = os.getpid()
+    my_full_pid = f"{socket.getfqdn().split('.')[0]}-{os.getpgrp()}-{my_pid}"
+    # go
+    if DRY_RUN:
+        # dry run, regardless of lock, not update DB
+        fetcher = FetchData(taskBuffer)
+        # loop over all fetch data methods to run and update to DB
+        for metric_name, period in metric_list:
+            main_logger.debug(f"(dry-run) start {metric_name}")
+            # fetch data and update DB
+            the_method = getattr(fetcher, metric_name)
+            fetched_data = the_method()
+            if fetched_data is None:
+                main_logger.warning(f"(dry-run) {metric_name} got no valid data")
+                continue
+            main_logger.debug(f"(dry-run) done {metric_name}")
+    else:
+        # real run, will update DB
+        # instantiate
+        tedb = TaskEvaluationDB(taskBuffer)
+        fetcher = FetchData(taskBuffer)
+        # loop over all fetch data methods to run and update to DB
+        for metric_name, period in metric_list:
+            # metric lock
+            lock_component_name = f"pandaTaskEval.{metric_name:.30}.{adler32(metric_name.encode('utf-8')):0x}"
+            # try to get lock
+            got_lock = taskBuffer.lockProcess_PANDA(component=lock_component_name, pid=my_full_pid, time_limit=period)
+            if got_lock:
+                main_logger.debug(f"got lock of {metric_name}")
+            else:
+                main_logger.debug(f"{metric_name} locked by other process; skipped...")
+                continue
+            main_logger.debug(f"start {metric_name}")
+            # clean up
+            tedb.clean_up(metric=metric_name, fresher_than_minutes_ago=120)
+            main_logger.debug(f"cleaned up {metric_name}")
+            # fetch data and update DB
+            the_method = getattr(fetcher, metric_name)
+            fetched_data = the_method()
+            if fetched_data is None:
+                main_logger.warning(f"{metric_name} got no valid data")
+                continue
+            tedb.update(metric=metric_name, entity_dict=fetched_data)
+            main_logger.debug(f"done {metric_name}")
+    # stop taskBuffer if created inside this script
+    if tbuf is None:
+        taskBuffer.cleanup(requester=requester_id)
+
+
+# run
+if __name__ == "__main__":
+    main()

pandaserver/daemons/scripts/tmpwatch.py

@@ -0,0 +1,70 @@
+import datetime
+import glob
+import optparse
+import os
+import sys
+
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+
+# main
+def main(argv=tuple(), **kwargs):
+    # options
+    optP = optparse.OptionParser(conflict_handler="resolve")
+    optP.add_option(
+        "-t",
+        action="store_const",
+        const=True,
+        dest="test",
+        default=False,
+        help="test mode",
+    )
+    optP.add_option(
+        "-h",
+        action="store",
+        type="int",
+        dest="limit",
+        default=12,
+        help="time limit in hour",
+    )
+    options, args = optP.parse_args(args=argv[1:])
+
+    # patterns of tmp files
+    tmpPatts = ["/tmp/tmp*", "/tmp/atlpan/tmp*", "/tmp/pansrv/tmp*"]
+
+    # limit
+    timeLimit = naive_utcnow() - datetime.timedelta(hours=options.limit)
+
+    # loop over all pattern
+    for tmpPatt in tmpPatts:
+        tmpFiles = glob.glob(tmpPatt)
+        # loop over all files
+        for tmpFile in tmpFiles:
+            try:
+                print(f"INFO: tmpfile -> {tmpFile}")
+                # only file
+                if not os.path.isfile(tmpFile):
+                    continue
+                # not symlink
+                if os.path.islink(tmpFile):
+                    continue
+                # writable
+                if not os.access(tmpFile, os.W_OK):
+                    continue
+                # check time stamp
+                timeStamp = os.path.getmtime(tmpFile)
+                timeStamp = datetime.datetime.fromtimestamp(timeStamp)
+                if timeStamp > timeLimit:
+                    continue
+                # remove
+                print(f"INFO:    remove {tmpFile}")
+                if not options.test:
+                    os.remove(tmpFile)
+            except Exception:
+                errType, errValue = sys.exc_info()[:2]
+                print(f"ERROR:   failed with {errType}:{errValue}")
+
+
+# run
+if __name__ == "__main__":
+    main(argv=sys.argv)

pandaserver/daemons/scripts/worker_synchronization.py

@@ -0,0 +1,97 @@
+import sys
+import time
+import traceback
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.thread_utils import GenericThread
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer.Utils import create_shards
+
+# logger
+_logger = PandaLogger().getLogger("worker_sync")
+
+
+def translate_status_to_command(pilot_status):
+    if pilot_status == "running":
+        return "SYNC_WORKERS_ACTIVATE"
+    if pilot_status == "finished":
+        return "SYNC_WORKERS_KILL"
+    return None
+
+
+class WorkerSync(object):
+    def __init__(self, tbuf):
+        self._logger = _logger
+        self.tbuf = tbuf
+        return
+
+    def run(self):
+        """
+        Identifies workers with stale harvester states and newer pilot states
+        :return:
+        """
+
+        # timing
+        time_start = time.time()
+        self._logger.debug("Start.")
+
+        # variables for the harvester command
+        status = "new"
+        ack_requested = False
+        lock_interval = None
+        com_interval = None
+
+        try:
+            stale_workers_per_harvester = self.tbuf.get_workers_to_synchronize()
+            for harvester_id in stale_workers_per_harvester:
+                for pilot_status in stale_workers_per_harvester[harvester_id]:
+                    command = translate_status_to_command(pilot_status)
+                    if command:
+                        workers = stale_workers_per_harvester[harvester_id][pilot_status]
+                        for worker_shard in create_shards(workers, 100):
+                            self._logger.debug(f"Processing harvester_id={harvester_id} pilot_status={pilot_status}. Workers to update: {worker_shard}")
+                            self.tbuf.commandToHarvester(
+                                harvester_id,
+                                command,
+                                ack_requested,
+                                status,
+                                lock_interval,
+                                com_interval,
+                                worker_shard,
+                            )
+        except Exception:
+            self._logger.error(traceback.format_exc())
+
+        # timing
+        time_stop = time.time()
+        self._logger.debug(f"Done. Worker sync took: {time_stop - time_start} s")
+
+        return
+
+
+# main
+def main(tbuf=None, **kwargs):
+    # instantiate TB
+    if tbuf is None:
+        from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+        requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+        taskBuffer.init(
+            panda_config.dbhost,
+            panda_config.dbpasswd,
+            nDBConnection=1,
+            useTimeout=True,
+            requester=requester_id,
+        )
+    else:
+        taskBuffer = tbuf
+    # run
+    WorkerSync(tbuf=taskBuffer).run()
+    # stop taskBuffer if created inside this script
+    if tbuf is None:
+        taskBuffer.cleanup(requester=requester_id)
+
+
+# run
+if __name__ == "__main__":
+    main()