panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jediexec/reniceJEDI.py

@@ -0,0 +1,25 @@
+import subprocess
+
+from pandajedi.jediconfig import jedi_config
+
+proc = subprocess.Popen(
+    f"ps -l x -U {jedi_config.master.uname}",
+    shell=True,
+    stdout=subprocess.PIPE,
+)
+stdoutList = str(proc.communicate()[0]).split("\n")
+for line in stdoutList:
+    try:
+        items = line.split()
+        if len(items) < 6:
+            continue
+        pid = items[3]
+        nice = int(items[7])
+        if "JediMaster.py" in line and nice > 0:
+            reniceProc = subprocess.Popen(
+                f"renice 0 {pid}",
+                shell=True,
+                stdout=subprocess.PIPE,
+            )
+    except Exception:
+        pass

pandajedi/jedigen/AtlasTaskGenerator.py

@@ -0,0 +1,110 @@
+import json
+import re
+import sys
+import uuid
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jedicore import Interaction
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+
+from .TaskGeneratorBase import TaskGeneratorBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# task generator for ATLAS
+class AtlasTaskGenerator(TaskGeneratorBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        TaskGeneratorBase.__init__(self, taskBufferIF, ddmIF)
+
+    # main to generate task
+    def doGenerate(self, taskSpec, taskParamMap, **varMap):
+        # make logger
+        tmpLog = MsgWrapper(logger, f"<jediTaskID={taskSpec.jediTaskID}>")
+        tmpLog.info(f"start taskType={taskSpec.taskType}")
+        tmpLog.info(str(varMap))
+        # returns
+        retFatal = self.SC_FATAL
+        retTmpError = self.SC_FAILED
+        retOK = self.SC_SUCCEEDED
+        try:
+            # check prodSourceLabel
+            if taskSpec.prodSourceLabel in ["managed", "test"]:
+                # check taskType
+                if taskSpec.taskType == "recov":
+                    # generate parent tasks for lost file recovery if it is not yet generated
+                    if "parentGenerated" in taskParamMap:
+                        tmpLog.info("skip since already generated parent tasks")
+                    else:
+                        tmpLog.info("generating parent tasks for lost file recovery")
+                        # missing files are undefined
+                        if "missingFilesMap" not in varMap:
+                            tmpLog.error("missing files are undefined")
+                            return retFatal
+                        missingFilesMap = varMap["missingFilesMap"]
+                        # check datasets
+                        for datasetName, datasetValMap in missingFilesMap.items():
+                            # dataset needs specify container
+                            datasetSpec = datasetValMap["datasetSpec"]
+                            if datasetSpec.containerName in ["", None]:
+                                errStr = f"cannot make parent tasks due to undefined container for datasetID={datasetSpec.datasetID}:{datasetName}"
+                                taskSpec.setErrDiag(errStr)
+                                tmpLog.error(errStr)
+                                return retFatal
+                        # make parameters for new task
+                        newJsonStrList = []
+                        for datasetName, datasetValMap in missingFilesMap.items():
+                            datasetSpec = datasetValMap["datasetSpec"]
+                            newTaskParamMap = {}
+                            newTaskParamMap["oldDatasetName"] = datasetName
+                            newTaskParamMap["lostFiles"] = datasetValMap["missingFiles"]
+                            newTaskParamMap["vo"] = taskSpec.vo
+                            newTaskParamMap["cloud"] = taskSpec.cloud
+                            newTaskParamMap["taskPriority"] = taskSpec.taskPriority
+                            newTaskParamMap["taskType"] = taskSpec.taskType
+                            newTaskParamMap["prodSourceLabel"] = taskSpec.prodSourceLabel
+                            logDatasetName = f"panda.jedi{taskSpec.taskType}.log.{uuid.uuid4()}"
+                            newTaskParamMap["log"] = {
+                                "dataset": logDatasetName,
+                                "type": "template",
+                                "param_type": "log",
+                                "token": "ATLASDATADISK",
+                                "value": f"{logDatasetName}.${{SN}}.log.tgz",
+                            }
+                            # make new datasetname
+                            outDatasetName = datasetName
+                            # remove /
+                            outDatasetName = re.sub("/$", "", outDatasetName)
+                            # remove extension
+                            outDatasetName = re.sub(f"\\.{taskSpec.taskType}\\d+$", "", outDatasetName)
+                            # add extension
+                            outDatasetName = outDatasetName + f".{taskSpec.taskType}{taskSpec.jediTaskID}"
+                            newTaskParamMap["output"] = {"dataset": outDatasetName}
+                            if datasetSpec.containerName not in ["", None]:
+                                newTaskParamMap["output"]["container"] = datasetSpec.containerName
+                            # make json
+                            jsonStr = json.dumps(newTaskParamMap)
+                            newJsonStrList.append(jsonStr)
+                        # change original task parameters to not repeat the same procedure and to use newly produced files
+                        taskParamMap["parentGenerated"] = True
+                        taskParamMap["useInFilesInContainer"] = True
+                        taskParamMap["useInFilesWithNewAttemptNr"] = True
+                        jsonStr = json.dumps(taskParamMap)
+                        # insert and update task parameters
+                        sTmp, newJediTaskIDs = self.taskBufferIF.insertUpdateTaskParams_JEDI(
+                            taskSpec.jediTaskID, taskSpec.vo, taskSpec.prodSourceLabel, jsonStr, newJsonStrList
+                        )
+                        if sTmp:
+                            tmpLog.info(f"inserted/updated tasks in DB : new jediTaskIDs={str(newJediTaskIDs)}")
+                        else:
+                            tmpLog.error("failed to insert/update tasks in DB")
+                            return retFatal
+            # return
+            tmpLog.info("done")
+            return retOK
+        except Exception:
+            errtype, errvalue = sys.exc_info()[:2]
+            tmpLog.error(f"doGenerate failed with {errtype.__name__}:{errvalue}")
+            return retFatal

pandajedi/jedigen/TaskGeneratorBase.py

@@ -0,0 +1,15 @@
+from pandajedi.jedicore import Interaction
+
+
+# base class for task generator
+class TaskGeneratorBase(object):
+    def __init__(self, taskBufferIF, ddmIF):
+        self.ddmIF = ddmIF
+        self.taskBufferIF = taskBufferIF
+        self.refresh()
+
+    def refresh(self):
+        self.siteMapper = self.taskBufferIF.get_site_mapper()
+
+
+Interaction.installSC(TaskGeneratorBase)

pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor

@@ -0,0 +1,71 @@
+import json
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# AI parameter adjuster plugin
+class AIParameterAdjuster(BaseMsgProcPlugin):
+    """
+    Parameter adjuster message processor whose source messages generated by external AI
+    Used for scout prediction, etc.
+    """
+    def process(self, msg_obj, decoded_data=None):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse
+        if decoded_data is None:
+            # json decode
+            try:
+                msg_dict = json.loads(msg_obj.data)
+            except Exception as e:
+                err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+                tmp_log.error(err_str)
+                raise
+        else:
+            msg_dict = decoded_data
+        # sanity check
+        try:
+            task_id = int(msg_dict["task_id"])
+            # message type
+            msg_type = msg_dict["msg_type"]
+            if msg_type == "scout_prediction":
+                pass
+            elif msg_type == "test":
+                pass
+            else:
+                raise ValueError(f"invalid msg_type value: {msg_type}")
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # run
+        try:
+            # initialize to_proceed
+            to_proceed = False
+            # type filters
+            if msg_type in ["scout_prediction", "test"]:
+                to_proceed = True
+            # whether to proceed the targets
+            if to_proceed:
+                # initialize
+                pass
+                # loop over targets
+                if msg_type == "scout_prediction":
+                    #TODO: to complete once the format of scout prediction messages is decided
+                    pass
+                pass
+            else:
+                # do nothing
+                tmp_log.debug(f"task_id={task_id}, msg_type={msg_type}, nothing done")
+        except Exception as e:
+            err_str = f"failed to process the message, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # done
+        tmp_log.info("done")

pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py

@@ -0,0 +1,66 @@
+import json
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+from pandajedi.jedimsgprocessor.hpo_msg_processor import HPOMsgProcPlugin
+from pandajedi.jedimsgprocessor.processing_msg_processor import ProcessingMsgProcPlugin
+from pandajedi.jedimsgprocessor.tape_carousel_msg_processor import (
+    TapeCarouselMsgProcPlugin,
+)
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# Atlas iDDS message processing plugin, a bridge connect to other idds related message processing plugins
+class AtlasIddsMsgProcPlugin(BaseMsgProcPlugin):
+    def initialize(self):
+        BaseMsgProcPlugin.initialize(self)
+        self.plugin_TapeCarousel = TapeCarouselMsgProcPlugin()
+        self.plugin_HPO = HPOMsgProcPlugin()
+        self.plugin_Processing = ProcessingMsgProcPlugin()
+        # for each plugin
+        for _plugin in [self.plugin_TapeCarousel, self.plugin_HPO, self.plugin_Processing]:
+            # initialize each
+            _plugin.initialize(in_collective=True)
+            # use the same taskBuffer interface
+            _plugin.tbIF = self.tbIF
+
+    def process(self, msg_obj):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse json
+        try:
+            msg_dict = json.loads(msg_obj.data)
+        except Exception as e:
+            err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # sanity check
+        try:
+            msg_type = msg_dict.get("msg_type")
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # run different plugins according to message type
+        try:
+            if msg_type in ("file_stagein", "collection_stagein", "work_stagein"):
+                tmp_log.debug("to tape_carousel")
+                self.plugin_TapeCarousel.process(msg_obj, decoded_data=msg_dict)
+            elif msg_type in ("file_hyperparameteropt", "collection_hyperparameteropt", "work_hyperparameteropt"):
+                tmp_log.debug("to hpo")
+                self.plugin_HPO.process(msg_obj, decoded_data=msg_dict)
+            elif msg_type in ("file_processing", "collection_processing", "work_processing"):
+                tmp_log.debug("to processing")
+                self.plugin_Processing.process(msg_obj, decoded_data=msg_dict)
+            else:
+                # Asked by iDDS and message broker guys, JEDI needs to consume unknown types of messages and do nothing...
+                warn_str = f"unknown msg_type: {msg_type} msg_dict: {str(msg_dict)}"
+                tmp_log.warning(warn_str)
+        except Exception:
+            raise
+        # done
+        tmp_log.info("done")

pandajedi/jedimsgprocessor/base_msg_processor.py

@@ -0,0 +1,33 @@
+from pandacommon.pandamsgbkr.msg_processor import SimpleMsgProcPluginBase
+
+from pandajedi.jedicore.JediTaskBuffer import JediTaskBuffer
+
+
+# Base simple message processing plugin
+class BaseMsgProcPlugin(SimpleMsgProcPluginBase):
+    def initialize(self, in_collective=False):
+        """
+        initialize plugin instance, run once before loop in thread
+        """
+        if in_collective:
+            # run in collective msg_proc plugin
+            pass
+        else:
+            # run as individual msg_proc plugin
+            self.set_tbIF()
+
+    def set_tbIF(self):
+        """
+        set up JEDI TaskBuffer interface
+        """
+        # set nDBConnection = n_threads to avoid DBProxy blocking amongs threads
+        n_db_conns = getattr(self, "n_threads", 1)
+        self.tbIF = JediTaskBuffer(None, nDBConnection=n_db_conns)
+
+    def process(self, msg_obj):
+        """
+        process the message
+        Get msg_obj from the incoming MQ (if any; otherwise msg_obj is None)
+        Returned value will be sent to the outgoing MQ (if any)
+        """
+        pass

pandajedi/jedimsgprocessor/forwarding_msg_processor.py

@@ -0,0 +1,29 @@
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# forwarding message processing plugin
+class ForwardingMsgProcPlugin(BaseMsgProcPlugin):
+    """
+    Simply forward the message from one queue to another
+    """
+
+    def process(self, msg_obj):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        # tmp_log.info('start')
+        # tmp_log.debug('sub_id={0} ; msg_id={1}'.format(msg_obj.sub_id, msg_obj.msg_id))
+        # run
+        try:
+            msg = msg_obj.data
+            tmp_log.debug(f"forward message {msg}")
+        except Exception as e:
+            err_str = f"failed to run, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # done
+        # tmp_log.info('done')
+        return msg

pandajedi/jedimsgprocessor/hpo_msg_processor.py

@@ -0,0 +1,93 @@
+import json
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# Hyper-Parameter-Optimization message processing plugin
+class HPOMsgProcPlugin(BaseMsgProcPlugin):
+    def process(self, msg_obj, decoded_data=None):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse
+        if decoded_data is None:
+            # json decode
+            try:
+                msg_dict = json.loads(msg_obj.data)
+            except Exception as e:
+                err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+                tmp_log.error(err_str)
+                raise
+        else:
+            msg_dict = decoded_data
+        # sanity check
+        try:
+            msg_type = msg_dict["msg_type"]
+            jeditaskid = int(msg_dict["workload_id"])
+            if msg_type == "file_hyperparameteropt":
+                target_list = msg_dict["files"]
+            elif msg_type == "collection_hyperparameteropt":
+                # to finish the task
+                pass
+            elif msg_type == "work_hyperparameteropt":
+                pass
+            else:
+                raise ValueError(f"invalid msg_type value: {msg_type}")
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # run
+        if msg_type == "file_hyperparameteropt":
+            # insert HPO events
+            try:
+                # event ids from the targets
+                event_id_list = []
+                for target in target_list:
+                    if target["status"] != "New":
+                        continue
+                    model_id = None
+                    try:
+                        path = json.loads(target["path"])
+                        if isinstance(path[0], (list, tuple)):
+                            model_id = path[0][0]
+                    except Exception:
+                        pass
+                    event_id_list.append((target["name"], model_id))
+                if event_id_list:
+                    n_events = len(event_id_list)
+                    # insert events
+                    res = self.tbIF.insertHpoEventAboutIdds_JEDI(jedi_task_id=jeditaskid, event_id_list=event_id_list)
+                    # check if ok
+                    if res:
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, inserted {n_events} events: {event_id_list}")
+                    else:
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, failed to insert events: {event_id_list}")
+            except Exception as e:
+                err_str = f"failed to parse message object, skipped. {e.__class__.__name__} : {e}"
+                tmp_log.error(err_str)
+                raise
+        elif msg_type == "collection_hyperparameteropt":
+            # finish the task
+            try:
+                # send finish command
+                retVal, retStr = self.tbIF.sendCommandTaskPanda(jeditaskid, "iDDS. HPO task finished", True, "finish", comQualifier="soft")
+                # check if ok
+                if retVal:
+                    tmp_log.debug(f"jeditaskid={jeditaskid}, finished the task")
+                else:
+                    tmp_log.warning(f"jeditaskid={jeditaskid}, failed finish the task: {retStr}")
+            except Exception as e:
+                err_str = f"failed to parse message object, skipped. {e.__class__.__name__} : {e}"
+                tmp_log.error(err_str)
+                raise
+        else:
+            # do nothing
+            tmp_log.debug(f"jeditaskid={jeditaskid}, msg_type={msg_type}, did nothing")
+        # done
+        tmp_log.info("done")

pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py

@@ -0,0 +1,63 @@
+import json
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jediddm.DDMInterface import DDMInterface
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+from pandajedi.jediorder.ContentsFeeder import ContentsFeederThread
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# Jedi Contents Feeder message processor plugin
+class JediContentsFeederMsgProcPlugin(BaseMsgProcPlugin):
+    """
+    Message-driven Contents Feeder
+    """
+
+    def initialize(self):
+        BaseMsgProcPlugin.initialize(self)
+        ddmIF = DDMInterface()
+        ddmIF.setupInterface()
+        the_pid = self.get_pid()
+        self.contents_feeder_thread_obj = ContentsFeederThread(taskDsList=None, threadPool=None, taskbufferIF=self.tbIF, ddmIF=ddmIF, pid=the_pid)
+
+    def process(self, msg_obj):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse json
+        try:
+            msg_dict = json.loads(msg_obj.data)
+        except Exception as e:
+            err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # sanity check
+        try:
+            msg_type = msg_dict["msg_type"]
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        if msg_type != "jedi_contents_feeder":
+            # FIXME
+            err_str = f"got unknown msg_type {msg_type} , skipped "
+            tmp_log.error(err_str)
+            raise
+        # run
+        try:
+            task_id = msg_dict["taskid"]
+            task_ds_list = self.tbIF.getDatasetsToFeedContents_JEDI(vo=None, prodSourceLabel=None, task_id=task_id)
+            if task_ds_list:
+                self.contents_feeder_thread_obj.feed_contents_to_tasks(task_ds_list)
+                tmp_log.info(f"fed datasets to task {task_id}")
+            else:
+                tmp_log.debug(f"got empty list of datasets to feed to task {task_id}; skipped ")
+        except Exception as e:
+            err_str = f"failed to run, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # done
+        tmp_log.info("done")