panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jedimsgprocessor/processing_msg_processor.py

@@ -0,0 +1,134 @@
+import json
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# processing message processing plugin
+class ProcessingMsgProcPlugin(BaseMsgProcPlugin):
+    def process(self, msg_obj, decoded_data=None):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse
+        if decoded_data is None:
+            # json decode
+            try:
+                msg_dict = json.loads(msg_obj.data)
+            except Exception as e:
+                err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+                tmp_log.error(err_str)
+                raise
+        else:
+            msg_dict = decoded_data
+        # sanity check
+        try:
+            jeditaskid = int(msg_dict["workload_id"])
+            # message type
+            msg_type = msg_dict["msg_type"]
+            if msg_type == "file_processing":
+                target_list = msg_dict["files"]
+            elif msg_type == "collection_processing":
+                target_list = msg_dict["collections"]
+            elif msg_type == "work_processing":
+                pass
+            else:
+                raise ValueError(f"invalid msg_type value: {msg_type}")
+            # relation type
+            relation_type = msg_dict.get("relation_type")
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # run
+        try:
+            # initialize to_proceed
+            to_proceed = False
+            # type filters
+            if msg_type in ["file_processing", "collection_processing"] and relation_type in ["input"]:
+                to_proceed = True
+            # whether to proceed the targets
+            if to_proceed:
+                # initialize
+                scope_name_dict_map = {}
+                missing_files_dict = {}
+                # loop over targets
+                for target in target_list:
+                    name = target["name"]
+                    scope = target["scope"]
+                    datasetid = target.get("external_coll_id", None)
+                    fileid = target.get("external_content_id", None)
+                    if (msg_type == "file_processing" and target["status"] in ["Available"]) or (
+                        msg_type == "collection_processing" and target["status"] in ["Closed"]
+                    ):
+                        scope_name_dict_map.setdefault(scope, {})
+                        scope_name_dict_map[scope][name] = (datasetid, fileid)
+                    elif msg_type == "file_processing" and target["status"] in ["Missing"]:
+                        # missing files
+                        missing_files_dict[name] = (datasetid, fileid)
+                    else:
+                        # got target in bad attributes, do nothing
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, scope={scope}, msg_type={msg_type}, status={target['status']}, did nothing for bad target")
+                        pass
+                # run by each scope
+                for scope, name_dict in scope_name_dict_map.items():
+                    # about files or datasets in good status
+                    if msg_type == "file_processing":
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, scope={scope}, update about files...")
+                        res = self.tbIF.updateInputFilesStaged_JEDI(jeditaskid, scope, name_dict, by="iDDS")
+                        if res is None:
+                            # got error and rollback in dbproxy
+                            err_str = f"jeditaskid={jeditaskid}, scope={scope}, failed to update files"
+                            raise RuntimeError(err_str)
+                        tmp_log.info(f"jeditaskid={jeditaskid}, scope={scope}, updated {res} files")
+                    elif msg_type == "collection_processing":
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, scope={scope}, update about datasets...")
+                        res = self.tbIF.updateInputDatasetsStaged_JEDI(jeditaskid, scope, name_dict, by="iDDS")
+                        if res is None:
+                            # got error and rollback in dbproxy
+                            err_str = f"jeditaskid={jeditaskid}, scope={scope}, failed to update datasets"
+                            raise RuntimeError(err_str)
+                        tmp_log.info(f"jeditaskid={jeditaskid}, scope={scope}, updated {res} datasets")
+                    # send message to contents feeder if new files are staged
+                    if res > 0 or msg_type == "collection_processing":
+                        tmp_s, task_spec = self.tbIF.getTaskWithID_JEDI(jeditaskid)
+                        if tmp_s and task_spec.is_msg_driven():
+                            push_ret = self.tbIF.push_task_trigger_message("jedi_contents_feeder", jeditaskid, task_spec=task_spec)
+                            if push_ret:
+                                tmp_log.debug(f"pushed trigger message to jedi_contents_feeder for jeditaskid={jeditaskid}")
+                            else:
+                                tmp_log.warning(f"failed to push trigger message to jedi_contents_feeder for jeditaskid={jeditaskid}")
+                    # check if all ok
+                    if res == len(target_list):
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, scope={scope}, all OK")
+                    elif res < len(target_list):
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, scope={scope}, only {res} out of {len(target_list)} done...")
+                    elif res > len(target_list):
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, scope={scope}, strangely, {res} out of {len(target_list)} done...")
+                    else:
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, scope={scope}, something unwanted happened...")
+                # handle missing files
+                n_missing = len(missing_files_dict)
+                if n_missing > 0:
+                    res = self.tbIF.setMissingFilesAboutIdds_JEDI(jeditaskid=jeditaskid, filenames_dict=missing_files_dict)
+                    if res == n_missing:
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, marked all {n_missing} files missing")
+                    elif res < n_missing:
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, only {res} out of {n_missing} files marked missing...")
+                    elif res > n_missing:
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, strangely, {res} out of {n_missing} files marked missing...")
+                    else:
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, res={res}, something unwanted happened about missing files...")
+            else:
+                # do nothing
+                tmp_log.debug(f"jeditaskid={jeditaskid}, msg_type={msg_type}, relation_type={relation_type}, nothing done")
+        except Exception as e:
+            err_str = f"failed to process the message, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # done
+        tmp_log.info("done")

pandajedi/jedimsgprocessor/status_report_msg_processor.py

@@ -0,0 +1,129 @@
+import copy
+import json
+import re
+import traceback
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jediconfig import jedi_config
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# list of task statuses to return to iDDS
+to_return_task_status_list = [
+    "defined",
+    "ready",
+    "scouting",
+    "pending",
+    "paused",
+    "running",
+    "prepared",
+    "done",
+    "finished",
+    "failed",
+    "broken",
+    "aborted",
+]
+
+to_return_job_status_list = [
+    "pending",
+    "defined",
+    "assigned",
+    "activated",
+    "starting",
+    "running",
+    "throttled",
+    "finished",
+    "failed",
+    "cancelled",
+    "closed",
+]
+
+
+# status report message processing plugin
+class StatusReportMsgProcPlugin(BaseMsgProcPlugin):
+    """
+    Messaging status of jobs and tasks
+    Forward the incoming message to other msg_processors plugin (e.g. Kafka) if configured in params
+    Return the processed message to send to iDDS via MQ
+    """
+
+    def initialize(self):
+        BaseMsgProcPlugin.initialize(self)
+        # forwarding plugins: incoming message will be forwarded to process method of these plugins
+        self.forwarding_plugins = []
+        forwarding_plugin_names = self.params.get("forwarding_plugins", [])
+        if "kafka" in forwarding_plugin_names:
+            # Kafka
+            from pandajedi.jedimsgprocessor.kafka_msg_processor import (
+                KafkaMsgProcPlugin,
+            )
+
+            plugin_inst = KafkaMsgProcPlugin()
+            plugin_inst.initialize()
+            self.forwarding_plugins.append(plugin_inst)
+
+    def process(self, msg_obj):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse json
+        try:
+            msg_dict = json.loads(msg_obj.data)
+        except Exception as exc:
+            err_str = f"failed to parse message json {msg_obj.data} , skipped. {exc.__class__.__name__} : {exc} ; {traceback.format_exc()}"
+            tmp_log.error(err_str)
+            raise
+        # sanity check
+        try:
+            msg_type = msg_dict["msg_type"]
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {exc.__class__.__name__} : {exc} ; {traceback.format_exc()}"
+            tmp_log.error(err_str)
+            raise
+        # whether to return the message
+        to_return_message = False
+        # run different plugins according to message type
+        try:
+            # DB source name from DB schema
+            db_source_name = re.sub("_PANDA", "", jedi_config.db.schemaJEDI).lower()
+            # process according to msg_type
+            if msg_type == "task_status":
+                tmp_log.debug("task_status")
+                # forwarding
+                for plugin_inst in self.forwarding_plugins:
+                    try:
+                        tmp_msg_dict = copy.deepcopy(msg_dict)
+                        tmp_msg_dict["db_source"] = db_source_name
+                        plugin_inst.process(msg_obj, decoded_data=tmp_msg_dict)
+                    except Exception as exc:
+                        tmp_log.error(f"failed to process message object dict {tmp_msg_dict}; {exc.__class__.__name__} : {exc} ; {traceback.format_exc()}")
+                # only return certain statuses
+                if msg_dict.get("status") in to_return_task_status_list:
+                    to_return_message = True
+            elif msg_type == "job_status":
+                tmp_log.debug("job_status")
+                # forwarding
+                for plugin_inst in self.forwarding_plugins:
+                    try:
+                        tmp_msg_dict = copy.deepcopy(msg_dict)
+                        tmp_msg_dict["db_source"] = db_source_name
+                        plugin_inst.process(msg_obj, decoded_data=tmp_msg_dict)
+                    except Exception as exc:
+                        tmp_log.error(f"failed to process message object dict {tmp_msg_dict}; {exc.__class__.__name__} : {exc} ; {traceback.format_exc()}")
+                # only return certain statuses
+                if msg_dict.get("status") in to_return_job_status_list:
+                    to_return_message = True
+            else:
+                warn_str = f"unknown msg_type : {msg_type}"
+                tmp_log.warning(warn_str)
+        except Exception:
+            raise
+        # done
+        tmp_log.debug(f"{msg_dict}; to_return={to_return_message}")
+        tmp_log.info("done")
+        if to_return_message:
+            return msg_obj.data

pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py

@@ -0,0 +1,118 @@
+import json
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# Tape carousel message processing plugin
+class TapeCarouselMsgProcPlugin(BaseMsgProcPlugin):
+    def process(self, msg_obj, decoded_data=None):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse
+        if decoded_data is None:
+            # json decode
+            try:
+                msg_dict = json.loads(msg_obj.data)
+            except Exception as e:
+                err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+                tmp_log.error(err_str)
+                raise
+        else:
+            msg_dict = decoded_data
+        # sanity check
+        try:
+            jeditaskid = int(msg_dict["workload_id"])
+            # message type
+            msg_type = msg_dict["msg_type"]
+            if msg_type == "file_stagein":
+                target_list = msg_dict["files"]
+            elif msg_type == "collection_stagein":
+                target_list = msg_dict["collections"]
+            elif msg_type == "work_stagein":
+                pass
+            else:
+                raise ValueError(f"invalid msg_type value: {msg_type}")
+            # relation type
+            relation_type = msg_dict.get("relation_type")
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # run
+        try:
+            # initialize to_proceed
+            to_proceed = False
+            # type filters
+            if msg_type in ["file_stagein", "collection_stagein"] and relation_type in ["output"]:
+                to_proceed = True
+            # whether to proceed the targets
+            if to_proceed:
+                # map
+                scope_name_dict_map = {}
+                # loop over targets
+                for target in target_list:
+                    name = target["name"]
+                    scope = target["scope"]
+                    datasetid = target.get("external_coll_id", None)
+                    fileid = target.get("external_content_id", None)
+                    if (msg_type == "file_stagein" and target["status"] in ["Available"]) or (
+                        msg_type == "collection_stagein" and target["status"] in ["Closed"]
+                    ):
+                        scope_name_dict_map.setdefault(scope, {})
+                        scope_name_dict_map[scope][name] = (datasetid, fileid)
+                    else:
+                        # got target in bad attributes, do nothing
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, scope={scope}, msg_type={msg_type}, status={target['status']}, did nothing for bad target")
+                        pass
+                # run by each scope
+                for scope, name_dict in scope_name_dict_map.items():
+                    # about files or datasets in good status
+                    if msg_type == "file_stagein":
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, scope={scope}, update about files...")
+                        res = self.tbIF.updateInputFilesStaged_JEDI(jeditaskid, scope, name_dict, by="iDDS")
+                        if res is None:
+                            # got error and rollback in dbproxy
+                            err_str = f"jeditaskid={jeditaskid}, scope={scope}, failed to update files"
+                            raise RuntimeError(err_str)
+                        tmp_log.info(f"jeditaskid={jeditaskid}, scope={scope}, updated {res} files")
+                    elif msg_type == "collection_stagein":
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, scope={scope}, update about datasets...")
+                        res = self.tbIF.updateInputDatasetsStaged_JEDI(jeditaskid, scope, name_dict, by="iDDS")
+                        if res is None:
+                            # got error and rollback in dbproxy
+                            err_str = f"jeditaskid={jeditaskid}, scope={scope}, failed to update datasets"
+                            raise RuntimeError(err_str)
+                        tmp_log.info(f"jeditaskid={jeditaskid}, scope={scope}, updated {res} datasets")
+                    # send message to contents feeder if new files are staged
+                    if res > 0 or msg_type == "collection_stagein":
+                        tmp_s, task_spec = self.tbIF.getTaskWithID_JEDI(jeditaskid)
+                        if tmp_s and task_spec.is_msg_driven():
+                            push_ret = self.tbIF.push_task_trigger_message("jedi_contents_feeder", jeditaskid, task_spec=task_spec)
+                            if push_ret:
+                                tmp_log.debug(f"pushed trigger message to jedi_contents_feeder for jeditaskid={jeditaskid}")
+                            else:
+                                tmp_log.warning(f"failed to push trigger message to jedi_contents_feeder for jeditaskid={jeditaskid}")
+                    # check if all ok
+                    if res == len(target_list):
+                        tmp_log.debug(f"jeditaskid={jeditaskid}, scope={scope}, all OK")
+                    elif res < len(target_list):
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, scope={scope}, only {res} out of {len(target_list)} done...")
+                    elif res > len(target_list):
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, scope={scope}, strangely, {res} out of {len(target_list)} done...")
+                    else:
+                        tmp_log.warning(f"jeditaskid={jeditaskid}, scope={scope}, something unwanted happened...")
+            else:
+                # do nothing
+                tmp_log.debug(f"jeditaskid={jeditaskid}, msg_type={msg_type}, relation_type={relation_type}, nothing done")
+        except Exception as e:
+            err_str = f"failed to process the message, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # done
+        tmp_log.info("done")

pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py

@@ -0,0 +1,89 @@
+import json
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+from pandaserver.workflow.workflow_core import WorkflowInterface
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# Workflow manager message processor plugin
+class WorkflowManagerMsgProcPlugin(BaseMsgProcPlugin):
+    """
+    Message-driven workflow manager
+    """
+
+    def initialize(self):
+        """
+        Initialize the plugin
+        """
+        BaseMsgProcPlugin.initialize(self)
+        self.workflow_interface = WorkflowInterface(self.tbIF)
+
+    def process(self, msg_obj):
+        """
+        Process the message
+        Typical message data looks like:
+            {"msg_type":"workflow", "workflow_id": 123, "timestamp": 987654321}
+            {"msg_type":"wfstep", "step_id": 456, "timestamp": 987654321}
+            {"msg_type":"wfdata", "data_id": 789, "timestamp": 987654321}
+
+        Args:
+            msg_obj: message object
+        """
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse json
+        try:
+            msg_dict = json.loads(msg_obj.data)
+        except Exception as e:
+            err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # sanity check
+        try:
+            msg_type = msg_dict["msg_type"]
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        if msg_type not in ("workflow", "wfstep", "wfdata"):
+            err_str = f"got unknown msg_type {msg_type} , skipped "
+            tmp_log.error(err_str)
+            raise
+        # run
+        try:
+            tmp_log.info(f"got message {msg_dict}")
+            if msg_type == "workflow":
+                workflow_id = msg_dict["workflow_id"]
+                workflow_spec = self.tbIF.get_workflow(workflow_id)
+                if workflow_spec is None:
+                    tmp_log.warning(f"workflow_id={workflow_id} not found; skipped")
+                    return
+                stats, workflow_spec = self.workflow_interface.process_workflow(workflow_spec, by="msg")
+                tmp_log.info(f"processed workflow_id={workflow_id}")
+            elif msg_type == "wfstep":
+                step_id = msg_dict["step_id"]
+                step_spec = self.tbIF.get_workflow_step(step_id)
+                if step_spec is None:
+                    tmp_log.warning(f"step_id={step_id} not found; skipped")
+                    return
+                stats, step_spec = self.workflow_interface.process_step(step_spec, by="msg")
+                tmp_log.info(f"processed step_id={step_id}")
+            elif msg_type == "wfdata":
+                data_id = msg_dict["data_id"]
+                data_spec = self.tbIF.get_workflow_data(data_id)
+                if data_spec is None:
+                    tmp_log.warning(f"data_id={data_id} not found; skipped")
+                    return
+                stats, data_spec = self.workflow_interface.process_data(data_spec, by="msg")
+                tmp_log.info(f"processed data_id={data_id}")
+        except Exception as e:
+            err_str = f"failed to run, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # done
+        tmp_log.info("done")