panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py

@@ -0,0 +1,185 @@
+import gc
+import json
+import time
+
+from pandacommon.pandalogger import logger_utils
+from pandacommon.pandautils.PandaUtils import try_malloc_trim
+
+from pandajedi.jedicore.ThreadUtils import ListWithLock
+from pandajedi.jediddm.DDMInterface import DDMInterface
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+from pandajedi.jediorder.JobGenerator import JobGeneratorThread, get_params_to_get_tasks
+from pandajedi.jediorder.TaskSetupper import TaskSetupper
+from pandaserver.srvcore import CoreUtils
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# Jedi Job Generator message processor plugin
+class JediJobGeneratorMsgProcPlugin(BaseMsgProcPlugin):
+    """
+    Message-driven Job Generator
+    """
+
+    def initialize(self):
+        BaseMsgProcPlugin.initialize(self)
+        # DDM interface
+        self.ddmIF = DDMInterface()
+        self.ddmIF.setupInterface()
+        self.task_setupper_map = {}
+        # cache heavy metadata objects with short TTL to reduce allocation churn
+        self._cache_ttl_sec = 300
+        self._site_mapper = None
+        self._site_mapper_ts = 0
+        self._work_queue_mapper = None
+        self._work_queue_mapper_ts = 0
+        self._resource_types = None
+        self._resource_types_ts = 0
+        self._params_to_get_tasks = {}
+        # memory limit to trigger early cleanup to avoid OOM killer. This is not a hard limit, just a threshold to trigger early cleanup.
+        self._mem_usage_threshold_mb = 1500
+        # get SiteMapper
+        # siteMapper = self.tbIF.get_site_mapper()
+        # get work queue mapper
+        # workQueueMapper = self.tbIF.getWorkQueueMap()
+        # get TaskSetupper
+        # taskSetupper = TaskSetupper(self.vos, self.prodSourceLabels)
+        # taskSetupper.initializeMods(self.tbIF, self.ddmIF)
+        self.pid = self.get_pid()
+
+    def _is_cache_valid(self, ts):
+        return (time.time() - ts) < self._cache_ttl_sec
+
+    def _get_site_mapper(self):
+        if self._site_mapper is None or not self._is_cache_valid(self._site_mapper_ts):
+            self._site_mapper = self.tbIF.get_site_mapper()
+            self._site_mapper_ts = time.time()
+        return self._site_mapper
+
+    def _get_work_queue_mapper(self):
+        if self._work_queue_mapper is None or not self._is_cache_valid(self._work_queue_mapper_ts):
+            self._work_queue_mapper = self.tbIF.getWorkQueueMap()
+            self._work_queue_mapper_ts = time.time()
+        return self._work_queue_mapper
+
+    def _get_resource_types(self):
+        if self._resource_types is None or not self._is_cache_valid(self._resource_types_ts):
+            self._resource_types = self.tbIF.load_resource_types()
+            self._resource_types_ts = time.time()
+        return self._resource_types
+
+    def process(self, msg_obj):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        input_list = None
+        gen_thr = None
+        tmp_list = None
+        taskSpec = None
+        taskSetupper = None
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse json
+        try:
+            msg_dict = json.loads(msg_obj.data)
+        except Exception as e:
+            err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # sanity check
+        try:
+            msg_type = msg_dict["msg_type"]
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        if msg_type != "jedi_job_generator":
+            # FIXME
+            err_str = f"got unknown msg_type {msg_type} , skipped"
+            tmp_log.error(err_str)
+            raise
+        # run
+        try:
+            # get task to generate jobs
+            task_id = int(msg_dict["taskid"])
+            _, taskSpec = self.tbIF.getTaskWithID_JEDI(task_id)
+            if not taskSpec:
+                tmp_log.debug(f"unknown task_id={task_id}, skipped")
+            else:
+                tmp_log.debug(f"processing task_id={task_id} workqueue_id={taskSpec.workQueue_ID} status={taskSpec.status}")
+                # get WQ
+                vo = taskSpec.vo
+                prodSourceLabel = taskSpec.prodSourceLabel
+                workQueue = self._get_work_queue_mapper().getQueueWithIDGshare(taskSpec.workQueue_ID, taskSpec.gshare)
+                # get resource types
+                resource_types = self._get_resource_types()
+                if not resource_types:
+                    raise RuntimeError("failed to get resource types")
+                # nFiles from shared JobGenerator config resolver
+                tmp_params = get_params_to_get_tasks(
+                    self.tbIF, self._params_to_get_tasks, vo, prodSourceLabel, workQueue.queue_name if workQueue else "", taskSpec.cloud
+                )
+                nFiles = tmp_params["nFiles"]
+                # get inputs
+                tmp_list = self.tbIF.getTasksToBeProcessed_JEDI(self.pid, None, workQueue, None, None, nFiles=nFiles, target_tasks=[task_id])
+                if tmp_list:
+                    input_list = ListWithLock(tmp_list)
+                    # run generator inline to avoid creating an extra worker thread.
+                    siteMapper = self._get_site_mapper()
+                    setupper_key = (vo, prodSourceLabel)
+                    taskSetupper = self.task_setupper_map.get(setupper_key)
+                    if taskSetupper is None:
+                        taskSetupper = TaskSetupper(vo, prodSourceLabel)
+                        taskSetupper.initializeMods(self.tbIF, self.ddmIF)
+                        self.task_setupper_map[setupper_key] = taskSetupper
+                    # log memory usage before running job generator to help identify if this is a source of memory bloat.
+                    gen_thr = JobGeneratorThread(
+                        input_list,
+                        None,
+                        self.tbIF,
+                        self.ddmIF,
+                        siteMapper,
+                        True,
+                        taskSetupper,
+                        self.pid,
+                        workQueue,
+                        "pjmsg",
+                        None,
+                        None,
+                        None,
+                        False,
+                        resource_types,
+                    )
+                    gen_thr.run()
+                    tmp_log.info(f"task_id={task_id} generated jobs")
+                else:
+                    tmp_log.debug(f"task_id={task_id} is not considered to be processed, skipped")
+        except Exception as e:
+            err_str = f"failed to run, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        finally:
+            # Explicitly clear large per-run maps to release references as early as possible.
+            if gen_thr is not None:
+                try:
+                    gen_thr.buildSpecMap.clear()
+                    gen_thr.finished_lib_specs_map.clear()
+                    gen_thr.active_lib_specs_map.clear()
+                    gen_thr.inputList = None
+                except Exception:
+                    pass
+            gen_thr = None
+            input_list = None
+            tmp_list = None
+            taskSpec = None
+            taskSetupper = None
+            # If memory usage is above the threshold, trigger early cleanup to avoid OOM killer.
+            mem_usage = CoreUtils.getMemoryUsage()
+            if mem_usage is None:
+                tmp_log.warning("failed to get memory usage, skipped memory cleanup")
+            elif mem_usage > self._mem_usage_threshold_mb:
+                gc.collect()
+                try_malloc_trim(tmp_log)
+                new_mem_usage = CoreUtils.getMemoryUsage()
+                tmp_log.debug(f"trimmed memory usage from {mem_usage} MB to {new_mem_usage} MB")
+        # done
+        tmp_log.info("done")

pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py

@@ -0,0 +1,82 @@
+import json
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jediconfig import jedi_config
+from pandajedi.jedicore.FactoryBase import FactoryBase
+from pandajedi.jediddm.DDMInterface import DDMInterface
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+from pandajedi.jediorder.PostProcessor import PostProcessorThread
+from pandaserver.srvcore.CoreUtils import convert_config_params, parse_init_params
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# Jedi Post-Processor message processor plugin
+class JediPostProcessorMsgProcPlugin(BaseMsgProcPlugin):
+    """
+    Message-driven Post-Processor
+    """
+
+    def initialize(self):
+        BaseMsgProcPlugin.initialize(self)
+        # DDM interface
+        ddmIF = DDMInterface()
+        ddmIF.setupInterface()
+        # factory bases and post processor thread object for all vos and prodsourcelabels
+        self.post_processor_thread_dict = dict()
+        for itemStr in jedi_config.postprocessor.procConfig.split(";"):
+            items = convert_config_params(itemStr)
+            vos = parse_init_params(items[0])
+            prodsourcelabels = parse_init_params(items[1])
+            tmp_factory_base_obj = FactoryBase(vos=vos, sourceLabels=prodsourcelabels, logger=base_logger, modConfig=jedi_config.postprocessor.modConfig)
+            tmp_post_processor_thread_obj = PostProcessorThread(
+                taskList=None, threadPool=None, taskbufferIF=self.tbIF, ddmIF=ddmIF, implFactory=tmp_factory_base_obj
+            )
+            for vo in vos:
+                for prodsourcelabel in prodsourcelabels:
+                    self.post_processor_thread_dict[(vo, prodsourcelabel)] = tmp_post_processor_thread_obj
+
+    def process(self, msg_obj):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+        # parse json
+        try:
+            msg_dict = json.loads(msg_obj.data)
+        except Exception as e:
+            err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # sanity check
+        try:
+            msg_type = msg_dict["msg_type"]
+        except Exception as e:
+            err_str = f"failed to parse message object dict {msg_dict} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        if msg_type != "jedi_post_processor":
+            # FIXME
+            err_str = f"got unknown msg_type {msg_type} , skipped "
+            tmp_log.error(err_str)
+            raise
+        # run
+        try:
+            task_id = msg_dict["taskid"]
+            vo = msg_dict["task_vo"]
+            prodsourcelabel = msg_dict["task_prodsourcelabel"]
+            ret_list = self.tbIF.prepareTasksToBeFinished_JEDI(vo, prodSourceLabel, jedi_config.postprocessor.nTasks, self.get_pid())
+            task_list = self.tbIF.getTasksToBeFinished_JEDI(vo, prodSourceLabel, self.get_pid(), jedi_config.postprocessor.nTasks, target_tasks=ret_list)
+            if task_list and task_id in [task_spec.jediTaskID for task_spec in task_list]:
+                tmp_post_processor_thread_obj = self.post_processor_thread_dict[(vo, prodsourcelabel)]
+                tmp_post_processor_thread_obj.post_process_tasks(task_list)
+                tmp_log.info(f"post processed tasks {task_list} including {task_id}")
+            else:
+                tmp_log.debug(f"did not get task {task_id}; skip ")
+        except Exception as e:
+            err_str = f"failed to run, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # done
+        tmp_log.info("done")

pandajedi/jedimsgprocessor/kafka_msg_processor.py

@@ -0,0 +1,44 @@
+import json
+
+from pandacommon.kafkapublisher.KafkaPublisher import KafkaPublisher
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# Kafka message processing plugin
+class KafkaMsgProcPlugin(BaseMsgProcPlugin):
+    def initialize(self):
+        """
+        initialize plugin instance, run once before loop in thread
+        """
+        self.publisher = KafkaPublisher()
+
+    def process(self, msg_obj, decoded_data=None):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+
+        # start
+        tmp_log.info("start")
+        tmp_log.debug(f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}")
+
+        # Parse and access the message content from msg_obj.data
+        if decoded_data is None:
+            # json decode
+            try:
+                message_content = json.loads(msg_obj.data)
+            except Exception as e:
+                err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+                tmp_log.error(err_str)
+                raise
+        else:
+            message_content = decoded_data
+
+        # Publish the message to Kafka
+        self.publisher.publish_message(message_content)
+        tmp_log.debug(f"sent {message_content}")
+        tmp_log.info("done")
+
+    def terminate(self):
+        self.publisher.close()

pandajedi/jedimsgprocessor/panda_callback_msg_processor.py

@@ -0,0 +1,182 @@
+import re
+import traceback
+
+import yaml
+from pandacommon.pandalogger import LogWrapper, logger_utils
+
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+from pandaserver.dataservice.ddm_handler import DDMHandler
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# panda dataset callback message processing plugin
+class PandaCallbackMsgProcPlugin(BaseMsgProcPlugin):
+
+    def __init__(self, **params):
+        super().__init__(**params)
+        self.activities_with_file_callback = []
+        self.component_action_map = []
+        self.site_mapper = None
+        self.verbose = False
+
+    def initialize(self, **params):
+        BaseMsgProcPlugin.initialize(self, **params)
+        # activity list to use file callback
+        self.activities_with_file_callback = self.params.get("activities_with_file_callback", [])
+        # component action map
+        # [{"event_type": "<event_type>", "component": "<component_name>", "criteria": {"key": <value>,}, "to_id": "<how_to_get_task_id>}, ...]
+        self.component_action_map = self.params.get("component_action_map", [])
+        # site mapper
+        self.site_mapper = self.tbIF.get_site_mapper()
+        # verbose logging
+        self.verbose = self.params.get("verbose", False)
+
+    def process(self, msg_obj):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        # tmp_log.info('start')
+        # tmp_log.debug('sub_id={0} ; msg_id={1}'.format(msg_obj.sub_id, msg_obj.msg_id))
+        # parse yaml
+        try:
+            message_dict = yaml.safe_load(msg_obj.data)
+        except Exception as e:
+            err_str = f"failed to parse message yaml {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # run
+        try:
+            to_continue = True
+            dsn = "UNKNOWN"
+            # check event type
+            if not isinstance(message_dict, dict):
+                err_str = f"skip due to invalid message format:{type(message_dict).__name__}. msg:{str(message_dict)}"
+                tmp_log.warning(err_str)
+                return
+            event_type = message_dict["event_type"]
+            message_ids = f"sub_id={msg_obj.sub_id} ; msg_id={msg_obj.msg_id}"
+            if event_type in ["datasetlock_ok"]:
+                self.process_dataset_callback(event_type, message_ids, message_dict, tmp_log)
+            elif self.activities_with_file_callback and event_type in ["transfer-done"]:
+                self.process_file_callback(event_type, message_ids, message_dict, tmp_log)
+            else:
+                if self.verbose:
+                    tmp_log.debug(f"skip event_type={event_type}")
+            # trigger component actions
+            if self.component_action_map:
+                self.trigger_component_action(event_type, message_ids, message_dict, tmp_log)
+            # end
+        except Exception as e:
+            err_str = f"failed to run, skipped. {e.__class__.__name__} : {e}\n{traceback.format_exc()}"
+            tmp_log.error(err_str)
+            raise
+
+    def process_dataset_callback(self, event_type: str, message_ids: str, message_dict: dict, tmp_log: LogWrapper.LogWrapper) -> None:
+        """
+        Process a dataset callback
+
+        Args:
+            event_type: message event type
+            message_ids: subscription and message IDs
+            message_dict: message dictionary
+            tmp_log: logger instance
+        """
+        message_payload = message_dict["payload"]
+        # only for _dis or _sub
+        dsn = message_payload["name"]
+        if (re.search(r"_dis\d+$", dsn) is None) and (re.search(r"_sub\d+$", dsn) is None):
+            return
+        tmp_log.debug(message_ids)
+        tmp_log.debug(f"{event_type} start")
+        # take action
+        scope = message_payload["scope"]
+        site = message_payload["rse"]
+        tmp_log.debug(f"{dsn} site={site} type={event_type}")
+        thr = DDMHandler(task_buffer=self.tbIF, vuid=None, site=site, dataset=dsn, scope=scope)
+        # just call run rather than start+join, to run it in main thread instead of spawning new thread
+        thr.run()
+        del thr
+        tmp_log.debug(f"done {dsn}")
+        return
+
+    def process_file_callback(self, event_type: str, message_ids: str, message_dict: dict, tmp_log: LogWrapper.LogWrapper) -> None:
+        """
+        Process a file callback
+
+        Args:
+            event_type: message event type
+            message_ids: subscription and message IDs
+            message_dict: message dictionary
+            tmp_log: logger instance
+        """
+        message_payload = message_dict["payload"]
+        # only for activities with file callback
+        activity = message_payload["activity"]
+        if activity not in self.activities_with_file_callback:
+            return
+        # update file status and get corresponding PandaIDs
+        filename = message_payload["name"]
+        endpoint = message_payload["dst-rse"]
+        tmp_log.debug(message_ids)
+        tmp_log.debug(f"{event_type} start for lfn={filename} activity={activity}")
+        sites = self.site_mapper.get_sites_for_endpoint(endpoint, "input")
+        panda_ids = self.tbIF.update_input_files_at_sites_and_get_panda_ids(filename, sites)
+        jobs = self.tbIF.peekJobs(panda_ids, fromActive=False, fromArchived=False, fromWaiting=False)
+        # activate jobs
+        self.tbIF.activateJobs(jobs)
+        tmp_log.debug(f"done")
+        return
+
+    def trigger_component_action(self, event_type: str, message_ids: str, message_dict: dict, tmp_log: LogWrapper.LogWrapper) -> None:
+        """
+        Trigger component action based on the event type
+        Args:
+            event_type: message event type
+            message_ids: subscription and message IDs
+            message_dict: message dictionary
+            tmp_log: logger instance
+        """
+        message_payload = message_dict["payload"]
+        for action_item in self.component_action_map:
+            # check event type
+            if action_item["event_type"] != event_type:
+                continue
+            # check criteria
+            criteria_matched = True
+            for key, value in action_item.get("criteria", {}).items():
+                dict_value = message_payload.get(key)
+                if dict_value is None:
+                    criteria_matched = False
+                    break
+                if dict_value != value and not re.match(value, str(dict_value)):
+                    criteria_matched = False
+                    break
+            if not criteria_matched:
+                continue
+            component_name = action_item["component"]
+            to_id = action_item["to_id"]
+            # extract task ID based on to_id
+            jedi_task_ids = None
+            if to_id == "from_input_dataset":
+                dataset_name = message_payload["name"].split(":")[-1]
+                jedi_task_ids = self.tbIF.get_task_ids_with_dataset_attributes({"datasetName": dataset_name, "type": "input"})
+            else:
+                tmp_log.warning(f"unknown to_id={to_id} for action_item={action_item} ; skipped")
+                continue
+            if jedi_task_ids is None:
+                tmp_log.warning(f"failed to extract jediTaskID for action_item={action_item} ; skipped")
+                continue
+            if not jedi_task_ids:
+                tmp_log.debug(f"no jediTaskID found for action_item={action_item} ; skipped")
+                continue
+            # loop over task IDs
+            for jedi_task_id in jedi_task_ids:
+                # release task just in case
+                self.tbIF.release_task_on_hold(jedi_task_id)
+                # push trigger message
+                push_ret = self.tbIF.push_task_trigger_message(component_name, jedi_task_ids)
+                if push_ret:
+                    tmp_log.debug(f"pushed trigger message to {component_name} for jediTaskID={jedi_task_ids}")
+                else:
+                    tmp_log.warning(f"failed to push trigger to {component_name} for jediTaskID={jedi_task_ids}")
+        return

pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py

@@ -0,0 +1,74 @@
+import json
+import os
+import socket
+
+from pandacommon.pandalogger import logger_utils
+
+from pandajedi.jedicore.ThreadUtils import ListWithLock, ThreadPool
+from pandajedi.jediddm.DDMInterface import DDMInterface
+from pandajedi.jedimsgprocessor.base_msg_processor import BaseMsgProcPlugin
+from pandajedi.jediorder.JobGenerator import JobGeneratorThread
+from pandajedi.jediorder.TaskSetupper import TaskSetupper
+
+base_logger = logger_utils.setup_logger(__name__.split(".")[-1])
+
+
+# plugin to process messages from Panda to JEDI
+class PandaToJediMsgProcPlugin(BaseMsgProcPlugin):
+    def initialize(self):
+        BaseMsgProcPlugin.initialize(self)
+        self.ddmIF = DDMInterface()
+        self.ddmIF.setupInterface()
+        self.pid = f"{socket.getfqdn().split('.')[0]}-{os.getpid()}_{os.getpgrp()}-pjmsg"
+
+    def process(self, msg_obj, decoded_data=None):
+        tmp_log = logger_utils.make_logger(base_logger, token=self.get_pid(), method_name="process")
+        # start
+        tmp_log.info("start")
+        # parse
+        if decoded_data is None:
+            # json decode
+            try:
+                msg_dict = json.loads(msg_obj.data)
+            except Exception as e:
+                err_str = f"failed to parse message json {msg_obj.data} , skipped. {e.__class__.__name__} : {e}"
+                tmp_log.error(err_str)
+                raise
+        else:
+            msg_dict = decoded_data
+        # run
+        try:
+            tmp_log.debug(f"got message {msg_dict}")
+            if msg_dict["msg_type"] == "generate_job":
+                # get task to generate jobs
+                jediTaskID = int(msg_dict["taskid"])
+                s, taskSpec = self.tbIF.getTaskWithID_JEDI(jediTaskID)
+                if not taskSpec:
+                    tmp_log.debug(f"unknown task {jediTaskID}")
+                else:
+                    # get WQ
+                    vo = taskSpec.vo
+                    prodSourceLabel = taskSpec.prodSourceLabel
+                    workQueue = self.tbIF.getWorkQueueMap().getQueueWithIDGshare(taskSpec.workQueue_ID, taskSpec.gshare)
+                    # get inputs
+                    tmpList = self.tbIF.getTasksToBeProcessed_JEDI(self.pid, None, workQueue, None, None, nFiles=1000, target_tasks=[jediTaskID])
+                    if tmpList:
+                        inputList = ListWithLock(tmpList)
+                        # create thread
+                        threadPool = ThreadPool()
+                        siteMapper = self.tbIF.get_site_mapper()
+                        taskSetupper = TaskSetupper(vo, prodSourceLabel)
+                        taskSetupper.initializeMods(self.tbIF, self.ddmIF)
+                        gen = JobGeneratorThread(
+                            inputList, threadPool, self.tbIF, self.ddmIF, siteMapper, True, taskSetupper, self.pid, workQueue, "pjmsg", None, None, None, False
+                        )
+                        gen.start()
+                        gen.join()
+            else:
+                tmp_log.debug(f"unknown message type : {msg_dict['msg_type']}")
+        except Exception as e:
+            err_str = f"failed to run, skipped. {e.__class__.__name__} : {e}"
+            tmp_log.error(err_str)
+            raise
+        # done
+        tmp_log.info("done")