flowcept 0.8.3__py3-none-any.whl → 0.8.5__py3-none-any.whl

flowcept/commons/autoflush_buffer.py

@@ -9,13 +9,14 @@ class AutoflushBuffer:
 
     def __init__(
         self,
-        max_size,
-        flush_interval,
         flush_function: Callable,
+        max_size=None,
+        flush_interval=None,
         flush_function_args=[],
         flush_function_kwargs={},
     ):
-        self._max_size = max_size
+        self._max_size = max_size or float("inf")
+        self._flush_interval = flush_interval or float("inf")
         self._flush_interval = flush_interval
         self._buffers = [[], []]
         self._current_buffer_index = 0

flowcept/commons/daos/mq_dao/mq_dao_base.py

@@ -2,9 +2,9 @@
 
 from abc import ABC, abstractmethod
 from typing import Union, List, Callable
-
+import csv
 import msgpack
-
+from time import time
 import flowcept.commons
 from flowcept.commons.autoflush_buffer import AutoflushBuffer
 
@@ -19,6 +19,7 @@ from flowcept.configs import (
     MQ_INSERTION_BUFFER_TIME,
     MQ_CHUNK_SIZE,
     MQ_TYPE,
+    MQ_TIMING,
 )
 
 from flowcept.commons.utils import GenericJSONEncoder
@@ -69,12 +70,21 @@ class MQDao(ABC):
         self._keyvalue_dao = KeyValueDAO()
         self._time_based_flushing_started = False
         self.buffer: Union[AutoflushBuffer, List] = None
-        self._flush_events = []
+        if MQ_TIMING:
+            self._flush_events = []
+            self.stop = self._stop_timed
+            self.send_message = self._send_message_timed
+            self._bulk_publish = self._bulk_publish_timed
+        else:
+            self.stop = self._stop
 
     @abstractmethod
     def _bulk_publish(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
         raise NotImplementedError()
 
+    def _bulk_publish_timed(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
+        raise NotImplementedError()
+
     def bulk_publish(self, buffer):
         """Publish it."""
         # self.logger.info(f"Going to flush {len(buffer)} to MQ...")
@@ -132,12 +142,10 @@ class MQDao(ABC):
         """Create the buffer."""
         if not self.started:
             if flowcept.configs.DB_FLUSH_MODE == "online":
-                # msg = "Starting MQ time-based flushing! bundle: "
-                # self.logger.debug(msg+f"{exec_bundle_id}; interceptor id: {interceptor_instance_id}")
                 self.buffer = AutoflushBuffer(
+                    flush_function=self.bulk_publish,
                     max_size=MQ_BUFFER_SIZE,
                     flush_interval=MQ_INSERTION_BUFFER_TIME,
-                    flush_function=self.bulk_publish,
                 )
                 self.register_time_based_thread_init(interceptor_instance_id, exec_bundle_id)
                 self._time_based_flushing_started = True
@@ -156,7 +164,18 @@ class MQDao(ABC):
             self.bulk_publish(self.buffer)
             self.buffer = list()
 
-    def stop(self, interceptor_instance_id: str, bundle_exec_id: int = None):
+    def _stop_timed(self, interceptor_instance_id: str, bundle_exec_id: int = None):
+        t1 = time()
+        self._stop(interceptor_instance_id, bundle_exec_id)
+        t2 = time()
+        self._flush_events.append(["final", t1, t2, t2 - t1, "n/a"])
+
+        with open(f"{MQ_TYPE}_{interceptor_instance_id}_{MQ_TYPE}_flush_events.csv", "w", newline="") as file:
+            writer = csv.writer(file)
+            writer.writerow(["type", "start", "end", "duration", "size"])
+            writer.writerows(self._flush_events)
+
+    def _stop(self, interceptor_instance_id: str, bundle_exec_id: int = None):
         """Stop it."""
         msg0 = "MQ publisher received stop signal! bundle: "
         self.logger.debug(msg0 + f"{bundle_exec_id}; interceptor id: {interceptor_instance_id}")
@@ -189,6 +208,11 @@ class MQDao(ABC):
         """Send a message."""
         raise NotImplementedError()
 
+    @abstractmethod
+    def _send_message_timed(self, message: dict, channel=MQ_CHANNEL, serializer=msgpack.dumps):
+        """Send a message."""
+        raise NotImplementedError()
+
     @abstractmethod
     def message_listener(self, message_handler: Callable):
         """Get message listener."""

flowcept/commons/daos/mq_dao/mq_dao_kafka.py

@@ -1,18 +1,15 @@
 """MQ kafka module."""
 
 from typing import Callable
-
-import msgpack
 from time import time
+import msgpack
 
 from confluent_kafka import Producer, Consumer, KafkaError
 from confluent_kafka.admin import AdminClient
 
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
-from flowcept.commons.utils import perf_log
 from flowcept.configs import (
     MQ_CHANNEL,
-    PERF_LOG,
     MQ_HOST,
     MQ_PORT,
 )
@@ -69,26 +66,44 @@ class MQDaoKafka(MQDao):
         self._producer.produce(channel, key=channel, value=serializer(message))
         self._producer.flush()
 
+    def _send_message_timed(self, message: dict, channel=MQ_CHANNEL, serializer=msgpack.dumps):
+        t1 = time()
+        self.send_message(message, channel, serializer)
+        t2 = time()
+        self._flush_events.append(["single", t1, t2, t2 - t1, len(str(message).encode())])
+
     def _bulk_publish(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
+        for message in buffer:
+            try:
+                self._producer.produce(channel, key=channel, value=serializer(message))
+            except Exception as e:
+                self.logger.exception(e)
+                self.logger.error("Some messages couldn't be flushed! Check the messages' contents!")
+                self.logger.error(f"Message that caused error: {message}")
+        try:
+            self._producer.flush()
+            self.logger.info(f"Flushed {len(buffer)} msgs to MQ!")
+        except Exception as e:
+            self.logger.exception(e)
+
+    def _bulk_publish_timed(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
         total = 0
         for message in buffer:
             try:
-                self.logger.debug(f"Going to send Message:\n\t[BEGIN_MSG]{message}\n[END_MSG]\t")
                 self._producer.produce(channel, key=channel, value=serializer(message))
                 total += len(str(message).encode())
             except Exception as e:
                 self.logger.exception(e)
                 self.logger.error("Some messages couldn't be flushed! Check the messages' contents!")
                 self.logger.error(f"Message that caused error: {message}")
-        t0 = 0
-        if PERF_LOG:
-            t0 = time()
         try:
+            t1 = time()
             self._producer.flush()
+            t2 = time()
+            self._flush_events.append(["bulk", t1, t2, t2 - t1, total])
             self.logger.info(f"Flushed {len(buffer)} msgs to MQ!")
         except Exception as e:
             self.logger.exception(e)
-        perf_log("mq_pipe_flush", t0)
 
     def liveness_test(self):
         """Get the livelyness of it."""

flowcept/commons/daos/mq_dao/mq_dao_mofka.py

@@ -9,8 +9,7 @@ import mochi.mofka.client as mofka
 from mochi.mofka.client import ThreadPool, AdaptiveBatchSize
 
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
-from flowcept.commons.utils import perf_log
-from flowcept.configs import PERF_LOG, MQ_SETTINGS, MQ_CHANNEL
+from flowcept.configs import MQ_SETTINGS, MQ_CHANNEL
 
 
 class MQDaoMofka(MQDao):
@@ -58,7 +57,29 @@ class MQDaoMofka(MQDao):
         self.producer.push(metadata=message)  # using metadata to send data
         self.producer.flush()
 
+    def _send_message_timed(self, message: dict, channel=MQ_CHANNEL, serializer=msgpack.dumps):
+        t1 = time()
+        self.send_message(message, channel, serializer)
+        t2 = time()
+        self._flush_events.append(["single", t1, t2, t2 - t1, len(str(message).encode())])
+
     def _bulk_publish(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
+        try:
+            self.logger.debug(f"Going to send Message:\n\t[BEGIN_MSG]{buffer}\n[END_MSG]\t")
+            for m in buffer:
+                self.producer.push(m)
+
+        except Exception as e:
+            self.logger.exception(e)
+            self.logger.error("Some messages couldn't be flushed! Check the messages' contents!")
+            self.logger.error(f"Message that caused error: {buffer}")
+        try:
+            self.producer.flush()
+            self.logger.info(f"Flushed {len(buffer)} msgs to MQ!")
+        except Exception as e:
+            self.logger.exception(e)
+
+    def _bulk_publish_timed(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
         total = 0
         try:
             self.logger.debug(f"Going to send Message:\n\t[BEGIN_MSG]{buffer}\n[END_MSG]\t")
@@ -71,15 +92,14 @@ class MQDaoMofka(MQDao):
             self.logger.exception(e)
             self.logger.error("Some messages couldn't be flushed! Check the messages' contents!")
             self.logger.error(f"Message that caused error: {buffer}")
-        t0 = 0
-        if PERF_LOG:
-            t0 = time()
         try:
+            t1 = time()
             self.producer.flush()
+            t2 = time()
+            self._flush_events.append(["bulk", t1, t2, t2 - t1, total])
             self.logger.info(f"Flushed {len(buffer)} msgs to MQ!")
         except Exception as e:
             self.logger.exception(e)
-        perf_log("mq_pipe_flush", t0)
 
     def liveness_test(self):
         """Test Mofka Liveness."""

flowcept/commons/daos/mq_dao/mq_dao_redis.py

@@ -7,11 +7,7 @@ import msgpack
 from time import time, sleep
 
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
-from flowcept.commons.utils import perf_log
-from flowcept.configs import (
-    MQ_CHANNEL,
-    PERF_LOG,
-)
+from flowcept.configs import MQ_CHANNEL
 
 
 class MQDaoRedis(MQDao):
@@ -62,26 +58,47 @@ class MQDaoRedis(MQDao):
         """Send the message."""
         self._producer.publish(channel, serializer(message))
 
+    def _send_message_timed(self, message: dict, channel=MQ_CHANNEL, serializer=msgpack.dumps):
+        """Send the message using timing for performance evaluation."""
+        t1 = time()
+        self.send_message(message, channel, serializer)
+        t2 = time()
+        self._flush_events.append(["single", t1, t2, t2 - t1, len(str(message).encode())])
+
     def _bulk_publish(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
+        pipe = self._producer.pipeline()
+        for message in buffer:
+            try:
+                pipe.publish(channel, serializer(message))
+            except Exception as e:
+                self.logger.exception(e)
+                self.logger.error("Some messages couldn't be flushed! Check the messages' contents!")
+                self.logger.error(f"Message that caused error: {message}")
+        try:
+            pipe.execute()
+            self.logger.debug(f"Flushed {len(buffer)} msgs to MQ!")
+        except Exception as e:
+            self.logger.exception(e)
+
+    def _bulk_publish_timed(self, buffer, channel=MQ_CHANNEL, serializer=msgpack.dumps):
         total = 0
         pipe = self._producer.pipeline()
         for message in buffer:
             try:
                 total += len(str(message).encode())
-                pipe.publish(MQ_CHANNEL, serializer(message))
+                pipe.publish(channel, serializer(message))
             except Exception as e:
                 self.logger.exception(e)
                 self.logger.error("Some messages couldn't be flushed! Check the messages' contents!")
                 self.logger.error(f"Message that caused error: {message}")
-        t0 = 0
-        if PERF_LOG:
-            t0 = time()
         try:
+            t1 = time()
             pipe.execute()
+            t2 = time()
+            self._flush_events.append(["bulk", t1, t2, t2 - t1, total])
             self.logger.debug(f"Flushed {len(buffer)} msgs to MQ!")
         except Exception as e:
             self.logger.exception(e)
-        perf_log("mq_pipe_execute", t0)
 
     def liveness_test(self):
         """Get the livelyness of it."""

flowcept/commons/flowcept_dataclasses/workflow_object.py

@@ -12,6 +12,7 @@ from flowcept.configs import (
     SYS_NAME,
     EXTRA_METADATA,
     ENVIRONMENT_ID,
+    SETTINGS_PATH,
 )
 
 
@@ -23,6 +24,7 @@ class WorkflowObject:
     workflow_id: AnyStr = None
     parent_workflow_id: AnyStr = None
     machine_info: Dict = None
+    conf: Dict = None
     flowcept_settings: Dict = None
     flowcept_version: AnyStr = None
     utc_timestamp: float = None
@@ -70,7 +72,7 @@ class WorkflowObject:
         """Enrich it."""
         self.utc_timestamp = get_utc_now()
         self.flowcept_settings = OmegaConf.to_container(settings) if isinstance(settings, DictConfig) else settings
-
+        self.conf = {"settings_path": SETTINGS_PATH}
         if adapter_key is not None:
             # TODO :base-interceptor-refactor: :code-reorg: :usability:
             # revisit all times we assume settings is not none

flowcept/configs.py

@@ -73,9 +73,9 @@ MQ_CHANNEL = settings["mq"].get("channel", "interception")
 MQ_PASSWORD = settings["mq"].get("password", None)
 MQ_HOST = os.getenv("MQ_HOST", settings["mq"].get("host", "localhost"))
 MQ_PORT = int(os.getenv("MQ_PORT", settings["mq"].get("port", "6379")))
-
-MQ_BUFFER_SIZE = int(settings["mq"].get("buffer_size", 50))
-MQ_INSERTION_BUFFER_TIME = int(settings["mq"].get("insertion_buffer_time_secs", 5))
+MQ_BUFFER_SIZE = settings["mq"].get("buffer_size", None)
+MQ_INSERTION_BUFFER_TIME = settings["mq"].get("insertion_buffer_time_secs", None)
+MQ_TIMING = settings["mq"].get("timing", False)
 MQ_CHUNK_SIZE = int(settings["mq"].get("chunk_size", -1))
 
 #####################
@@ -127,7 +127,7 @@ if not LMDB_ENABLED and not MONGO_ENABLED:
 ##########################
 db_buffer_settings = settings["db_buffer"]
 # In seconds:
-INSERTION_BUFFER_TIME = int(db_buffer_settings.get("insertion_buffer_time_secs", 5))
+INSERTION_BUFFER_TIME = db_buffer_settings.get("insertion_buffer_time_secs", None)
 ADAPTIVE_DB_BUFFER_SIZE = db_buffer_settings.get("adaptive_buffer_size", True)
 DB_MAX_BUFFER_SIZE = int(db_buffer_settings.get("max_buffer_size", 50))
 DB_MIN_BUFFER_SIZE = max(1, int(db_buffer_settings.get("min_buffer_size", 10)))

flowcept/flowcept_api/flowcept_controller.py

@@ -262,6 +262,7 @@ class Flowcept(object):
         if not MQDao.build().liveness_test():
             logger.error("MQ Not Ready!")
             return False
+
         if MONGO_ENABLED:
             from flowcept.commons.daos.docdb_dao.mongodb_dao import MongoDBDAO
 

flowcept/flowceptor/adapters/dask/dask_interceptor.py

@@ -21,32 +21,6 @@ from flowcept.flowceptor.adapters.instrumentation_interceptor import Instrumenta
 
 def get_run_spec_data(task_msg: TaskObject, run_spec):
     """Get the run specs."""
-    # def _get_arg(arg_name):
-    #     if type(run_spec) == dict:
-    #         return run_spec.get(arg_name, None)
-    #     elif hasattr(run_spec, arg_name):
-    #         return getattr(run_spec, arg_name)
-    #     return None
-    #
-    # def _parse_dask_tuple(_tuple: tuple):
-    #     forth_elem = None
-    #     if len(_tuple) == 3:
-    #         _, _, value_tuple = _tuple
-    #     elif len(_tuple) == 4:
-    #         _, _, value_tuple, forth_elem = _tuple
-    #
-    #     _, value = value_tuple
-    #     if len(value) == 1:  # Value is always an array here
-    #         value = value[0]
-    #     ret_obj = {"value": value}
-    #
-    #     if forth_elem is not None and type(forth_elem) == dict:
-    #         ret_obj.update(forth_elem)
-    #     else:
-    #         pass  # We don't know yet what to do if this happens. So just pass.
-    #
-    #     return ret_obj
-
     func = run_spec[0]
     task_msg.activity_id = func.__name__
     args = run_spec[1]
@@ -91,7 +65,7 @@ class DaskWorkerInterceptor(BaseInterceptor):
         self._plugin_key = plugin_key
         self._worker = None
         self.kind = kind
-        # super().__init__ goes to setup_worker.
+        # super().__init__ goes to setup_worker. I'm leaving this commented here as a reminder.
 
     def setup_worker(self, worker):
         """Set the worker.
@@ -101,10 +75,13 @@ class DaskWorkerInterceptor(BaseInterceptor):
         """
         self._worker = worker
         super().__init__(plugin_key=self._plugin_key, kind=self.kind)
-        # TODO: :refactor: Below is just to avoid the auto-generation of workflow id, which doesnt make sense in Dask.
+        self._worker.flowcept_tasks = {}
+        # TODO: :refactor: Below is just to avoid the auto-generation of workflow id, which doesn't make sense in Dask.
         self._generated_workflow_id = True
         super().start(bundle_exec_id=self._worker.scheduler.address)
 
+        self._worker._interceptor = self
+
         instrumentation = INSTRUMENTATION.get("enabled", False)
         if instrumentation:
             InstrumentationInterceptor.get_instance().start(
@@ -129,6 +106,8 @@ class DaskWorkerInterceptor(BaseInterceptor):
                 task_msg.address = self._worker.worker_address
                 if self.settings.worker_create_timestamps:
                     task_msg.started_at = get_utc_now()
+
+                self._worker.flowcept_tasks[task_id] = task_msg
             elif ts.state == "memory":
                 task_msg.status = Status.FINISHED
                 if self.settings.worker_create_timestamps:

flowcept/flowceptor/adapters/dask/dask_plugins.py

@@ -1,8 +1,11 @@
 """Dask plugin module."""
 
+from typing import Optional
+
 from distributed import Client, WorkerPlugin
 
 from flowcept import WorkflowObject
+from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
 from flowcept.configs import INSTRUMENTATION
 from flowcept.flowceptor.adapters.dask.dask_interceptor import (
     DaskWorkerInterceptor,
@@ -10,38 +13,6 @@ from flowcept.flowceptor.adapters.dask.dask_interceptor import (
 from flowcept.flowceptor.adapters.instrumentation_interceptor import InstrumentationInterceptor
 
 
-# def _set_workflow_on_scheduler(
-#     dask_scheduler=None,
-#     workflow_id=None,
-#     custom_metadata: dict = None,
-#     campaign_id: str = None,
-#     workflow_name: str = None,
-#     used: dict = None,
-# ):
-#     custom_metadata = custom_metadata or {}
-#     wf_obj = WorkflowObject()
-#     wf_obj.workflow_id = workflow_id
-#     custom_metadata.update(
-#         {
-#             "workflow_type": "DaskWorkflow",
-#             "scheduler": dask_scheduler.address_safe,
-#             "scheduler_id": dask_scheduler.id,
-#             "scheduler_pid": dask_scheduler.proc.pid,
-#             "clients": len(dask_scheduler.clients),
-#             "n_workers": len(dask_scheduler.workers),
-#         }
-#     )
-#     wf_obj.custom_metadata = custom_metadata
-#     wf_obj.used = used
-#     wf_obj.campaign_id = campaign_id
-#     wf_obj.name = workflow_name
-#
-#     interceptor = BaseInterceptor(plugin_key="dask")
-#     interceptor.start(bundle_exec_id=dask_scheduler.address)
-#     interceptor.send_workflow_message(wf_obj)
-#     interceptor.stop()
-
-
 def _set_workflow_on_workers(dask_worker, workflow_id, campaign_id=None):
     setattr(dask_worker, "current_workflow_id", workflow_id)
     if campaign_id:
@@ -53,6 +24,23 @@ def set_workflow_info_on_workers(dask_client: Client, wf_obj: WorkflowObject):
     dask_client.run(_set_workflow_on_workers, workflow_id=wf_obj.workflow_id, campaign_id=wf_obj.campaign_id)
 
 
+def get_flowcept_task() -> Optional[TaskObject]:
+    """Get the Flowcept Task Object inside a Worker's task."""
+    from distributed import get_worker
+    from distributed.worker import thread_state
+
+    worker = get_worker()
+    try:
+        task_id = thread_state.key if hasattr(thread_state, "key") else None
+        if hasattr(worker, "flowcept_tasks") and task_id in worker.flowcept_tasks:
+            return worker.flowcept_tasks[task_id]
+        else:
+            return None
+    except Exception as e:
+        print(e)
+        return None
+
+
 class FlowceptDaskWorkerAdapter(WorkerPlugin):
     """Dask worker adapter."""
 

flowcept/flowceptor/consumers/consumer_utils.py

@@ -45,13 +45,11 @@ def curate_task_msg(task_msg_dict: dict, convert_times=True):
         task_msg_dict["workflow_id"] = task_msg_dict["used"].pop("workflow_id")
 
     if convert_times:
-        has_time_fields = False
         for time_field in TaskObject.get_time_field_names():
             if time_field in task_msg_dict:
-                has_time_fields = True
                 task_msg_dict[time_field] = datetime.fromtimestamp(task_msg_dict[time_field], pytz.utc)
 
-        if not has_time_fields:
+        if "registered_at" not in task_msg_dict:
             task_msg_dict["registered_at"] = datetime.fromtimestamp(time(), pytz.utc)
 
 

flowcept/flowceptor/consumers/document_inserter.py

@@ -71,10 +71,10 @@ class DocumentInserter:
         self._bundle_exec_id = bundle_exec_id
         self.check_safe_stops = check_safe_stops
         self.buffer: AutoflushBuffer = AutoflushBuffer(
-            max_size=self._curr_max_buffer_size,
-            flush_interval=INSERTION_BUFFER_TIME,
             flush_function=DocumentInserter.flush_function,
             flush_function_kwargs={"logger": self.logger, "doc_daos": self._doc_daos},
+            max_size=self._curr_max_buffer_size,
+            flush_interval=INSERTION_BUFFER_TIME,
         )
 
     def _set_buffer_size(self):

flowcept/flowceptor/telemetry_capture.py

@@ -119,8 +119,8 @@ class GPUCapture:
         if "name" in gpu_conf:
             flowcept_gpu_info["name"] = nvmlDeviceGetName(device)
 
-        if "ix" in gpu_conf:
-            flowcept_gpu_info["gpu_ix"] = gpu_ix
+        if "id" in gpu_conf:
+            flowcept_gpu_info["id"] = nvmlDeviceGetUUID(device)
 
         return flowcept_gpu_info
 
@@ -160,7 +160,6 @@ class GPUCapture:
             }
         if "others" in gpu_conf:
             flowcept_gpu_info["others"] = {
-                "uuid": amdsmi_get_gpu_device_uuid(device),
                 "current_gfxclk": all_metrics["current_gfxclk"],
                 "current_socclk": all_metrics["current_socclk"],
                 "current_uclk": all_metrics["current_uclk"],
@@ -168,6 +167,9 @@ class GPUCapture:
                 "current_dclk0": all_metrics["current_dclk0"],
             }
 
+        if "id" in gpu_conf:
+            flowcept_gpu_info["id"] = (amdsmi_get_gpu_device_uuid(device),)
+
         return flowcept_gpu_info
 
 
@@ -193,6 +195,7 @@ elif GPUCapture.GPU_VENDOR == "nvidia":
         nvmlDeviceGetTemperature,
         nvmlDeviceGetPowerUsage,
         NVML_TEMPERATURE_GPU,
+        nvmlDeviceGetUUID,
     )
 
     FlowceptLogger().debug("Imported Nvidia modules!")

flowcept/instrumentation/flowcept_loop.py

@@ -156,13 +156,13 @@ class FlowceptLoop:
             "used": {"i": self._next_counter, self._item_name: self._current_item},
             "parent_task_id": self._parent_task_id,
         }
-        tel = FlowceptLoop._interceptor.telemetry_capture.capture()
-        if tel:
-            iteration_task["telemetry_at_start"] = tel.to_dict()
         return iteration_task
 
-    def _end_iteration_task(self, iteration_task):
-        iteration_task["status"] = Status.FINISHED.value
+    def _end_iteration_task(self, _):
+        self._last_iteration_task["status"] = Status.FINISHED.value
+        tel = FlowceptLoop._interceptor.telemetry_capture.capture()
+        if tel:
+            self._last_iteration_task["telemetry_at_end"] = tel.to_dict()
         FlowceptLoop._interceptor.intercept(self._last_iteration_task)
 
     def _do_nothing_in_end_iter(self, *args, **kwargs):

flowcept/version.py

@@ -4,4 +4,4 @@
 # The expected format is: <Major>.<Minor>.<Patch>
 # This file is supposed to be automatically modified by the CI Bot.
 # See .github/workflows/version_bumper.py
-__version__ = "0.8.3"
+__version__ = "0.8.5"

{flowcept-0.8.3.dist-info → flowcept-0.8.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.8.3
+Version: 0.8.5
 Summary: Capture and query workflow provenance data using data observability
 Project-URL: GitHub, https://github.com/ORNL/flowcept
 Author: Oak Ridge National Laboratory

{flowcept-0.8.3.dist-info → flowcept-0.8.5.dist-info}/RECORD

@@ -1,12 +1,12 @@
 flowcept/__init__.py,sha256=CukmdzTUvm6Y_plTKPq4kKn7w9LdR36j7V_C_UQyjhU,2011
-flowcept/configs.py,sha256=f9lgOouFJW2It0dBKFgmqxna9zy6H3Mq8Tqa8JYFBg4,7471
-flowcept/version.py,sha256=Nab3pVvVwFctq6iY0yZ9Rf57aFEh8V4wLrm4yh5C7jU,306
+flowcept/configs.py,sha256=_-jhoI_HGKjzymjYTlDuysbM38Gr2aunc0Q-Stlmcwk,7511
+flowcept/version.py,sha256=RW_aTLB2vWTDjrpIPcWytUXxQhZLynom14B2UHfVVcU,306
 flowcept/analytics/__init__.py,sha256=46q-7vsHq_ddPNrzNnDgEOiRgvlx-5Ggu2ocyROMV0w,641
 flowcept/analytics/analytics_utils.py,sha256=FRJdBtQa7Hrk2oR_FFhmhmMf3X6YyZ4nbH5RIYh7KL4,8753
 flowcept/analytics/data_augmentation.py,sha256=Dyr5x316Zf-k1e8rVoQMCpFOrklYVHjfejRPrtoycmc,1641
 flowcept/analytics/plot.py,sha256=L56y1HRnTE6-Fxs62Y0rV2OtDwjSwgSP3yLdalkiRBQ,2932
 flowcept/commons/__init__.py,sha256=W94CqapS0IGuuIGHHaz4sNuuiYhgtJWtpDEbnI0pGwI,26
-flowcept/commons/autoflush_buffer.py,sha256=dOwMAyY3mW_GWUFPRNDm3vSDxS3gOdFgH9cgG6Ljqjc,2501
+flowcept/commons/autoflush_buffer.py,sha256=8M0fcIeHck-mSGQ2HFpW3_Af8-dHswhIbUMX5FATm48,2589
 flowcept/commons/flowcept_logger.py,sha256=0asRucrDMeRXvsdhuCmH6lWO7lAt_Z5o5uW7rrQhcjc,1857
 flowcept/commons/query_utils.py,sha256=3tyK5VYA10iDtmtzNwa8OQGn93DBxsu6rTjHDphftSc,2208
 flowcept/commons/settings_factory.py,sha256=vpvnpqabPKK-mzoZ5iLdnefpemW1MldDOZhii_m9wNk,1814
@@ -19,18 +19,18 @@ flowcept/commons/daos/docdb_dao/docdb_dao_base.py,sha256=YbfSVJPwZGK2GBYkeapRC83
 flowcept/commons/daos/docdb_dao/lmdb_dao.py,sha256=dJOLgCx_lwdz6MKiMpM_UE4rm0angDCPaVz_WU5KqIA,10407
 flowcept/commons/daos/docdb_dao/mongodb_dao.py,sha256=-Kxjep1FbjKiGjvzyvePVHDf-Q1lOIce1EzBURSKubc,38037
 flowcept/commons/daos/mq_dao/__init__.py,sha256=Xxm4FmbBUZDQ7XIAmSFbeKE_AdHsbgFmSuftvMWSykQ,21
-flowcept/commons/daos/mq_dao/mq_dao_base.py,sha256=SZTwqnYbn70AC6kYQxfvl9AWPyE3iFQuWr23pufHoEc,8077
-flowcept/commons/daos/mq_dao/mq_dao_kafka.py,sha256=o8MmhF00XRaLEeCrh8btkXBupD4sLMEH-vRp7bCiYMw,3589
-flowcept/commons/daos/mq_dao/mq_dao_mofka.py,sha256=xjlMBnu1kR-WyHbZgM_b7Hrda6rtFPQHZConnVMIAo4,3006
-flowcept/commons/daos/mq_dao/mq_dao_redis.py,sha256=9qbRyJjgUkC75VN0lK6XmOZ6H8C9x6bMsSZWyFtH6GI,3304
+flowcept/commons/daos/mq_dao/mq_dao_base.py,sha256=EAqOhy7Q8V29JFDG8C50nRK34KsPxEICkG4elk4ZfX8,9020
+flowcept/commons/daos/mq_dao/mq_dao_kafka.py,sha256=bf-bZvWw9JJk8Kdfzx2UkAnQC95rSrKXDEyYkrcncOk,4400
+flowcept/commons/daos/mq_dao/mq_dao_mofka.py,sha256=aZ810wN5Wkjk7oRUxDWJWOIREUsmq57oI4AxY1bWBuk,3940
+flowcept/commons/daos/mq_dao/mq_dao_redis.py,sha256=Br97SoDIkt4dHH937Yjg3wtkn1xGT-x9t-8E3VD5TeU,4277
 flowcept/commons/flowcept_dataclasses/__init__.py,sha256=8KkiJh0WSRAB50waVluxCSI8Tb9X1L9nup4c8RN3ulc,30
 flowcept/commons/flowcept_dataclasses/base_settings_dataclasses.py,sha256=Cjw2PGYtZDfnwecz6G3S42Ncmxj7AIZVEBx05bsxRUo,399
 flowcept/commons/flowcept_dataclasses/task_object.py,sha256=3DD5ZNMz7EVILS9PRkQ3khboav7lIKoUC5W6sKMFauQ,4694
 flowcept/commons/flowcept_dataclasses/telemetry.py,sha256=9_5ONCo-06r5nKHXmi5HfIhiZSuPgmTECiq_u9MlxXM,2822
-flowcept/commons/flowcept_dataclasses/workflow_object.py,sha256=t9M0eVdcq3NLZgT4fwXrR3x2oOgjtE5Jo5_MAi4-0YM,4283
+flowcept/commons/flowcept_dataclasses/workflow_object.py,sha256=FBpel5PulrN3mCCk3hrQhoYiFqd-4aNV4tT03bCV3DE,4376
 flowcept/flowcept_api/__init__.py,sha256=T1ty86YlocQ5Z18l5fUqHj_CC6Unq_iBv0lFyiI7Ao8,22
 flowcept/flowcept_api/db_api.py,sha256=hKXep-n50rp9cAzV0ljk2QVEF8O64yxi3ujXv5_Ibac,9723
-flowcept/flowcept_api/flowcept_controller.py,sha256=zq4cRM14xoeRA6HPL6JwLwDLpsVra5ej1EAPmiVKUIs,11932
+flowcept/flowcept_api/flowcept_controller.py,sha256=lkHR7O0zAAfbGtVa4o9tjZMdZquYN7vdnymRKzc4B8s,11933
 flowcept/flowcept_api/task_query_api.py,sha256=SrwB0OCVtbpvCPECkE2ySM10G_g8Wlk5PJ8h-0xEaNc,23821
 flowcept/flowcept_webserver/__init__.py,sha256=8411GIXGddKTKoHUvbo_Rq6svosNG7tG8VzvUEBd7WI,28
 flowcept/flowcept_webserver/app.py,sha256=VUV8_JZbIbx9u_1O7m7XtRdhZb_7uifUa-iNlPhmZws,658
@@ -38,15 +38,15 @@ flowcept/flowcept_webserver/resources/__init__.py,sha256=XOk5yhLeLU6JmVXxbl3TY2z
 flowcept/flowcept_webserver/resources/query_rsrc.py,sha256=Mk1XDC_wVYkMk0eaazqWWrTC07gQU9U0toKfip0ihZE,1353
 flowcept/flowcept_webserver/resources/task_messages_rsrc.py,sha256=0u68it2W-9NzUUx5fWOZCqvRKe5EsLI8oyvto9634Ng,666
 flowcept/flowceptor/__init__.py,sha256=wVxRXUv07iNx6SMRRma2vqhR_GIcRl0re_WCYG65PUs,29
-flowcept/flowceptor/telemetry_capture.py,sha256=8LK4o3OaZD8B6KTpgpvD9D834dFpAJpX-NN2bc91jZU,13658
+flowcept/flowceptor/telemetry_capture.py,sha256=FlWGLpzb6pBJOKVi349kyZKzAHeTsL86BCQd4Wtxpig,13746
 flowcept/flowceptor/adapters/__init__.py,sha256=SuZbSZVVQeBJ9zXW-M9jF09dw3XIjre3lSGrUO1Y8Po,27
 flowcept/flowceptor/adapters/base_interceptor.py,sha256=99a_Ipnj6g8qZMHWLBEYJh0Cox033ADxOKPFrivr9gw,6056
 flowcept/flowceptor/adapters/instrumentation_interceptor.py,sha256=DhK2bBnpghqPSeA62BUqRg6pl8zxuYrP33dK4x6PhRE,733
 flowcept/flowceptor/adapters/interceptor_state_manager.py,sha256=xRzmi5YFKBEqNtX8F5s6XlMTRe27ml4BmQtBO4WtG2c,919
 flowcept/flowceptor/adapters/dask/__init__.py,sha256=GKreb5L_nliD2BEckyB943zOQ-b6Gn1fLDj81FqSK2Y,23
 flowcept/flowceptor/adapters/dask/dask_dataclasses.py,sha256=6LTG-kdcc6AUuVINvkqB5QHw6pchg1aMqj0sdWt2Ef8,580
-flowcept/flowceptor/adapters/dask/dask_interceptor.py,sha256=vHEUpp2LusOieQq4V3qTXK3_FpGaaevCCs9jiw-3xaA,6554
-flowcept/flowceptor/adapters/dask/dask_plugins.py,sha256=rIrG8CCaggJRZ3hRxu-1cpCcCyAbkThDzGBzr2dIsDc,2874
+flowcept/flowceptor/adapters/dask/dask_interceptor.py,sha256=uBQpLluYXzlT1gBDfTe4_WueC_fWBEs5Xr8ntpOmljE,5869
+flowcept/flowceptor/adapters/dask/dask_plugins.py,sha256=s1ENAi9N61PC_6RiFvOYhJsgWzSm_lFWm3w87V-R1YY,2473
 flowcept/flowceptor/adapters/mlflow/__init__.py,sha256=3mzHrvh1XQOy68qx1A3so9Nq27tIb0i2mSXfv3F6gZg,25
 flowcept/flowceptor/adapters/mlflow/interception_event_handler.py,sha256=-SsIRdOcZjQUTzWgsZ41ouqpla4Qd32jIWXIAGU1pPw,494
 flowcept/flowceptor/adapters/mlflow/mlflow_dao.py,sha256=dPEgCduiw14_pzT5WCjuokwaN7p5Tu7UvWS2rtGh4qk,4589
@@ -59,15 +59,15 @@ flowcept/flowceptor/adapters/zambeze/__init__.py,sha256=1e9_hK2cUKDXhQ0kBRftwcJj
 flowcept/flowceptor/adapters/zambeze/zambeze_dataclasses.py,sha256=nn9MxvcdzgmOa8n5Jwdl7UzlSzxEu9bA-Ls6cHyb91c,849
 flowcept/flowceptor/adapters/zambeze/zambeze_interceptor.py,sha256=Bjyi48JW0DXJLJuvwPxaD8zxxsSoEFgSoXl8YcbwFWk,3782
 flowcept/flowceptor/consumers/__init__.py,sha256=foxtVEb2ZEe9g1slfYIKM4tIFv-He1l7XS--SYs7nlQ,28
-flowcept/flowceptor/consumers/consumer_utils.py,sha256=JmyjQeZPqMj_yqFlxxw9k2_JZvZkAmX7kySV__YvEVc,5719
-flowcept/flowceptor/consumers/document_inserter.py,sha256=od1M3A_N74nIXociWRHAajPLLCWtJdfI8JgNQ9eSPf8,9339
+flowcept/flowceptor/consumers/consumer_utils.py,sha256=7bvFJWusJkfA4j0gwZLDIIsIOyfk9wRq6s5liS3JAV0,5665
+flowcept/flowceptor/consumers/document_inserter.py,sha256=rAK3rs3VNW5a6koesE05scQ1mR_4BhuxLurP10ipURs,9339
 flowcept/instrumentation/__init__.py,sha256=M5bTmg80E4QyN91gUX3qfw_nbtJSXwGWcKxdZP3vJz0,34
-flowcept/instrumentation/flowcept_loop.py,sha256=9Ap7-PfpNdwS7DaRDaB-R9G3X_G3RZvGVkNVUZAix5A,12164
+flowcept/instrumentation/flowcept_loop.py,sha256=RvETm3Pn37dIw_a1RXigyh2U7MCBHqi46dPmbrz3RMQ,12171
 flowcept/instrumentation/flowcept_task.py,sha256=l_BAYEUZ_SeBt8QJN_E9D9QcZVYRnW9qO_XRnqvmePE,5993
 flowcept/instrumentation/flowcept_torch.py,sha256=KXA1HBwz8l5Qp7PkZ7nsbYlM8IcwWD_u04NxaAcZPzM,23395
 flowcept/instrumentation/task_capture.py,sha256=u82r_SgzoVKyb6_SWtfB-meBUZgjrXvF5dxkH9vnMDs,4776
-resources/sample_settings.yaml,sha256=fRWyuIXizQUjzChIL8ze5zy5bwFgETNZCByJ35NWZb4,3408
-flowcept-0.8.3.dist-info/METADATA,sha256=jgA-zJnRY-cbB1b70nEdtXs3kToTLdt41gi9cJVbqQo,17543
-flowcept-0.8.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-flowcept-0.8.3.dist-info/licenses/LICENSE,sha256=r5-2P6tFTuRGWT5TiX32s1y0tnp4cIqBEC1QjTaXe2k,1086
-flowcept-0.8.3.dist-info/RECORD,,
+resources/sample_settings.yaml,sha256=aZRAZRkgCe52i-8czQvZsEIAz8dGau-OF2YClUF3QGs,3427
+flowcept-0.8.5.dist-info/METADATA,sha256=uZzz3Hz-Pee3zpeyNYLk5K-JbS_miCTMXIXlqtMBXX0,17543
+flowcept-0.8.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+flowcept-0.8.5.dist-info/licenses/LICENSE,sha256=r5-2P6tFTuRGWT5TiX32s1y0tnp4cIqBEC1QjTaXe2k,1086
+flowcept-0.8.5.dist-info/RECORD,,

resources/sample_settings.yaml

@@ -13,7 +13,7 @@ log:
   log_stream_level: error
 
 telemetry_capture:
-  gpu: ~ # ~ means None. This is a list with GPU metrics. AMD=[activity,used,power,temperature,others]; NVIDIA=[used,temperature,power,name,ix]
+  gpu: ~ # ~ means None. This is a list with GPU metrics. AMD=[activity,used,power,temperature,others,id]; NVIDIA=[used,temperature,power,name,id]
   cpu: true
   per_cpu: true
   process_info: true
@@ -44,6 +44,7 @@ mq:
   channel: interception
   buffer_size: 50
   insertion_buffer_time_secs: 5
+  timing: false
   chunk_size: -1  # use 0 or -1 to disable this. Or simply omit this from the config file.
 
 kv_db: