flowcept 0.8.10__py3-none-any.whl → 0.8.11__py3-none-any.whl

flowcept/flowceptor/consumers/document_inserter.py

@@ -2,11 +2,12 @@
 
 from threading import Thread
 from time import time, sleep
-from typing import Dict
+from typing import Dict, Callable, Tuple
 from uuid import uuid4
 
+from flowcept.commons.task_data_preprocess import summarize_telemetry, tag_critical_task
+from flowcept.flowceptor.consumers.base_consumer import BaseConsumer
 from flowcept.commons.autoflush_buffer import AutoflushBuffer
-from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
 from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
 from flowcept.commons.flowcept_dataclasses.workflow_object import (
     WorkflowObject,
@@ -30,8 +31,18 @@ from flowcept.flowceptor.consumers.consumer_utils import (
 )
 
 
-class DocumentInserter:
-    """Document class."""
+class DocumentInserter(BaseConsumer):
+    """
+    DocumentInserter is a message consumer in Flowcept.
+
+    It handles messages related to tasks, workflows, and control signals, processes them
+    (e.g., adds metadata, sanitizes fields), and then inserts them into one or more configured
+    document databases (e.g., MongoDB, LMDB). It buffers incoming messages to reduce insertion
+    overhead and supports both time-based and size-based flushing.
+
+    The inserter is intended to run in a thread or process alongside other Flowcept consumers,
+    ensuring provenance data is persisted reliably and in a structured format.
+    """
 
     DECODER = GenericJSONDecoder if JSON_SERIALIZER == "complex" else None
 
@@ -52,8 +63,8 @@ class DocumentInserter:
         check_safe_stops=True,
         bundle_exec_id=None,
     ):
-        self._mq_dao = MQDao.build()
         self._doc_daos = []
+        self.logger = FlowceptLogger()
         if MONGO_ENABLED:
             from flowcept.commons.daos.docdb_dao.mongodb_dao import MongoDBDAO
 
@@ -62,8 +73,13 @@ class DocumentInserter:
             from flowcept.commons.daos.docdb_dao.lmdb_dao import LMDBDAO
 
             self._doc_daos.append(LMDBDAO())
+        self._should_start = True
+        if not len(self._doc_daos):
+            self._should_start = False
+            return
+
+        super().__init__()
         self._previous_time = time()
-        self.logger = FlowceptLogger()
         self._main_thread: Thread = None
         self._curr_db_buffer_size = DB_BUFFER_SIZE
         self._bundle_exec_id = bundle_exec_id
@@ -77,7 +93,18 @@ class DocumentInserter:
 
     @staticmethod
     def flush_function(buffer, doc_daos, logger):
-        """Flush it."""
+        """
+        Flush the buffer contents to all configured document databases.
+
+        Parameters
+        ----------
+        buffer : list
+            List of messages to be flushed to the databases.
+        doc_daos : list
+            List of DAO instances to insert data into (e.g., MongoDBDAO, LMDBDAO).
+        logger : FlowceptLogger
+            Logger instance for debug and info logging.
+        """
         logger.info(f"Current Doc buffer size: {len(buffer)}, Gonna flush {len(buffer)} msgs to DocDBs!")
         for dao in doc_daos:
             dao.insert_and_update_many_tasks(buffer, TaskObject.task_id_field())
@@ -93,9 +120,12 @@ class DocumentInserter:
                 message["workflow_id"] = wf_id
 
         if "campaign_id" not in message:
-            campaign_id = self._mq_dao._keyvalue_dao.get_key("current_campaign_id")
-            if campaign_id:
-                message["campaign_id"] = campaign_id
+            try:
+                campaign_id = self._mq_dao._keyvalue_dao.get_key("current_campaign_id")
+                if campaign_id:
+                    message["campaign_id"] = campaign_id
+            except Exception as e:
+                self.logger.error(e)
 
         if "subtype" not in message and "group_id" in message:
             message["subtype"] = "iteration"
@@ -113,6 +143,23 @@ class DocumentInserter:
 
         if ENRICH_MESSAGES:
             TaskObject.enrich_task_dict(message)
+            if (
+                "telemetry_at_start" in message
+                and message["telemetry_at_start"]
+                and "telemetry_at_end" in message
+                and message["telemetry_at_end"]
+            ):
+                try:
+                    telemetry_summary = summarize_telemetry(message)
+                    message["telemetry_summary"] = telemetry_summary
+                    # TODO: make this dynamic
+                    tags = tag_critical_task(
+                        generated=message.get("generated", {}), telemetry_summary=telemetry_summary, thresholds=None
+                    )
+                    if tags:
+                        message["tags"] = tags
+                except Exception as e:
+                    self.logger.error(e)  # TODO: check if cpu, etc is in the fields in for the telemetry_summary
 
         if REMOVE_EMPTY_FIELDS:
             remove_empty_fields_from_dict(message)
@@ -139,36 +186,71 @@ class DocumentInserter:
                 f"in DocInserter from the interceptor "
                 f"{'' if exec_bundle_id is None else exec_bundle_id}_{interceptor_instance_id}!"
             )
-            self.logger.info(
-                f"Begin register_time_based_thread_end "
-                f"{'' if exec_bundle_id is None else exec_bundle_id}_{interceptor_instance_id}!"
-            )
-            self._mq_dao.register_time_based_thread_end(interceptor_instance_id, exec_bundle_id)
-            self.logger.info(
-                f"Done register_time_based_thread_end "
-                f"{'' if exec_bundle_id is None else exec_bundle_id}_{interceptor_instance_id}!"
-            )
+            if self.check_safe_stops:
+                self.logger.info(
+                    f"Begin register_time_based_thread_end "
+                    f"{'' if exec_bundle_id is None else exec_bundle_id}_{interceptor_instance_id}!"
+                )
+                self._mq_dao.register_time_based_thread_end(interceptor_instance_id, exec_bundle_id)
+                self.logger.info(
+                    f"Done register_time_based_thread_end "
+                    f"{'' if exec_bundle_id is None else exec_bundle_id}_{interceptor_instance_id}!"
+                )
             return "continue"
         elif message["info"] == "stop_document_inserter":
-            self.logger.info("Document Inserter is stopping...")
-            return "stop"
-
-    def start(self, threaded=True) -> "DocumentInserter":
-        """Start it."""
-        self._mq_dao.subscribe()
-        if threaded:
-            self._main_thread = Thread(target=self._start)
-            self._main_thread.start()
-        else:
-            self._start()
+            exec_bundle_id = message.get("exec_bundle_id", None)
+            if self._bundle_exec_id == exec_bundle_id:
+                self.logger.info(f"Document Inserter for exec_id {exec_bundle_id} is stopping...")
+                return "stop"
+            else:
+                return "continue"
+
+    def start(self, target: Callable = None, args: Tuple = (), threaded: bool = True, daemon=True):
+        """
+        Start the DocumentInserter thread.
+
+        Parameters
+        ----------
+        target : Callable, optional
+            Target function to run. Defaults to `self.thread_target`.
+        args : tuple, optional
+            Arguments to pass to the target function. Defaults to empty tuple.
+        threaded : bool, optional
+            Whether to run the inserter in a separate thread. Defaults to True.
+        daemon : bool, optional
+            Whether the thread should be a daemon. Defaults to True.
+
+        Returns
+        -------
+        DocumentInserter
+            The current instance of the DocumentInserter.
+        """
+        if not self._should_start:
+            self.logger.info("Doc Inserter cannot start as all DocDBs are disabled.")
+            return self
+        super().start(target=self.thread_target, threaded=threaded, daemon=daemon)
         return self
 
-    def _start(self):
-        self._mq_dao.message_listener(self._message_handler)
+    def thread_target(self):
+        """Function to be used in the self.start method."""
+        super().default_thread_target()
         self.buffer.stop()
         self.logger.info("Ok, we broke the doc inserter message listen loop!")
 
-    def _message_handler(self, msg_obj: dict):
+    def message_handler(self, msg_obj: Dict):
+        """
+        Overrides the message_handler method by determining message's type and dispatching to the appropriate handler.
+
+        Parameters
+        ----------
+        msg_obj : dict
+            The message object received from the message queue.
+
+        Returns
+        -------
+        bool
+            False if a stop control message is received, True otherwise.
+        """
         msg_type = msg_obj.get("type")
         if msg_type == "flowcept_control":
             r = self._handle_control_message(msg_obj)
@@ -197,7 +279,22 @@ class DocumentInserter:
             return True
 
     def stop(self, bundle_exec_id=None):
-        """Stop document inserter."""
+        """
+        Stop the DocumentInserter safely, waiting for all time-based threads to end.
+
+        Parameters
+        ----------
+        bundle_exec_id : str, optional
+            The execution bundle ID to check for safe stopping. If None, will not use it as a filter.
+
+        Notes
+        -----
+        This method flushes remaining buffered data, stops internal threads,
+        closes database connections, and clears campaign state from the key-value store.
+        """
+        if not self._should_start:
+            self.logger.info("Doc Inserter has not been started, so it can't stop.")
+            return self
         if self.check_safe_stops:
             trial = 0
             while not self._mq_dao.all_time_based_threads_ended(bundle_exec_id):
@@ -216,12 +313,14 @@ class DocumentInserter:
                     msg = f"DocInserter {id(self)} gave up waiting for signal. "
                     self.logger.critical(msg + "Safe to stop now.")
                     break
+            self._mq_dao.delete_current_campaign_id()
+
         self.logger.info("Sending message to stop document inserter.")
-        self._mq_dao.send_document_inserter_stop()
+        self._mq_dao.send_document_inserter_stop(exec_bundle_id=self._bundle_exec_id)
         self.logger.info(f"Doc Inserter {id(self)} Sent message to stop itself.")
         self._main_thread.join()
         for dao in self._doc_daos:
             self.logger.info(f"Closing document_inserter {dao.__class__.__name__} connection.")
             dao.close()
-        self._mq_dao.delete_current_campaign_id()
+
         self.logger.info("Document Inserter is stopped.")

flowcept/flowceptor/telemetry_capture.py

@@ -215,7 +215,7 @@ class TelemetryCapture:
 
     def capture(self) -> Telemetry:
         """Capture it."""
-        if self.conf is None:
+        if not self.conf:
             return None
         tel = Telemetry()
         if self.conf.get("process_info", False):

flowcept/instrumentation/task_capture.py

@@ -59,6 +59,7 @@ class FlowceptTask(object):
         campaign_id: str = None,
         activity_id: str = None,
         used: Dict = None,
+        subtype: str = None,
         custom_metadata: Dict = None,
     ):
         if not INSTRUMENTATION_ENABLED:
@@ -66,13 +67,16 @@ class FlowceptTask(object):
             return
         self._task = TaskObject()
         self._interceptor = InstrumentationInterceptor.get_instance()
-        self._task.telemetry_at_start = self._interceptor.telemetry_capture.capture()
+        tel = self._interceptor.telemetry_capture.capture()
+        if tel:
+            self._task.telemetry_at_start = tel
         self._task.activity_id = activity_id
         self._task.started_at = time()
         self._task.task_id = task_id or self._gen_task_id()
         self._task.workflow_id = workflow_id or Flowcept.current_workflow_id
         self._task.campaign_id = campaign_id or Flowcept.campaign_id
         self._task.used = used
+        self._task.subtype = subtype
         self._task.custom_metadata = custom_metadata
         self._ended = False
 
@@ -124,7 +128,9 @@ class FlowceptTask(object):
         """
         if not INSTRUMENTATION_ENABLED:
             return
-        self._task.telemetry_at_end = self._interceptor.telemetry_capture.capture()
+        tel = self._interceptor.telemetry_capture.capture()
+        if tel:
+            self._task.telemetry_at_end = tel
         self._task.ended_at = ended_at or time()
         self._task.status = status
         self._task.stderr = stderr

flowcept/version.py

@@ -4,4 +4,4 @@
 # The expected format is: <Major>.<Minor>.<Patch>
 # This file is supposed to be automatically modified by the CI Bot.
 # See .github/workflows/version_bumper.py
-__version__ = "0.8.10"
+__version__ = "0.8.11"

{flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.8.10
+Version: 0.8.11
 Summary: Capture and query workflow provenance data using data observability
 Project-URL: GitHub, https://github.com/ORNL/flowcept
 Author: Oak Ridge National Laboratory
@@ -29,6 +29,7 @@ Requires-Dist: furo; extra == 'all'
 Requires-Dist: jupyterlab; extra == 'all'
 Requires-Dist: mlflow-skinny; extra == 'all'
 Requires-Dist: nbmake; extra == 'all'
+Requires-Dist: paho-mqtt; extra == 'all'
 Requires-Dist: pika; extra == 'all'
 Requires-Dist: plotly; extra == 'all'
 Requires-Dist: pymongo; extra == 'all'
@@ -65,6 +66,12 @@ Requires-Dist: furo; extra == 'docs'
 Requires-Dist: sphinx; extra == 'docs'
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka<=2.8.0; extra == 'kafka'
+Provides-Extra: llm-agent
+Requires-Dist: faiss-cpu; extra == 'llm-agent'
+Requires-Dist: langchain-community; extra == 'llm-agent'
+Requires-Dist: mcp[cli]; extra == 'llm-agent'
+Requires-Dist: sentence-transformers; extra == 'llm-agent'
+Requires-Dist: tiktoken; extra == 'llm-agent'
 Provides-Extra: ml-dev
 Requires-Dist: datasets==2.17.0; extra == 'ml-dev'
 Requires-Dist: nltk; extra == 'ml-dev'
@@ -80,6 +87,8 @@ Requires-Dist: sqlalchemy; extra == 'mlflow'
 Requires-Dist: watchdog; extra == 'mlflow'
 Provides-Extra: mongo
 Requires-Dist: pymongo; extra == 'mongo'
+Provides-Extra: mqtt
+Requires-Dist: paho-mqtt; extra == 'mqtt'
 Provides-Extra: nvidia
 Requires-Dist: nvidia-ml-py; extra == 'nvidia'
 Provides-Extra: tensorboard

{flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/RECORD

@@ -1,7 +1,7 @@
 flowcept/__init__.py,sha256=CukmdzTUvm6Y_plTKPq4kKn7w9LdR36j7V_C_UQyjhU,2011
-flowcept/cli.py,sha256=WXslw2Utrn29DmJOHliptjNGpZC239IdjQdQ1dmuGVI,8638
-flowcept/configs.py,sha256=p4VB5cLmAI4NKpxrZ92jNEI1ZRPrex8Ef0Dxy23ecDk,7276
-flowcept/version.py,sha256=_nToyachA8IpRDDoW02cQ8d923faUb4KptrClM0bRf8,307
+flowcept/cli.py,sha256=5rFPMle_fyXNiOvAgfCh8eIqR_vZKoY3aXbeORU8bcY,15117
+flowcept/configs.py,sha256=PyZ0svU0DKLeSNMaLq8G62zsOrSL2RO1mPZorjTQs_Q,7458
+flowcept/version.py,sha256=kZzuCD8eKygfSNYWrKZruE9fLy68IKgtP2XirWNoQso,307
 flowcept/analytics/__init__.py,sha256=46q-7vsHq_ddPNrzNnDgEOiRgvlx-5Ggu2ocyROMV0w,641
 flowcept/analytics/analytics_utils.py,sha256=FRJdBtQa7Hrk2oR_FFhmhmMf3X6YyZ4nbH5RIYh7KL4,8753
 flowcept/analytics/data_augmentation.py,sha256=Dyr5x316Zf-k1e8rVoQMCpFOrklYVHjfejRPrtoycmc,1641
@@ -10,28 +10,30 @@ flowcept/commons/__init__.py,sha256=W94CqapS0IGuuIGHHaz4sNuuiYhgtJWtpDEbnI0pGwI,
 flowcept/commons/autoflush_buffer.py,sha256=8M0fcIeHck-mSGQ2HFpW3_Af8-dHswhIbUMX5FATm48,2589
 flowcept/commons/flowcept_logger.py,sha256=0asRucrDMeRXvsdhuCmH6lWO7lAt_Z5o5uW7rrQhcjc,1857
 flowcept/commons/query_utils.py,sha256=3tyK5VYA10iDtmtzNwa8OQGn93DBxsu6rTjHDphftSc,2208
-flowcept/commons/settings_factory.py,sha256=vpvnpqabPKK-mzoZ5iLdnefpemW1MldDOZhii_m9wNk,1814
-flowcept/commons/utils.py,sha256=rT8ZbfVBXbnsDvINvRvcTRuz3tU0DJh79z9YM2Ydgrc,8416
+flowcept/commons/settings_factory.py,sha256=bMTjgXRfb5HsL2lPnLfem-9trqELbNWE04Ie7lSlxYM,1731
+flowcept/commons/task_data_preprocess.py,sha256=skTkT8XpO1gsIfu5YGHPj4A1qK4O139rELpwVmMryeY,6767
+flowcept/commons/utils.py,sha256=nzNSAupZidI54ngNF67MX5H0gIj1YOq0GTv6A-z6eEE,8418
 flowcept/commons/vocabulary.py,sha256=_GzHJ1wSYJlLsu_uu1Am6N3zvc59S4FCuT5yp7lynPw,713
 flowcept/commons/daos/__init__.py,sha256=RO51svfHOg9naN676zuQwbj_RQ6IFHu-RALeefvtwwk,23
-flowcept/commons/daos/keyvalue_dao.py,sha256=03xHhQIfZas0LQLP1DbGJ5DoskXyZNXQKINhti0wH9A,3809
+flowcept/commons/daos/keyvalue_dao.py,sha256=g7zgC9hVC1NTllwUAqGt44YqdqYUgAKgPlX8_G4BRGw,3599
+flowcept/commons/daos/redis_conn.py,sha256=gFyW-5yf6B8ExEYopCmbap8ki-iEwuIw-KH9f6o7UGQ,1495
 flowcept/commons/daos/docdb_dao/__init__.py,sha256=qRvXREeUJ4mkhxdC9bzpOsVX6M2FB5hDyLFxhMxTGhs,30
 flowcept/commons/daos/docdb_dao/docdb_dao_base.py,sha256=YbfSVJPwZGK2GBYkeapRC83HkmP0c6Msv5TriD88RcI,11812
 flowcept/commons/daos/docdb_dao/lmdb_dao.py,sha256=dJOLgCx_lwdz6MKiMpM_UE4rm0angDCPaVz_WU5KqIA,10407
 flowcept/commons/daos/docdb_dao/mongodb_dao.py,sha256=0y9RiL54e1GxSTkRHFlMrLFAHWuB3YyNS2zLsnBPtxg,38456
 flowcept/commons/daos/mq_dao/__init__.py,sha256=Xxm4FmbBUZDQ7XIAmSFbeKE_AdHsbgFmSuftvMWSykQ,21
-flowcept/commons/daos/mq_dao/mq_dao_base.py,sha256=EAqOhy7Q8V29JFDG8C50nRK34KsPxEICkG4elk4ZfX8,9020
-flowcept/commons/daos/mq_dao/mq_dao_kafka.py,sha256=bf-bZvWw9JJk8Kdfzx2UkAnQC95rSrKXDEyYkrcncOk,4400
-flowcept/commons/daos/mq_dao/mq_dao_mofka.py,sha256=Q_mgZ3C_4gTTvnuJ2ZLmJgJfbAOopeSR9jvznI4JRuo,3948
-flowcept/commons/daos/mq_dao/mq_dao_redis.py,sha256=Br97SoDIkt4dHH937Yjg3wtkn1xGT-x9t-8E3VD5TeU,4277
+flowcept/commons/daos/mq_dao/mq_dao_base.py,sha256=gXOHRhWHiJmN7mNRE4QIbnOfrN-314Ds2LONzsBw3Ug,9184
+flowcept/commons/daos/mq_dao/mq_dao_kafka.py,sha256=Idq5TKm-dNZwlfkmLxUy7IBrn0DBtCekrC4n5dXUpn4,4379
+flowcept/commons/daos/mq_dao/mq_dao_mofka.py,sha256=tRdMGYDzdeIJxad-B4-DE6u8Wzs61eTzOW4ojZrnTxs,4057
+flowcept/commons/daos/mq_dao/mq_dao_redis.py,sha256=WKPoMPBSce4shqbBkgsnuqJAJoZZ4U_hdebhyFqtejQ,5535
 flowcept/commons/flowcept_dataclasses/__init__.py,sha256=8KkiJh0WSRAB50waVluxCSI8Tb9X1L9nup4c8RN3ulc,30
 flowcept/commons/flowcept_dataclasses/base_settings_dataclasses.py,sha256=Cjw2PGYtZDfnwecz6G3S42Ncmxj7AIZVEBx05bsxRUo,399
-flowcept/commons/flowcept_dataclasses/task_object.py,sha256=3DD5ZNMz7EVILS9PRkQ3khboav7lIKoUC5W6sKMFauQ,4694
+flowcept/commons/flowcept_dataclasses/task_object.py,sha256=ZzUINqIm6s0J7M6a9OnzEcrBq5z8QZ-m_5n8UE0I6kI,5715
 flowcept/commons/flowcept_dataclasses/telemetry.py,sha256=9_5ONCo-06r5nKHXmi5HfIhiZSuPgmTECiq_u9MlxXM,2822
 flowcept/commons/flowcept_dataclasses/workflow_object.py,sha256=f8aB0b3xcUr3KQTlloF7R_P6xQejzDPOm-s6dLhGMeA,4383
 flowcept/flowcept_api/__init__.py,sha256=T1ty86YlocQ5Z18l5fUqHj_CC6Unq_iBv0lFyiI7Ao8,22
 flowcept/flowcept_api/db_api.py,sha256=hKXep-n50rp9cAzV0ljk2QVEF8O64yxi3ujXv5_Ibac,9723
-flowcept/flowcept_api/flowcept_controller.py,sha256=lkHR7O0zAAfbGtVa4o9tjZMdZquYN7vdnymRKzc4B8s,11933
+flowcept/flowcept_api/flowcept_controller.py,sha256=_aB7x7ANOthpqD-xCA8UgiyeyabbzLpXgtpzkKXBBbw,12530
 flowcept/flowcept_api/task_query_api.py,sha256=SrwB0OCVtbpvCPECkE2ySM10G_g8Wlk5PJ8h-0xEaNc,23821
 flowcept/flowcept_webserver/__init__.py,sha256=8411GIXGddKTKoHUvbo_Rq6svosNG7tG8VzvUEBd7WI,28
 flowcept/flowcept_webserver/app.py,sha256=VUV8_JZbIbx9u_1O7m7XtRdhZb_7uifUa-iNlPhmZws,658
@@ -39,11 +41,18 @@ flowcept/flowcept_webserver/resources/__init__.py,sha256=XOk5yhLeLU6JmVXxbl3TY2z
 flowcept/flowcept_webserver/resources/query_rsrc.py,sha256=Mk1XDC_wVYkMk0eaazqWWrTC07gQU9U0toKfip0ihZE,1353
 flowcept/flowcept_webserver/resources/task_messages_rsrc.py,sha256=0u68it2W-9NzUUx5fWOZCqvRKe5EsLI8oyvto9634Ng,666
 flowcept/flowceptor/__init__.py,sha256=wVxRXUv07iNx6SMRRma2vqhR_GIcRl0re_WCYG65PUs,29
-flowcept/flowceptor/telemetry_capture.py,sha256=wSXyQJ-vPVzeldD4KqoLQA2rg7V0EOQo_11ErJE5oQQ,13743
+flowcept/flowceptor/telemetry_capture.py,sha256=kDlRA1chkR31ckNuuQu7PmJPAO_yxwJ1-KRHJKW8csA,13739
 flowcept/flowceptor/adapters/__init__.py,sha256=SuZbSZVVQeBJ9zXW-M9jF09dw3XIjre3lSGrUO1Y8Po,27
-flowcept/flowceptor/adapters/base_interceptor.py,sha256=99a_Ipnj6g8qZMHWLBEYJh0Cox033ADxOKPFrivr9gw,6056
+flowcept/flowceptor/adapters/base_interceptor.py,sha256=m2k_775gsiG7uEYEKLgA0KC4fysGTafbINCC4jkqd3M,6388
 flowcept/flowceptor/adapters/instrumentation_interceptor.py,sha256=DhK2bBnpghqPSeA62BUqRg6pl8zxuYrP33dK4x6PhRE,733
 flowcept/flowceptor/adapters/interceptor_state_manager.py,sha256=xRzmi5YFKBEqNtX8F5s6XlMTRe27ml4BmQtBO4WtG2c,919
+flowcept/flowceptor/adapters/agents/__init__.py,sha256=5x_qyGaazppgvNDWbNhC_gfTav0b3r_P4CX7QwFv4BQ,32
+flowcept/flowceptor/adapters/agents/agents_utils.py,sha256=JDxxsgcCdRR3CV3odIuLcHQcq-VFuApZaioqs_yHkuU,2798
+flowcept/flowceptor/adapters/agents/flowcept_agent.py,sha256=13CobrSp_uObvLSVW3QNQOcqVa5rvwKBwFa4mjuLvb8,8173
+flowcept/flowceptor/adapters/agents/flowcept_llm_prov_capture.py,sha256=2qzcItS3FHF-Wym_u9jZ-XzgDJnDgm9rGGRhfaqJZ8M,5992
+flowcept/flowceptor/adapters/agents/prompts.py,sha256=VipBULibXYRF_mMCf_yYJ2Sb-aRNY_y7YTDmed4AfgM,2587
+flowcept/flowceptor/adapters/brokers/__init__.py,sha256=mhQXVmh0JklvL93GUtJZLJnPRYX9Nmb8IqcyKJGQBzk,36
+flowcept/flowceptor/adapters/brokers/mqtt_interceptor.py,sha256=wx3STMYPHeXB6ilUn-UQYsxesGp2hF7TRlfn52hNJtY,4845
 flowcept/flowceptor/adapters/dask/__init__.py,sha256=GKreb5L_nliD2BEckyB943zOQ-b6Gn1fLDj81FqSK2Y,23
 flowcept/flowceptor/adapters/dask/dask_dataclasses.py,sha256=6LTG-kdcc6AUuVINvkqB5QHw6pchg1aMqj0sdWt2Ef8,580
 flowcept/flowceptor/adapters/dask/dask_interceptor.py,sha256=uBQpLluYXzlT1gBDfTe4_WueC_fWBEs5Xr8ntpOmljE,5869
@@ -52,24 +61,27 @@ flowcept/flowceptor/adapters/mlflow/__init__.py,sha256=3mzHrvh1XQOy68qx1A3so9Nq2
 flowcept/flowceptor/adapters/mlflow/interception_event_handler.py,sha256=-SsIRdOcZjQUTzWgsZ41ouqpla4Qd32jIWXIAGU1pPw,494
 flowcept/flowceptor/adapters/mlflow/mlflow_dao.py,sha256=dPEgCduiw14_pzT5WCjuokwaN7p5Tu7UvWS2rtGh4qk,4589
 flowcept/flowceptor/adapters/mlflow/mlflow_dataclasses.py,sha256=vbijpDW6npHdsA9-28otXw94O4a9R-PWtq3xlJapsyY,690
-flowcept/flowceptor/adapters/mlflow/mlflow_interceptor.py,sha256=EjjcUx9xK7gx2rGJ83NfnK6noGWsfrOir_qH-aDreGY,3734
+flowcept/flowceptor/adapters/mlflow/mlflow_interceptor.py,sha256=OLmVBdOCMS3GPcdxSdCD794RDbW6p4f8eBh1PXWcvHE,3799
 flowcept/flowceptor/adapters/tensorboard/__init__.py,sha256=LrcR4WCIlBwwHIUSteQ8k8JBdCJTFqLvvgAfnoLeREw,30
 flowcept/flowceptor/adapters/tensorboard/tensorboard_dataclasses.py,sha256=lSfDd6TucVNzGxbm69BYyCVgMr2p9iUEQjnsS4jIfeI,554
-flowcept/flowceptor/adapters/tensorboard/tensorboard_interceptor.py,sha256=VJmfnmtfvQ0ORYd21kMCb8D3XJgtV4Gf2sY_AIMCFzM,4803
-flowcept/flowceptor/adapters/zambeze/__init__.py,sha256=1e9_hK2cUKDXhQ0kBRftwcJjGFqbMVGisNP9oAuWpzk,26
-flowcept/flowceptor/adapters/zambeze/zambeze_dataclasses.py,sha256=nn9MxvcdzgmOa8n5Jwdl7UzlSzxEu9bA-Ls6cHyb91c,849
-flowcept/flowceptor/adapters/zambeze/zambeze_interceptor.py,sha256=Bjyi48JW0DXJLJuvwPxaD8zxxsSoEFgSoXl8YcbwFWk,3782
+flowcept/flowceptor/adapters/tensorboard/tensorboard_interceptor.py,sha256=PUKGlCsYcybsk1HK573Brs6FiXQRoaj6MKgZ3Oyeec4,4881
 flowcept/flowceptor/consumers/__init__.py,sha256=foxtVEb2ZEe9g1slfYIKM4tIFv-He1l7XS--SYs7nlQ,28
+flowcept/flowceptor/consumers/base_consumer.py,sha256=xvs_BMrNFJhNX3ACEkf0DUH7JIJNtjNgg2xMRmujoiE,2935
 flowcept/flowceptor/consumers/consumer_utils.py,sha256=7bvFJWusJkfA4j0gwZLDIIsIOyfk9wRq6s5liS3JAV0,5665
-flowcept/flowceptor/consumers/document_inserter.py,sha256=dVUQPcWmvo68HhxJGlwUF9HtDLvofwwSTiNRwIe38PA,9164
+flowcept/flowceptor/consumers/document_inserter.py,sha256=p5g18ghOrSbrR1b3btnBwZi48qMIvMT9n3QuRBq89LQ,13374
+flowcept/flowceptor/consumers/agent/__init__.py,sha256=R1uvjBPeTLw9SpYgyUc6Qmo16pE84PFHcELTTFvyTWU,56
+flowcept/flowceptor/consumers/agent/base_agent_context_manager.py,sha256=dRgWFuCacB_7lj1NfEeZM2uERmRTxh9QOxx1ywXtzfY,3226
+flowcept/flowceptor/consumers/agent/client_agent.py,sha256=5dtjTRMHACIDwU_0k2xsKia24iasMM_6IMxWYCt5Ql8,1639
+flowcept/flowceptor/consumers/agent/flowcept_agent_context_manager.py,sha256=RSdQ2zFkCizD2CtgdCIRRgs9J2E_G-49wmjNEZYoPMc,5302
+flowcept/flowceptor/consumers/agent/flowcept_qa_manager.py,sha256=FaUTTixKncWjs-6xPw9BdNdPBFopFQ9tNRNT9kLPtyc,4156
 flowcept/instrumentation/__init__.py,sha256=M5bTmg80E4QyN91gUX3qfw_nbtJSXwGWcKxdZP3vJz0,34
 flowcept/instrumentation/flowcept_loop.py,sha256=RvETm3Pn37dIw_a1RXigyh2U7MCBHqi46dPmbrz3RMQ,12171
 flowcept/instrumentation/flowcept_task.py,sha256=l_BAYEUZ_SeBt8QJN_E9D9QcZVYRnW9qO_XRnqvmePE,5993
 flowcept/instrumentation/flowcept_torch.py,sha256=mH4sI2FMtBpGk4hN3U6MUwqd6sOPER8TbigUkexfhDY,23437
-flowcept/instrumentation/task_capture.py,sha256=bYL6ZricaGpkEu0vHIIldF0GERyZRCI31PpkuQH5jbc,4948
-resources/sample_settings.yaml,sha256=luy3EDwgNMTgJIfMrz08ahRK52vXmazvbA-wF_Lgt7U,5812
-flowcept-0.8.10.dist-info/METADATA,sha256=yxqHJsth888tg7cETFUogsas2QDjZjc3KBWrAL9julk,18426
-flowcept-0.8.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-flowcept-0.8.10.dist-info/entry_points.txt,sha256=i8q67WE0201rVxYI2lyBtS52shvgl93x2Szp4q8zMlw,47
-flowcept-0.8.10.dist-info/licenses/LICENSE,sha256=r5-2P6tFTuRGWT5TiX32s1y0tnp4cIqBEC1QjTaXe2k,1086
-flowcept-0.8.10.dist-info/RECORD,,
+flowcept/instrumentation/task_capture.py,sha256=I3So1eysuyJ4KS3Fya14WldbgG0pBJ1CTvTikVKSEsM,5090
+resources/sample_settings.yaml,sha256=cdMyRirOorBBVidUS2RVLs8NKlBjbN-SJZ2XDnloc3A,6313
+flowcept-0.8.11.dist-info/METADATA,sha256=xZpXRV3xItrqcWvCFipJljywnlW8BGSMOC0d02q5qao,18811
+flowcept-0.8.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+flowcept-0.8.11.dist-info/entry_points.txt,sha256=i8q67WE0201rVxYI2lyBtS52shvgl93x2Szp4q8zMlw,47
+flowcept-0.8.11.dist-info/licenses/LICENSE,sha256=r5-2P6tFTuRGWT5TiX32s1y0tnp4cIqBEC1QjTaXe2k,1086
+flowcept-0.8.11.dist-info/RECORD,,

resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.8.10 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.8.11 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
@@ -39,18 +39,22 @@ experiment:
 mq:
   type: redis  # or kafka or mofka; Please adjust the port (kafka's default is 9092; redis is 6379). If mofka, adjust the group_file.
   host: localhost
-  # instances: ["localhost:6379"] # We can have multiple redis instances being accessed by the consumers but each interceptor will currently access one single redis.
+  # uri: ?
+  # instances: ["localhost:6379"] # We can have multiple MQ instances being accessed by the consumers but each interceptor will currently access one single MQ..
   port: 6379
   # group_file: mofka.json
   channel: interception
   buffer_size: 50
   insertion_buffer_time_secs: 5
   timing: false
+  # uri: use Redis connection uri here
   chunk_size: -1  # use 0 or -1 to disable this. Or simply omit this from the config file.
+  same_as_kvdb: false # Set this to true if you are using the same Redis instance both as an MQ and as the KV_DB. In that case, no need to repeat connection parameters in MQ. Use only what you define in KV_DB.
 
-kv_db:  # You can optionally use KV == MQ if MQ is Redis. Otherwise, these will be the Redis instance credentials.
+kv_db:
   host: localhost
   port: 6379
+  enabled: true
   # uri: use Redis connection uri here
 
 web_server:
@@ -75,6 +79,15 @@ db_buffer:
   stop_max_trials: 240    # Maximum number of trials before giving up when waiting for a fully safe stop (i.e., all records have been inserted as expected).
   stop_trials_sleep: 0.01   # Sleep duration (in seconds) between trials when waiting for a fully safe stop.
 
+agent:
+  enabled: false
+  mcp_host: localhost
+  mcp_port: 8000
+  llm_server_url: '?'
+  api_key: '?'
+  model: '?'
+  model_kwargs: {}
+
 databases:
 
   lmdb:
@@ -90,16 +103,26 @@ databases:
 
 adapters:
   # For each key below, you can have multiple instances. Like mlflow1, mlflow2; zambeze1, zambeze2. Use an empty dict, {}, if you won't use any adapter.
-  zambeze:
-    kind: zambeze
-    host: localhost
-    port: 5672
-    queue_names:
-      - hello
-      - hello2
-#    key_values_to_filter:
-#      - key: activity_status
-#        value: CREATED
+
+  broker_mqtt:
+    kind: broker
+    host: h
+    port: 30011
+    protocol: mqtt3.1.1
+    queues: ["#"]
+    username: postman
+    password: p
+    qos: 2
+    task_subtype: intersect_msg
+    tracked_keys:
+      used: payload
+      generated: ~
+      custom_metadata: [headers, msgId]
+      activity_id: operationId
+      submitted_at: ~
+      started_at: ~
+      ended_at: ~
+      registered_at: ~
 
   mlflow:
     kind: mlflow
@@ -122,3 +145,4 @@ adapters:
     worker_should_get_output: true
     scheduler_create_timestamps: true
     worker_create_timestamps: false
+

flowcept/flowceptor/adapters/zambeze/__init__.py

@@ -1 +0,0 @@
-"""Zambeze subpackage."""

flowcept/flowceptor/adapters/zambeze/zambeze_dataclasses.py

@@ -1,41 +0,0 @@
-"""Zambeze dataclass module."""
-
-from dataclasses import dataclass
-from typing import List, Dict
-
-from flowcept.commons.flowcept_dataclasses.base_settings_dataclasses import (
-    BaseSettings,
-    KeyValue,
-)
-
-
-@dataclass
-class ZambezeMessage:
-    """Zambeze message."""
-
-    name: str
-    activity_id: str
-    campaign_id: str
-    origin_agent_id: str
-    files: List[str]
-    command: str
-    activity_status: str
-    arguments: List[str]
-    kwargs: Dict
-    depends_on: List[str]
-
-
-@dataclass
-class ZambezeSettings(BaseSettings):
-    """Zambeze settings."""
-
-    host: str
-    port: int
-    queue_names: List[str]
-    key_values_to_filter: List[KeyValue] = None
-    kind = "zambeze"
-
-    def __post_init__(self):
-        """Set attributes after init."""
-        self.observer_type = "message_broker"
-        self.observer_subtype = "rabbit_mq"