flowcept 0.9.18__tar.gz → 0.9.19__tar.gz

{flowcept-0.9.18 → flowcept-0.9.19}/Makefile

@@ -77,13 +77,13 @@ run:
 	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_HOST=flowcept_mongo --network flowcept_default -it flowcept
 
 tests-in-container-mongo:
-	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_HOST=flowcept_mongo -e MONGO_ENABLED=true -e LMDB_ENABLED=false --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest --ignore=tests/instrumentation_tests/ml_tests
+	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_HOST=flowcept_mongo -e MONGO_ENABLED=true -e LMDB_ENABLED=false --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest tests --ignore=tests/instrumentation_tests/ml_tests
 
 tests-in-container:
-	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_ENABLED=false -e LMDB_ENABLED=true --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest --ignore=tests/instrumentation_tests/ml_tests
+	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_ENABLED=false -e LMDB_ENABLED=true --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest tests --ignore=tests/instrumentation_tests/ml_tests
 
 tests-in-container-kafka:
-	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=kafka -e MONGO_HOST=flowcept_mongo  -e MQ_PORT=29092 -e MQ_TYPE=kafka -e MONGO_ENABLED=true -e LMDB_ENABLED=false --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest --ignore=tests/instrumentation_tests/ml_tests
+	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=kafka -e MONGO_HOST=flowcept_mongo  -e MQ_PORT=29092 -e MQ_TYPE=kafka -e MONGO_ENABLED=true -e LMDB_ENABLED=false --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest tests --ignore=tests/instrumentation_tests/ml_tests
 
 # This command can be removed once we have our CLI
 liveness:

{flowcept-0.9.18 → flowcept-0.9.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.18
+Version: 0.9.19
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT

{flowcept-0.9.18 → flowcept-0.9.19}/deployment/Dockerfile

@@ -1,9 +1,6 @@
 # Use the command `make build` to build this image.
-FROM miniconda:local
-
-RUN apt-get update && \
-    apt-get install -y vim curl wget make patch gcc \
-    && rm -rf /var/lib/apt/lists/*
+# Use conda-forge base image consistently across local/CI.
+FROM condaforge/miniforge3:23.11.0-0
 
 WORKDIR /flowcept
 

flowcept-0.9.19/docs/agent.rst

@@ -0,0 +1,43 @@
+Flowcept Agent
+==============
+
+The Flowcept Agent is an MCP-powered LLM interface for querying provenance data. It exposes a small set of tools
+(e.g., ``prompt_handler``) that route natural-language questions to in-memory queries over captured task summaries.
+
+Online-first design
+------------------
+Like Flowcept as a whole, the agent is designed to run **while a workflow is still executing**. In online mode,
+it consumes messages from the MQ (typically Redis) so it can respond to queries in near real time. This is the
+recommended setup for interactive RAG/MCP analysis during live runs.
+
+Offline (file-based) queries
+----------------------------
+For simple tests or disconnected environments, the agent can also be initialized from a **JSONL buffer file**.
+In this mode, Flowcept writes messages to disk (``dump_buffer``), and the agent loads the file once at startup
+before serving queries.
+
+This is a minimal offline example:
+
+.. code-block:: python
+
+   import json
+   from flowcept import Flowcept, flowcept_task
+   from flowcept.agents.flowcept_agent import FlowceptAgent
+
+   @flowcept_task
+   def sum_one(x):
+       return x + 1
+
+   # Run a small workflow and dump the buffer to disk
+   with Flowcept(start_persistence=False, save_workflow=False, check_safe_stops=False) as f:
+       sum_one(1)
+       f.dump_buffer("flowcept_buffer.jsonl")
+
+   # Start the agent from the buffer file and query it
+   agent = FlowceptAgent(buffer_path="flowcept_buffer.jsonl")
+   agent.start()
+   resp = agent.query("how many tasks?")
+   print(json.loads(resp))
+   agent.stop()
+
+In the future, this page will include a full **online** example (live MQ + Redis) and deployment guidance.

{flowcept-0.9.18 → flowcept-0.9.19}/docs/index.rst

@@ -57,6 +57,7 @@ Flowcept
    quick_start
    architecture
    setup
+   agent
    prov_capture
    telemetry_capture
    prov_storage

{flowcept-0.9.18 → flowcept-0.9.19}/docs/prov_capture.rst

@@ -124,7 +124,7 @@ Optional Arguments
 
 When creating a ``Flowcept`` instance (with or without a context manager), you can pass:
 
-- **interceptors**: list of interceptors (e.g., ``"instrumentation"``, ``"dask"``, ``"mlflow"``). Defaults to ``["instrumentation"]`` if enabled.
+- **interceptors**: list of interceptors (e.g., ``"instrumentation"``, ``"dask"``, ``"mlflow"``). Defaults to ``["instrumentation"]`` if enabled. Instrumentation defaults to enabled unless explicitly set to ``false`` in settings.
 - **bundle_exec_id**: identifier for grouping interceptors. Defaults to ``id(self)``.
 - **campaign_id**: unique identifier for the campaign. Defaults to a generated UUID.
 - **workflow_id**: unique identifier for the workflow. Defaults to a generated UUID.

{flowcept-0.9.18 → flowcept-0.9.19}/pyproject.toml

@@ -130,5 +130,12 @@ packages = ["src/flowcept"]
 [tool.hatch.build.targets.wheel.force-include]
 "resources/sample_settings.yaml" = "resources/sample_settings.yaml"
 
+[tool.pytest.ini_options]
+filterwarnings = [
+    "ignore:websockets\\.legacy is deprecated:DeprecationWarning",
+    "ignore:websockets\\.server\\.WebSocketServerProtocol is deprecated:DeprecationWarning",
+    "ignore:Use `streamable_http_client` instead\\.:DeprecationWarning",
+]
+
 [project.scripts]
-flowcept = "flowcept.cli:main"
+flowcept = "flowcept.cli:main"

{flowcept-0.9.18 → flowcept-0.9.19}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.9.18 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.19 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.

{flowcept-0.9.18 → flowcept-0.9.19}/src/flowcept/agents/agent_client.py

@@ -1,4 +1,6 @@
 import asyncio
+import json
+import re
 from typing import Dict, List, Callable
 
 from flowcept.configs import AGENT_HOST, AGENT_PORT
@@ -48,10 +50,14 @@ def run_tool(
                 result: List[TextContent] = await session.call_tool(tool_name, arguments=kwargs)
                 actual_result = []
                 for r in result.content:
-                    if isinstance(r, str):
-                        actual_result.append(r)
-                    else:
-                        actual_result.append(r.text)
+                    text = r if isinstance(r, str) else r.text
+                    try:
+                        json.loads(text)
+                        actual_result.append(text)
+                    except Exception:
+                        match = re.search(r"Error code:\\s*(\\d+)", text)
+                        code = int(match.group(1)) if match else 400
+                        actual_result.append(json.dumps({"code": code, "result": text, "tool_name": tool_name}))
 
                 return actual_result
 

{flowcept-0.9.18 → flowcept-0.9.19}/src/flowcept/agents/agents_utils.py

@@ -139,8 +139,8 @@ def build_llm_model(
     if _service_provider == "sambanova":
         from langchain_community.llms.sambanova import SambaStudio
 
-        os.environ["SAMBASTUDIO_URL"] = AGENT.get("llm_server_url")
-        os.environ["SAMBASTUDIO_API_KEY"] = AGENT.get("api_key")
+        os.environ["SAMBASTUDIO_URL"] = os.environ.get("SAMBASTUDIO_URL", AGENT.get("llm_server_url"))
+        os.environ["SAMBASTUDIO_API_KEY"] = os.environ.get("SAMBASTUDIO_API_KEY", AGENT.get("api_key"))
 
         llm = SambaStudio(model_kwargs=_model_kwargs)
     elif _service_provider == "azure":
@@ -155,7 +155,16 @@ def build_llm_model(
         from langchain_openai import ChatOpenAI
 
         api_key = os.environ.get("OPENAI_API_KEY", AGENT.get("api_key", None))
-        llm = ChatOpenAI(openai_api_key=api_key, **model_kwargs)
+        base_url = os.environ.get("OPENAI_BASE_URL", AGENT.get("llm_server_url") or None)
+        org = os.environ.get("OPENAI_ORG_ID", AGENT.get("organization", None))
+
+        init_kwargs = {"api_key": api_key}
+        if base_url:
+            init_kwargs["base_url"] = base_url
+        if org:
+            init_kwargs["organization"] = org
+
+        llm = ChatOpenAI(**init_kwargs, **_model_kwargs)
     elif _service_provider == "google":
         if "claude" in _model_kwargs["model"]:
             api_key = os.environ.get("GOOGLE_API_KEY", AGENT.get("api_key", None))
@@ -168,22 +177,6 @@ def build_llm_model(
             from flowcept.agents.llms.gemini25 import Gemini25LLM
 
             llm = Gemini25LLM(**_model_kwargs)
-    elif _service_provider == "openai":
-        from langchain_openai import ChatOpenAI
-
-        api_key = os.environ.get("OPENAI_API_KEY", AGENT.get("api_key"))
-        base_url = os.environ.get("OPENAI_BASE_URL", AGENT.get("llm_server_url") or None)  # optional
-        org = os.environ.get("OPENAI_ORG_ID", AGENT.get("organization", None))  # optional
-
-        init_kwargs = {"api_key": api_key}
-        if base_url:
-            init_kwargs["base_url"] = base_url
-        if org:
-            init_kwargs["organization"] = org
-
-        # IMPORTANT: use the merged kwargs so `model` and temps flow through
-        llm = ChatOpenAI(**init_kwargs, **_model_kwargs)
-
     else:
         raise Exception("Currently supported providers are sambanova, openai, azure, and google.")
     if track_tools:

flowcept-0.9.19/src/flowcept/agents/flowcept_agent.py

@@ -0,0 +1,134 @@
+import json
+import os
+from threading import Thread
+
+from flowcept.agents import check_liveness
+from flowcept.agents.agents_utils import ToolResult
+from flowcept.agents.tools.general_tools import prompt_handler
+from flowcept.agents.agent_client import run_tool
+from flowcept.agents.flowcept_ctx_manager import mcp_flowcept, ctx_manager
+from flowcept.commons.flowcept_logger import FlowceptLogger
+from flowcept.configs import AGENT_HOST, AGENT_PORT, DUMP_BUFFER_PATH, MQ_ENABLED
+from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
+from uuid import uuid4
+
+import uvicorn
+
+
+class FlowceptAgent:
+    """
+    Flowcept agent server wrapper with optional offline buffer loading.
+    """
+
+    def __init__(self, buffer_path: str | None = None):
+        """
+        Initialize a FlowceptAgent.
+
+        Parameters
+        ----------
+        buffer_path : str or None
+            Optional path to a JSONL buffer file. When MQ is disabled, the agent
+            loads this file once at startup.
+        """
+        self.buffer_path = buffer_path
+        self.logger = FlowceptLogger()
+        self._server_thread: Thread | None = None
+        self._server = None
+
+    def _load_buffer_once(self) -> int:
+        """
+        Load messages from a JSONL buffer file into the agent context.
+
+        Returns
+        -------
+        int
+            Number of messages loaded.
+        """
+        path = self.buffer_path or DUMP_BUFFER_PATH
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"Buffer file not found: {path}")
+
+        count = 0
+        self.logger.info(f"Loading agent buffer from {path}")
+        if ctx_manager.agent_id is None:
+            agent_id = str(uuid4())
+            BaseAgentContextManager.agent_id = agent_id
+            ctx_manager.agent_id = agent_id
+        with open(path, "r") as handle:
+            for line in handle:
+                line = line.strip()
+                if not line:
+                    continue
+                msg_obj = json.loads(line)
+                ctx_manager.message_handler(msg_obj)
+                count += 1
+        self.logger.info(f"Loaded {count} messages from buffer.")
+        return count
+
+    def _run_server(self):
+        """Run the MCP server (blocking call)."""
+        config = uvicorn.Config(mcp_flowcept.streamable_http_app, host=AGENT_HOST, port=AGENT_PORT, lifespan="on")
+        self._server = uvicorn.Server(config)
+        self._server.run()
+
+    def start(self):
+        """
+        Start the agent server in a background thread.
+
+        Returns
+        -------
+        FlowceptAgent
+            The current instance.
+        """
+        if not MQ_ENABLED:
+            self._load_buffer_once()
+
+        self._server_thread = Thread(target=self._run_server, daemon=False)
+        self._server_thread.start()
+        self.logger.info(f"Flowcept agent server started on {AGENT_HOST}:{AGENT_PORT}")
+        return self
+
+    def stop(self):
+        """Stop the agent server and wait briefly for shutdown."""
+        if self._server is not None:
+            self._server.should_exit = True
+        if self._server_thread is not None:
+            self._server_thread.join(timeout=5)
+
+    def wait(self):
+        """Block until the server thread exits."""
+        if self._server_thread is not None:
+            self._server_thread.join()
+
+    def query(self, message: str) -> ToolResult:
+        """
+        Send a prompt to the agent's main router tool and return the response.
+        """
+        try:
+            resp = run_tool(tool_name=prompt_handler, kwargs={"message": message})[0]
+        except Exception as e:
+            return ToolResult(code=400, result=f"Error executing tool prompt_handler: {e}", tool_name="prompt_handler")
+
+        try:
+            return ToolResult(**json.loads(resp))
+        except Exception as e:
+            return ToolResult(
+                code=499,
+                result=f"Could not parse tool response as JSON: {resp}",
+                extra=str(e),
+                tool_name="prompt_handler",
+            )
+
+
+def main():
+    """
+    Start the MCP server.
+    """
+    agent = FlowceptAgent().start()
+    # Wake up tool call
+    print(run_tool(check_liveness, host=AGENT_HOST, port=AGENT_PORT)[0])
+    agent.wait()
+
+
+if __name__ == "__main__":
+    main()

{flowcept-0.9.18 → flowcept-0.9.19}/src/flowcept/agents/flowcept_ctx_manager.py

@@ -103,7 +103,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         self.schema_tracker = DynamicSchemaTracker(**self.tracker_config)
         self.msgs_counter = 0
         self.context_chunk_size = 1  # Should be in the settings
-        super().__init__()
+        super().__init__(allow_mq_disabled=True)
 
     def message_handler(self, msg_obj: Dict):
         """
@@ -133,12 +133,15 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
                 if task_msg.activity_id == "reset_user_context":
                     self.context.reset_context()
                     self.msgs_counter = 0
-                    FlowceptTask(
-                        agent_id=self.agent_id,
-                        generated={"msg": "Provenance Agent reset context."},
-                        subtype="agent_task",
-                        activity_id="reset_user_context",
-                    ).send()
+                    if self._mq_dao is None:
+                        self.logger.warning("MQ is disabled; skipping reset_user_context response message.")
+                    else:
+                        FlowceptTask(
+                            agent_id=self.agent_id,
+                            generated={"msg": "Provenance Agent reset context."},
+                            subtype="agent_task",
+                            activity_id="reset_user_context",
+                        ).send()
                     return True
                 elif task_msg.activity_id == "provenance_query":
                     self.logger.info("Received a prov query message!")
@@ -161,14 +164,17 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
                         status = Status.ERROR
                         error = f"Could not convert the following into a ToolResult:\n{resp}\nException: {e}"
                         generated = {"text": str(resp)}
-                    FlowceptTask(
-                        agent_id=self.agent_id,
-                        generated=generated,
-                        stderr=error,
-                        status=status,
-                        subtype="agent_task",
-                        activity_id="provenance_query_response",
-                    ).send()
+                    if self._mq_dao is None:
+                        self.logger.warning("MQ is disabled; skipping provenance_query response message.")
+                    else:
+                        FlowceptTask(
+                            agent_id=self.agent_id,
+                            generated=generated,
+                            stderr=error,
+                            status=status,
+                            subtype="agent_task",
+                            activity_id="provenance_query_response",
+                        ).send()
 
                     return True
 
@@ -200,12 +206,10 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
                         ]
                     )
                 except Exception as e:
-                    self.logger.error(
-                        f"Could not add these tasks to buffer!\n"
-                        f"{
-                            self.context.task_summaries[self.msgs_counter - self.context_chunk_size : self.msgs_counter]
-                        }"
-                    )
+                    task_slice = self.context.task_summaries[
+                        self.msgs_counter - self.context_chunk_size : self.msgs_counter
+                    ]
+                    self.logger.error(f"Could not add these tasks to buffer!\n{task_slice}")
                     self.logger.exception(e)
 
                 # self.monitor_chunk()
@@ -232,9 +236,12 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         if len(result):
             content = result[0].text
             if content != "Error executing tool":
-                msg = {"type": "flowcept_agent", "info": "monitor", "content": content}
-                self._mq_dao.send_message(msg)
-                self.logger.debug(str(content))
+                if self._mq_dao is None:
+                    self.logger.warning("MQ is disabled; skipping monitor message.")
+                else:
+                    msg = {"type": "flowcept_agent", "info": "monitor", "content": content}
+                    self._mq_dao.send_message(msg)
+                    self.logger.debug(str(content))
             else:
                 self.logger.error(content)
 

{flowcept-0.9.18 → flowcept-0.9.19}/src/flowcept/commons/daos/keyvalue_dao.py

@@ -1,7 +1,5 @@
 """Key value module."""
 
-from flowcept.commons.daos.redis_conn import RedisConn
-
 from flowcept.commons.flowcept_logger import FlowceptLogger
 from flowcept.configs import (
     KVDB_HOST,
@@ -26,12 +24,23 @@ class KeyValueDAO:
 
     def __init__(self):
         if not hasattr(self, "_initialized"):
-            self._initialized = True
             self.logger = FlowceptLogger()
+            from flowcept.commons.daos.redis_conn import RedisConn
+
             self.redis_conn = RedisConn.build_redis_conn_pool(
                 host=KVDB_HOST, port=KVDB_PORT, password=KVDB_PASSWORD, uri=KVDB_URI
             )
 
+            self._initialized = True
+
+    @staticmethod
+    def get_set_name(set_id: str, exec_bundle_id=None) -> str:
+        """Return a consistent set name for KVDB sets."""
+        set_name = set_id
+        if exec_bundle_id is not None:
+            set_name += "_" + str(exec_bundle_id)
+        return set_name
+
     def delete_set(self, set_name: str):
         """Delete it."""
         self.redis_conn.delete(set_name)

{flowcept-0.9.18 → flowcept-0.9.19}/src/flowcept/commons/daos/mq_dao/mq_dao_base.py

@@ -7,6 +7,7 @@ import msgpack
 from time import time
 import flowcept.commons
 from flowcept.commons.autoflush_buffer import AutoflushBuffer
+from flowcept.commons.daos.keyvalue_dao import KeyValueDAO
 from flowcept.commons.utils import chunked
 from flowcept.commons.flowcept_logger import FlowceptLogger
 from flowcept.configs import (
@@ -29,6 +30,8 @@ class MQDao(object):
 
     ENCODER = GenericJSONEncoder if JSON_SERIALIZER == "complex" else None
     # TODO we don't have a unit test to cover complex dict!
+    MQ_THREAD_SET_ID = "started_mq_thread_execution"
+    MQ_FLUSH_COMPLETE_SET_ID = "pending_mq_flush_complete"
 
     @staticmethod
     def build(*args, **kwargs) -> "MQDao":
@@ -51,20 +54,6 @@ class MQDao(object):
         else:
             raise NotImplementedError
 
-    @staticmethod
-    def _get_set_name(exec_bundle_id=None):
-        """Get the set name.
-
-        :param exec_bundle_id: A way to group one or many interceptors, and
-         treat each group as a bundle to control when their time_based
-         threads started and ended.
-        :return:
-        """
-        set_id = "started_mq_thread_execution"
-        if exec_bundle_id is not None:
-            set_id += "_" + str(exec_bundle_id)
-        return set_id
-
     def __init__(self, adapter_settings=None):
         self.logger = FlowceptLogger()
         self.started = False
@@ -103,22 +92,36 @@ class MQDao(object):
 
     def register_time_based_thread_init(self, interceptor_instance_id: str, exec_bundle_id=None):
         """Register the time."""
-        set_name = MQDao._get_set_name(exec_bundle_id)
+        set_name = KeyValueDAO.get_set_name(MQDao.MQ_THREAD_SET_ID, exec_bundle_id)
         # self.logger.info(
         #     f"Register start of time_based MQ flush thread {set_name}.{interceptor_instance_id}"
         # )
         self._keyvalue_dao.add_key_into_set(set_name, interceptor_instance_id)
+        flush_set_name = KeyValueDAO.get_set_name(MQDao.MQ_FLUSH_COMPLETE_SET_ID, exec_bundle_id)
+        self._keyvalue_dao.add_key_into_set(flush_set_name, interceptor_instance_id)
 
     def register_time_based_thread_end(self, interceptor_instance_id: str, exec_bundle_id=None):
         """Register time."""
-        set_name = MQDao._get_set_name(exec_bundle_id)
+        set_name = KeyValueDAO.get_set_name(MQDao.MQ_THREAD_SET_ID, exec_bundle_id)
         self.logger.info(f"Registering end of time_based MQ flush thread {set_name}.{interceptor_instance_id}")
         self._keyvalue_dao.remove_key_from_set(set_name, interceptor_instance_id)
         self.logger.info(f"Done registering time_based MQ flush thread {set_name}.{interceptor_instance_id}")
 
     def all_time_based_threads_ended(self, exec_bundle_id=None):
         """Get all time."""
-        set_name = MQDao._get_set_name(exec_bundle_id)
+        set_name = KeyValueDAO.get_set_name(MQDao.MQ_THREAD_SET_ID, exec_bundle_id)
+        return self._keyvalue_dao.set_is_empty(set_name)
+
+    def register_flush_complete(self, interceptor_instance_id: str, exec_bundle_id=None):
+        """Register a flush-complete signal for an interceptor."""
+        set_name = KeyValueDAO.get_set_name(MQDao.MQ_FLUSH_COMPLETE_SET_ID, exec_bundle_id)
+        self.logger.info(f"Registering flush completion {set_name}.{interceptor_instance_id}")
+        self._keyvalue_dao.remove_key_from_set(set_name, interceptor_instance_id)
+        self.logger.info(f"Done registering flush completion {set_name}.{interceptor_instance_id}")
+
+    def all_flush_complete_received(self, exec_bundle_id=None):
+        """Return True when all interceptors in the bundle reported flush completion."""
+        set_name = KeyValueDAO.get_set_name(MQDao.MQ_FLUSH_COMPLETE_SET_ID, exec_bundle_id)
         return self._keyvalue_dao.set_is_empty(set_name)
 
     def set_campaign_id(self, campaign_id=None):
@@ -172,11 +175,14 @@ class MQDao(object):
             if self._time_based_flushing_started:
                 self.buffer.stop()
                 self._time_based_flushing_started = False
+                self.logger.debug("MQ time-based flushed for the last time!")
             else:
                 self.logger.error("MQ time-based flushing is not started")
         else:
             self.buffer = list()
 
+        self.logger.debug("Buffer closed.")
+
     def _stop_timed(self, interceptor_instance_id: str, check_safe_stops: bool = True, bundle_exec_id: int = None):
         t1 = time()
         self._stop(interceptor_instance_id, check_safe_stops, bundle_exec_id)
@@ -190,10 +196,12 @@ class MQDao(object):
 
     def _stop(self, interceptor_instance_id: str = None, check_safe_stops: bool = True, bundle_exec_id: int = None):
         """Stop MQ publisher."""
-        self.logger.debug(f"MQ pub received stop sign: bundle={bundle_exec_id}, interceptor={interceptor_instance_id}")
         self._close_buffer()
-        self.logger.debug("Flushed MQ for the last time!")
-        if check_safe_stops:
+        if check_safe_stops and MQ_ENABLED:
+            self.logger.debug(
+                f"Sending flush-complete msg. Bundle: {bundle_exec_id}; interceptor id: {interceptor_instance_id}"
+            )
+            self._send_mq_dao_flush_complete(interceptor_instance_id, bundle_exec_id)
             self.logger.debug(f"Sending stop msg. Bundle: {bundle_exec_id}; interceptor id: {interceptor_instance_id}")
             self._send_mq_dao_time_thread_stop(interceptor_instance_id, bundle_exec_id)
         self.started = False
@@ -210,6 +218,15 @@ class MQDao(object):
         # self.logger.info("Control msg sent: " + str(msg))
         self.send_message(msg)
 
+    def _send_mq_dao_flush_complete(self, interceptor_instance_id, exec_bundle_id=None):
+        msg = {
+            "type": "flowcept_control",
+            "info": "mq_flush_complete",
+            "interceptor_instance_id": interceptor_instance_id,
+            "exec_bundle_id": exec_bundle_id,
+        }
+        self.send_message(msg)
+
     def send_document_inserter_stop(self, exec_bundle_id=None):
         """Send the document."""
         # These control_messages are handled by the document inserter

{flowcept-0.9.18 → flowcept-0.9.19}/src/flowcept/configs.py

@@ -9,7 +9,7 @@ from flowcept.version import __version__
 PROJECT_NAME = "flowcept"
 
 DEFAULT_SETTINGS = {
-    "version": __version__,
+    "flowcept_version": __version__,
     "log": {"log_file_level": "disable", "log_stream_level": "disable"},
     "project": {"dump_buffer": {"enabled": True}},
     "telemetry_capture": {},
@@ -81,7 +81,7 @@ FLOWCEPT_USER = settings["experiment"].get("user", "blank_user")
 
 MQ_INSTANCES = settings["mq"].get("instances", None)
 MQ_SETTINGS = settings["mq"]
-MQ_ENABLED = os.getenv("MQ_ENABLED", settings["mq"].get("enabled", True))
+MQ_ENABLED = os.getenv("MQ_ENABLED", str(settings["mq"].get("enabled", True))).strip().lower() in _TRUE_VALUES
 MQ_TYPE = os.getenv("MQ_TYPE", settings["mq"].get("type", "redis"))
 MQ_CHANNEL = os.getenv("MQ_CHANNEL", settings["mq"].get("channel", "interception"))
 MQ_PASSWORD = settings["mq"].get("password", None)
@@ -103,6 +103,11 @@ KVDB_PORT = int(os.getenv("KVDB_PORT", settings["kv_db"].get("port", "6379")))
 KVDB_URI = os.getenv("KVDB_URI", settings["kv_db"].get("uri", None))
 KVDB_ENABLED = settings["kv_db"].get("enabled", False)
 
+if MQ_ENABLED and not KVDB_ENABLED:
+    raise ValueError(
+        "Invalid configuration: MQ is enabled but kv_db is disabled. "
+        "Enable kv_db.enabled (and KVDB) when MQ is enabled."
+    )
 
 DATABASES = settings.get("databases", {})
 
@@ -160,6 +165,12 @@ JSON_SERIALIZER = settings["project"].get("json_serializer", "default")
 REPLACE_NON_JSON_SERIALIZABLE = settings["project"].get("replace_non_json_serializable", True)
 ENRICH_MESSAGES = settings["project"].get("enrich_messages", True)
 
+if DB_FLUSH_MODE == "online" and not MQ_ENABLED:
+    raise ValueError(
+        "Invalid configuration: project.db_flush_mode is 'online' but MQ is disabled. "
+        "Enable mq.enabled (or MQ_ENABLED=true) or set project.db_flush_mode to 'offline'."
+    )
+
 # Default: enable dump buffer only when running in offline flush mode.
 _DEFAULT_DUMP_BUFFER_ENABLED = DB_FLUSH_MODE == "offline"
 DUMP_BUFFER_ENABLED = (

{flowcept-0.9.18 → flowcept-0.9.19}/src/flowcept/flowceptor/consumers/agent/base_agent_context_manager.py

@@ -45,12 +45,12 @@ class BaseAgentContextManager(BaseConsumer):
 
     agent_id = None
 
-    def __init__(self):
+    def __init__(self, allow_mq_disabled: bool = False):
         """
         Initializes the agent and resets its context state.
         """
         self._started = False
-        super().__init__()
+        super().__init__(allow_mq_disabled=allow_mq_disabled)
         # self.context = BaseAppContext(tasks=[])
         self.agent_id = BaseAgentContextManager.agent_id