flowcept 0.9.9__py3-none-any.whl → 0.9.11__py3-none-any.whl

flowcept/__init__.py

@@ -41,6 +41,11 @@ def __getattr__(name):
 
         return FlowceptLoop
 
+    elif name == "FlowceptLightweightLoop":
+        from flowcept.instrumentation.flowcept_loop import FlowceptLightweightLoop
+
+        return FlowceptLightweightLoop
+
     elif name == "telemetry_flowcept_task":
         from flowcept.instrumentation.flowcept_task import telemetry_flowcept_task
 
@@ -59,7 +64,7 @@ def __getattr__(name):
         return FlowceptDaskWorkerAdapter
 
     elif name == "SETTINGS_PATH":
-        from configs import SETTINGS_PATH
+        from flowcept.configs import SETTINGS_PATH
 
         return SETTINGS_PATH
     elif name == "TaskQueryAPI":
@@ -74,6 +79,7 @@ __all__ = [
     "TaskQueryAPI",
     "flowcept_task",
     "FlowceptLoop",
+    "FlowceptLightweightLoop",
     "FlowceptTask",
     "telemetry_flowcept_task",
     "lightweight_flowcept_task",

flowcept/agents/gui/agent_gui.py

@@ -39,9 +39,10 @@ def main():
     st.caption(
         "💡 Quick help\n"
         "Ask about workflow metrics, plots, or summaries.\n\n"
-        "Inputs → used, outputs → generated.\n"
-        "Commands: @record <note> add • @show records list • reset context clear • save context save.\n"
-        "Tip: set result = df to run ad-hoc Python on the in-memory DataFrame."
+        "I have an internal DataFrame in my context to which you can ask direct questions."
+        "Tasks inputs are mapped to `used.*` fields, and outputs to `generated.*`\n"
+        "Commands: `@record <note>`; \n `@show records`; \n  `reset context` ; `save context` \n"
+        "Tip: Inputs like `result = df[some valid df query]` will run direct queries to the df in context."
     )
 
     user_input = st.chat_input("Send a message")

flowcept/agents/tools/general_tools.py

@@ -104,25 +104,6 @@ def reset_records() -> ToolResult:
     except Exception as e:
         return ToolResult(code=499, result=str(e))
 
-@mcp_flowcept.tool()
-def show_records() -> ToolResult:
-    """
-    Lists all recorded user guidance.
-    """
-    try:
-        ctx = mcp_flowcept.get_context()
-        custom_guidance: List = ctx.request_context.lifespan_context.custom_guidance
-        if not custom_guidance:
-            message = "There is no recorded user guidance."
-        else:
-            message = "This is the list of custom guidance I have in my memory:"
-            message += "\n".join(f"- {msg}" for msg in custom_guidance)
-
-        return ToolResult(code=201, result=message)
-    except Exception as e:
-        return ToolResult(code=499, result=str(e))
-
-
 
 @mcp_flowcept.tool()
 def prompt_handler(message: str) -> ToolResult:

flowcept/cli.py

@@ -106,7 +106,7 @@ def stream_messages(messages_file_path: Optional[str] = None, keys_to_show: List
     Listen to Flowcept's message stream and optionally echo/save messages.
 
     Parameters.
-    ----------
+    -----------
     messages_file_path : str, optional
         If provided, append each message as JSON (one per line) to this file.
         If the file already exists, a new timestamped file is created instead.
@@ -350,7 +350,7 @@ def agent_client(tool_name: str, kwargs: str = None):
     """Agent Client.
 
     Parameters.
-    ----------
+    -----------
     tool_name : str
         Name of the tool
     kwargs : str, optional
@@ -459,23 +459,85 @@ def start_mongo() -> None:
         databases:
             mongodb:
               - bin : str (required) path to the mongod executable
+              - db_path: str, required path to the db data directory
               - log_path : str, optional (adds --fork --logpath)
               - lock_file_path : str, optional (adds --pidfilepath)
 
+
     Builds and runs the startup command.
     """
+    import time
+    import socket
+    from flowcept.configs import MONGO_HOST, MONGO_PORT, MONGO_URI
+
+    def _port_open(host: str, port: int, timeout: float = 0.5) -> bool:
+        try:
+            with socket.create_connection((host, port), timeout=timeout):
+                return True
+        except OSError:
+            return False
+
+    def _await_mongo(host: str, port: int, uri: str | None, timeout: float = 20.0) -> bool:
+        """Wait until MongoDB is accepting connections (and ping if pymongo is available)."""
+        deadline = time.time() + timeout
+        have_pymongo = False
+        try:
+            from pymongo import MongoClient  # optional
+
+            have_pymongo = True
+        except Exception:
+            pass
+
+        while time.time() < deadline:
+            if not _port_open(host, port):
+                time.sleep(0.25)
+                continue
+
+            if not have_pymongo:
+                return True  # port is open; assume OK
+
+            try:
+                from pymongo import MongoClient
+
+                client = MongoClient(uri or f"mongodb://{host}:{port}", serverSelectionTimeoutMS=800)
+                client.admin.command("ping")
+                return True
+            except Exception:
+                time.sleep(0.25)
+
+        return False
+
+    def _tail(path: str, lines: int = 40) -> str:
+        try:
+            with open(path, "rb") as f:
+                f.seek(0, os.SEEK_END)
+                size = f.tell()
+                block = 1024
+                data = b""
+                while size > 0 and data.count(b"\n") <= lines:
+                    size = max(0, size - block)
+                    f.seek(size)
+                    data = f.read(min(block, size)) + data
+                return data.decode(errors="replace").splitlines()[-lines:]
+        except Exception:
+            return []
+
     # Safe nested gets
     settings = getattr(configs, "settings", {}) or {}
     databases = settings.get("databases") or {}
     mongodb = databases.get("mongodb") or {}
 
     bin_path = mongodb.get("bin")
-    log_path = mongodb.get("log_path")
-    lock_file_path = mongodb.get("lock_file_path")
+    db_path = mongodb.get("db_path")
+    log_path = mongodb.get("log_path", None)
+    lock_file_path = mongodb.get("lock_file_path", None)
 
     if not bin_path:
         print("Error: settings['databases']['mongodb']['bin'] is required.")
         return
+    if not db_path:
+        print("Error: settings['databases']['mongodb']['db_path'] is required.")
+        return
 
     # Build command
     parts = [shlex.quote(str(bin_path))]
@@ -483,12 +545,29 @@ def start_mongo() -> None:
         parts += ["--fork", "--logpath", shlex.quote(str(log_path))]
     if lock_file_path:
         parts += ["--pidfilepath", shlex.quote(str(lock_file_path))]
+    if db_path:
+        parts += ["--dbpath", shlex.quote(str(db_path))]
 
     cmd = " ".join(parts)
     try:
+        # Background start returns immediately because --fork is set
         out = _run_command(cmd, check_output=True)
         if out:
             print(out)
+        print(f"mongod launched (logs: {log_path}). Waiting for readiness on {MONGO_HOST}:{MONGO_PORT} ...")
+
+        ok = _await_mongo(MONGO_HOST, MONGO_PORT, MONGO_URI, timeout=20.0)
+        if ok:
+            print("✅ MongoDB is up and responding.")
+        else:
+            print("❌ MongoDB did not become ready in time.")
+            if log_path:
+                last_lines = _tail(log_path, 60)
+                if last_lines:
+                    print("---- mongod last log lines ----")
+                    for line in last_lines:
+                        print(line)
+                    print("---- end ----")
     except subprocess.CalledProcessError as e:
         print(f"Failed to start MongoDB: {e}")
 

flowcept/commons/daos/mq_dao/mq_dao_base.py

@@ -19,7 +19,6 @@ from flowcept.configs import (
     MQ_TIMING,
     KVDB_ENABLED,
     MQ_ENABLED,
-    DUMP_BUFFER_PATH,
 )
 
 from flowcept.commons.utils import GenericJSONEncoder
@@ -96,22 +95,11 @@ class MQDao(object):
     def bulk_publish(self, buffer):
         """Publish it."""
         # self.logger.info(f"Going to flush {len(buffer)} to MQ...")
-        if flowcept.configs.DB_FLUSH_MODE == "offline":
-            if DUMP_BUFFER_PATH is not None:
-                import orjson
-
-                with open(DUMP_BUFFER_PATH, "wb", buffering=1_048_576) as f:
-                    for obj in buffer:
-                        obj.pop("data", None)  # We are not going to store data in the buffer file.
-                        f.write(orjson.dumps(obj))
-                        f.write(b"\n")
-                self.logger.info(f"Saved Flowcept messages into {DUMP_BUFFER_PATH}.")
+        if MQ_CHUNK_SIZE > 1:
+            for chunk in chunked(buffer, MQ_CHUNK_SIZE):
+                self._bulk_publish(chunk)
         else:
-            if MQ_CHUNK_SIZE > 1:
-                for chunk in chunked(buffer, MQ_CHUNK_SIZE):
-                    self._bulk_publish(chunk)
-            else:
-                self._bulk_publish(buffer)
+            self._bulk_publish(buffer)
 
     def register_time_based_thread_init(self, interceptor_instance_id: str, exec_bundle_id=None):
         """Register the time."""
@@ -174,6 +162,12 @@ class MQDao(object):
             self.started = True
 
     def _close_buffer(self):
+        if flowcept.configs.DUMP_BUFFER_ENABLED and flowcept.configs.DUMP_BUFFER_PATH is not None:
+            from flowcept.commons.utils import buffer_to_disk
+
+            _buf = self.buffer.current_buffer if isinstance(self.buffer, AutoflushBuffer) else self.buffer
+            buffer_to_disk(_buf, flowcept.configs.DUMP_BUFFER_PATH, self.logger)
+
         if flowcept.configs.DB_FLUSH_MODE == "online":
             if self._time_based_flushing_started:
                 self.buffer.stop()
@@ -181,7 +175,6 @@ class MQDao(object):
             else:
                 self.logger.error("MQ time-based flushing is not started")
         else:
-            self.bulk_publish(self.buffer)
             self.buffer = list()
 
     def _stop_timed(self, interceptor_instance_id: str, check_safe_stops: bool = True, bundle_exec_id: int = None):

flowcept/commons/utils.py

@@ -4,7 +4,7 @@ import argparse
 from datetime import datetime, timedelta, timezone
 import json
 from time import time, sleep
-from typing import Callable
+from typing import Callable, List, Dict
 import os
 import platform
 import subprocess
@@ -245,6 +245,24 @@ def get_current_config_values():
     return _vars
 
 
+def buffer_to_disk(buffer: List[Dict], path: str, logger):
+    """
+    Append the in-memory buffer to a JSON Lines (JSONL) file on disk.
+    """
+    if not buffer:
+        logger.warning("The buffer is currently empty.")
+        return
+    with open(path, "ab", buffering=1_048_576) as f:
+        for obj in buffer:
+            obj.pop("data", None)  # We are not going to store data in the buffer file.
+            from orjson import orjson
+
+            f.write(orjson.dumps(obj))
+            f.write(b"\n")
+
+    logger.info(f"Saved Flowcept buffer into {path}.")
+
+
 class GenericJSONDecoder(json.JSONDecoder):
     """JSON decoder class."""
 

flowcept/commons/vocabulary.py

@@ -35,3 +35,23 @@ class Status(str, Enum):
     def get_finished_statuses():
         """Get finished status."""
         return [Status.FINISHED, Status.ERROR]
+
+
+class MimeType(Enum):
+    """MimeTypes used in Flowcept."""
+
+    JPEG = "image/jpeg"
+    PNG = "image/png"
+    GIF = "image/gif"
+    BMP = "image/bmp"
+    TIFF = "image/tiff"
+    WEBP = "image/webp"
+    SVG = "image/svg+xml"
+
+    # Documents
+    PDF = "application/pdf"
+
+    # Data formats
+    JSON = "application/json"
+    CSV = "text/csv"
+    JSONL = "application/x-ndjson"  # standard for JSON Lines

flowcept/configs.py

@@ -11,7 +11,7 @@ PROJECT_NAME = "flowcept"
 DEFAULT_SETTINGS = {
     "version": __version__,
     "log": {"log_file_level": "disable", "log_stream_level": "disable"},
-    "project": {"dump_buffer_path": "flowcept_messages.jsonl"},
+    "project": {"dump_buffer": {"enabled": True}},
     "telemetry_capture": {},
     "instrumentation": {},
     "experiment": {},
@@ -27,7 +27,9 @@ DEFAULT_SETTINGS = {
     "agent": {},
 }
 
-USE_DEFAULT = os.getenv("FLOWCEPT_USE_DEFAULT", "False").lower() == "true"
+_TRUE_VALUES = {"1", "true", "yes", "y", "t"}
+
+USE_DEFAULT = os.getenv("FLOWCEPT_USE_DEFAULT", "False").lower() in _TRUE_VALUES
 
 if USE_DEFAULT:
     settings = DEFAULT_SETTINGS.copy()
@@ -86,8 +88,8 @@ MQ_PASSWORD = settings["mq"].get("password", None)
 MQ_HOST = os.getenv("MQ_HOST", settings["mq"].get("host", "localhost"))
 MQ_PORT = int(os.getenv("MQ_PORT", settings["mq"].get("port", "6379")))
 MQ_URI = os.getenv("MQ_URI", settings["mq"].get("uri", None))
-MQ_BUFFER_SIZE = settings["mq"].get("buffer_size", None)
-MQ_INSERTION_BUFFER_TIME = settings["mq"].get("insertion_buffer_time_secs", None)
+MQ_BUFFER_SIZE = settings["mq"].get("buffer_size", 1)
+MQ_INSERTION_BUFFER_TIME = settings["mq"].get("insertion_buffer_time_secs", 1)
 MQ_TIMING = settings["mq"].get("timing", False)
 MQ_CHUNK_SIZE = int(settings["mq"].get("chunk_size", -1))
 
@@ -158,10 +160,21 @@ PERF_LOG = settings["project"].get("performance_logging", False)
 JSON_SERIALIZER = settings["project"].get("json_serializer", "default")
 REPLACE_NON_JSON_SERIALIZABLE = settings["project"].get("replace_non_json_serializable", True)
 ENRICH_MESSAGES = settings["project"].get("enrich_messages", True)
-DUMP_BUFFER_PATH = settings["project"].get("dump_buffer_path", "flowcept_messages.jsonl")
+
+_DEFAULT_DUMP_BUFFER_ENABLED = DB_FLUSH_MODE == "offline"
+DUMP_BUFFER_ENABLED = (
+    os.getenv(
+        "DUMP_BUFFER", str(settings["project"].get("dump_buffer", {}).get("enabled", _DEFAULT_DUMP_BUFFER_ENABLED))
+    )
+    .strip()
+    .lower()
+    in _TRUE_VALUES
+)
+DUMP_BUFFER_PATH = settings["project"].get("dump_buffer", {}).get("path", "flowcept_buffer.jsonl")
 
 TELEMETRY_CAPTURE = settings.get("telemetry_capture", None)
-TELEMETRY_ENABLED = TELEMETRY_CAPTURE is not None and len(TELEMETRY_CAPTURE)
+TELEMETRY_ENABLED = os.getenv("TELEMETRY_ENABLED", "true").strip().lower() in _TRUE_VALUES
+TELEMETRY_ENABLED = TELEMETRY_ENABLED and (TELEMETRY_CAPTURE is not None) and (len(TELEMETRY_CAPTURE) > 0)
 
 ######################
 # SYS METADATA #
@@ -235,13 +248,9 @@ INSTRUMENTATION = settings.get("instrumentation", {})
 INSTRUMENTATION_ENABLED = INSTRUMENTATION.get("enabled", True)
 
 AGENT = settings.get("agent", {})
-AGENT_AUDIO = os.getenv("AGENT_AUDIO", str(settings["agent"].get("audio_enabled", "false"))).strip().lower() in {
-    "1",
-    "true",
-    "yes",
-    "y",
-    "t",
-}
+AGENT_AUDIO = (
+    os.getenv("AGENT_AUDIO", str(settings["agent"].get("audio_enabled", "false"))).strip().lower() in _TRUE_VALUES
+)
 AGENT_HOST = os.getenv("AGENT_HOST", settings["agent"].get("mcp_host", "localhost"))
 AGENT_PORT = int(os.getenv("AGENT_PORT", settings["agent"].get("mcp_port", "8000")))
 

flowcept/flowcept_api/flowcept_controller.py

@@ -1,5 +1,6 @@
 """Controller module."""
 
+import os
 from typing import List, Dict, Any
 from uuid import uuid4
 
@@ -9,7 +10,7 @@ from flowcept.commons.flowcept_dataclasses.workflow_object import (
     WorkflowObject,
 )
 from flowcept.commons.flowcept_logger import FlowceptLogger
-from flowcept.commons.utils import ClassProperty
+from flowcept.commons.utils import ClassProperty, buffer_to_disk
 from flowcept.configs import (
     MQ_INSTANCES,
     INSTRUMENTATION_ENABLED,
@@ -44,7 +45,7 @@ class Flowcept(object):
     def __init__(
         self,
         interceptors: List[str] = None,
-        bundle_exec_id=None,
+        bundle_exec_id: str = None,
         campaign_id: str = None,
         workflow_id: str = None,
         workflow_name: str = None,
@@ -52,6 +53,7 @@ class Flowcept(object):
         start_persistence=True,
         check_safe_stops=True,  # TODO add to docstring
         save_workflow=True,
+        delete_buffer_file=True,
         *args,
         **kwargs,
     ):
@@ -68,7 +70,7 @@ class Flowcept(object):
             Examples: "instrumentation", "dask", "mlflow", ...
             The order of interceptors matters — place the outer-most interceptor first,
 
-        bundle_exec_id : Any, optional
+        bundle_exec_id : str, optional
             Identifier for grouping interceptors in a bundle, essential for the correct initialization and stop of
             interceptors. If not provided, a unique ID is assigned.
 
@@ -90,6 +92,9 @@ class Flowcept(object):
         save_workflow : bool, default=True
             If True, a workflow object message is sent.
 
+        delete_buffer_file : bool, default=True
+            if True, deletes an existing existing buffer file or ignores if it doesn't exist.
+
         Additional arguments (`*args`, `**kwargs`) are used for specific adapters.
             For example, when using the Dask interceptor, the `dask_client` argument
             should be provided in `kwargs` to enable saving the Dask workflow, which is recommended.
@@ -101,9 +106,9 @@ class Flowcept(object):
         self.buffer = None
         self._check_safe_stops = check_safe_stops
         if bundle_exec_id is None:
-            self._bundle_exec_id = id(self)
+            self.bundle_exec_id = str(id(self))
         else:
-            self._bundle_exec_id = bundle_exec_id
+            self.bundle_exec_id = str(bundle_exec_id)
 
         self.enabled = True
         self.is_started = False
@@ -129,8 +134,11 @@ class Flowcept(object):
         self.workflow_name = workflow_name
         self.workflow_args = workflow_args
 
+        if delete_buffer_file:
+            Flowcept.delete_buffer_file()
+
     def start(self):
-        """Start it."""
+        """Start Flowcept Controller."""
         if self.is_started or not self.enabled:
             self.logger.warning("DB inserter may be already started or instrumentation is not set")
             return self
@@ -154,7 +162,7 @@ class Flowcept(object):
                 Flowcept.current_workflow_id = self.current_workflow_id
 
                 interceptor_inst = BaseInterceptor.build(interceptor)
-                interceptor_inst.start(bundle_exec_id=self._bundle_exec_id, check_safe_stops=self._check_safe_stops)
+                interceptor_inst.start(bundle_exec_id=self.bundle_exec_id, check_safe_stops=self._check_safe_stops)
                 self._interceptor_instances.append(interceptor_inst)
                 if isinstance(interceptor_inst._mq_dao.buffer, AutoflushBuffer):
                     Flowcept.buffer = self.buffer = interceptor_inst._mq_dao.buffer.current_buffer
@@ -170,11 +178,88 @@ class Flowcept(object):
         self.logger.debug("Flowcept started successfully.")
         return self
 
+    def get_buffer(self, return_df: bool = False):
+        """
+        Retrieve the in-memory message buffer.
+
+        Parameters
+        ----------
+        return_df : bool, optional
+            If False (default), return the raw buffer as a list of dictionaries.
+            If True, normalize the buffer into a pandas DataFrame with dotted
+            notation for nested keys. Requires ``pandas`` to be installed.
+
+        Returns
+        -------
+        list of dict or pandas.DataFrame
+            - If ``return_df=False``: the buffer as a list of dictionaries.
+            - If ``return_df=True``: the buffer as a normalized DataFrame.
+
+        Raises
+        ------
+        ModuleNotFoundError
+            If ``return_df=True`` but ``pandas`` is not installed.
+
+        Examples
+        --------
+        >>> buf = flowcept.get_buffer()
+        >>> isinstance(buf, list)
+        True
+
+        >>> df = flowcept.get_buffer(return_df=True)
+        >>> "generated.attention" in df.columns
+        True
+        """
+        if return_df:
+            try:
+                import pandas as pd
+            except ModuleNotFoundError as e:
+                raise ModuleNotFoundError("pandas is required when return_df=True. Please install pandas.") from e
+            return pd.json_normalize(self.buffer, sep=".")
+        return self.buffer
+
     def _publish_buffer(self):
         self._interceptor_instances[0]._mq_dao.bulk_publish(self.buffer)
 
+    def dump_buffer(self, path: str = None):
+        """
+        Dump the current in-memory buffer to a JSON Lines (JSONL) file.
+
+        Each element of the buffer (a dictionary) is serialized as a single line
+        of JSON. If no path is provided, the default path from the settings file
+        is used.
+
+        Parameters
+        ----------
+        path : str, optional
+            Destination file path for the JSONL output. If not provided,
+            defaults to ``DUMP_BUFFER_PATH`` as configured in the settings.
+
+        Returns
+        -------
+        None
+            The buffer is written to disk, no value is returned.
+
+        Notes
+        -----
+        - The buffer is expected to be a list of dictionaries.
+        - Existing files at the specified path will be overwritten.
+        - Logging is performed through the class logger.
+
+        Examples
+        --------
+        >>> flowcept.dump_buffer("buffer.jsonl")
+        # Writes buffer contents to buffer.jsonl
+
+        >>> flowcept.dump_buffer()
+        # Writes buffer contents to the default path defined in settings
+        """
+        if path is None:
+            path = DUMP_BUFFER_PATH
+        buffer_to_disk(self.buffer, path, self.logger)
+
     @staticmethod
-    def read_messages_file(file_path: str | None = None, return_df: bool = False):
+    def read_buffer_file(file_path: str | None = None, return_df: bool = False, normalize_df: bool = False):
         """
         Read a JSON Lines (JSONL) file containing captured Flowcept messages.
 
@@ -187,12 +272,15 @@ class Flowcept(object):
         Parameters
         ----------
         file_path : str, optional
-            Path to the messages file. If not provided, defaults to the value of
+            Path to the buffer file. If not provided, defaults to the value of
             ``DUMP_BUFFER_PATH`` from the configuration. If neither is provided,
             an assertion error is raised.
         return_df : bool, default False
             If True, return a normalized pandas DataFrame. If False, return the
             parsed list of dictionaries.
+        normalize_df: bool, default False
+            If True, normalize the inner dicts (e.g., used, generated, custom_metadata) as individual columns in the
+            returned DataFrame.
 
         Returns
         -------
@@ -215,13 +303,13 @@ class Flowcept(object):
         --------
         Read messages as a list:
 
-        >>> msgs = read_messages_file("offline_buffer.jsonl")
+        >>> msgs = read_buffer_file("offline_buffer.jsonl")
         >>> len(msgs) > 0
         True
 
         Read messages as a normalized DataFrame:
 
-        >>> df = read_messages_file("offline_buffer.jsonl", return_df=True)
+        >>> df = read_buffer_file("offline_buffer.jsonl", return_df=True)
         >>> "generated.attention" in df.columns
         True
         """
@@ -232,7 +320,7 @@ class Flowcept(object):
             file_path = DUMP_BUFFER_PATH
         assert file_path is not None, "Please indicate file_path either in the argument or in the config file."
         if not os.path.exists(file_path):
-            raise FileNotFoundError(f"File '{file_path}' was not found. It is created only in fully offline mode.")
+            raise FileNotFoundError(f"Flowcept buffer file '{file_path}' was not found.")
 
         with open(file_path, "rb") as f:
             lines = [ln for ln in f.read().splitlines() if ln]
@@ -244,10 +332,57 @@ class Flowcept(object):
                 import pandas as pd
             except ModuleNotFoundError as e:
                 raise ModuleNotFoundError("pandas is required when return_df=True. Please install pandas.") from e
-            return pd.json_normalize(buffer, sep=".")
+            if normalize_df:
+                return pd.json_normalize(buffer, sep=".")
+            else:
+                return pd.read_json(file_path, lines=True)
 
         return buffer
 
+    @staticmethod
+    def delete_buffer_file(path: str = None):
+        """
+        Delete the buffer file from disk if it exists.
+
+        If no path is provided, the default path from the settings file
+        is used. Logs whether the file was successfully removed or not found.
+
+        Parameters
+        ----------
+        path : str, optional
+            Path to the buffer JSONL file. If not provided,
+            defaults to ``DUMP_BUFFER_PATH`` as configured in the settings.
+
+        Returns
+        -------
+        None
+            The file is deleted from disk if it exists, no value is returned.
+
+        Notes
+        -----
+        - This operation only affects the file on disk. It does not clear
+          the in-memory buffer.
+        - Logging is performed through the class logger.
+
+        Examples
+        --------
+        >>> flowcept.delete_buffer_file("buffer.jsonl")
+        # Deletes buffer.jsonl if it exists
+
+        >>> flowcept.delete_buffer_file()
+        # Deletes the default buffer file defined in settings
+        """
+        if path is None:
+            path = DUMP_BUFFER_PATH
+
+        try:
+            if os.path.exists(path):
+                os.remove(path)
+                FlowceptLogger().info(f"Buffer file deleted: {path}")
+        except Exception as e:
+            FlowceptLogger().error(f"Failed to delete buffer file: {path}")
+            FlowceptLogger().exception(e)
+
     def save_workflow(self, interceptor: str, interceptor_instance: BaseInterceptor):
         """
         Save the current workflow and send its metadata using the provided interceptor.
@@ -297,12 +432,12 @@ class Flowcept(object):
 
         from flowcept.flowceptor.consumers.document_inserter import DocumentInserter
 
-        doc_inserter = DocumentInserter(check_safe_stops=self._check_safe_stops, bundle_exec_id=self._bundle_exec_id)
+        doc_inserter = DocumentInserter(check_safe_stops=self._check_safe_stops, bundle_exec_id=self.bundle_exec_id)
         doc_inserter.start()
         self._db_inserters.append(doc_inserter)
 
     def stop(self):
-        """Stop it."""
+        """Stop Flowcept controller."""
         if not self.is_started or not self.enabled:
             self.logger.warning("Flowcept is already stopped or may never have been started!")
             return
@@ -316,7 +451,7 @@ class Flowcept(object):
         if len(self._db_inserters):
             self.logger.info("Stopping DB Inserters...")
             for db_inserter in self._db_inserters:
-                db_inserter.stop(bundle_exec_id=self._bundle_exec_id)
+                db_inserter.stop(bundle_exec_id=self.bundle_exec_id)
 
         Flowcept.buffer = self.buffer = None
         self.is_started = False
@@ -346,8 +481,7 @@ class Flowcept(object):
         - The method tests the liveness of the MQ service using `MQDao`.
         - If `MONGO_ENABLED` is True, it also checks the liveness of the MongoDB service
           using `MongoDBDAO`.
-        - Logs errors if any service is not ready, and logs success when both services are
-        operational.
+        - Logs errors if any service is not ready, and logs success when both services are operational.
 
         Examples
         --------

flowcept/flowceptor/adapters/base_interceptor.py

@@ -10,6 +10,7 @@ from flowcept.commons.flowcept_dataclasses.workflow_object import (
 from flowcept.configs import (
     ENRICH_MESSAGES,
     TELEMETRY_ENABLED,
+    TELEMETRY_CAPTURE,
 )
 from flowcept.commons.flowcept_logger import FlowceptLogger
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
@@ -135,7 +136,7 @@ class BaseInterceptor(object):
             # TODO :base-interceptor-refactor: :code-reorg: :usability:
             raise Exception(f"This interceptor {id(self)} has never been started!")
         workflow_obj.interceptor_ids = [self._interceptor_instance_id]
-        if self.telemetry_capture:
+        if self.telemetry_capture and TELEMETRY_CAPTURE.get("machine_info", False):
             machine_info = self.telemetry_capture.capture_machine_info()
             if workflow_obj.machine_info is None:
                 workflow_obj.machine_info = dict()

flowcept/instrumentation/flowcept_loop.py

@@ -1,4 +1,4 @@
-"""FlowCept Loop module."""
+"""Flowcept Loop module."""
 
 import uuid
 from time import time
@@ -49,6 +49,7 @@ class FlowceptLoop:
         ----------
         items : Union[Sized, Iterator, int]
             The items to iterate over. Can be:
+
             - A sized iterable (e.g., list, range).
             - An integer (interpreted as ``range(items)``).
             - An iterator (requires ``items_length`` if length cannot be inferred).
@@ -216,8 +217,7 @@ class FlowceptLightweightLoop:
 
     The `FlowceptLightweightLoop` class supports iterating over a collection of items or a numeric
     range while capturing metadata for each iteration and for the loop as a whole.
-    This is particularly useful in scenarios where tracking and instrumentation of loop executions
-     is required.
+    This is particularly useful in scenarios where tracking and instrumentation of loop executions is required.
 
     Parameters
     ----------
@@ -269,6 +269,7 @@ class FlowceptLightweightLoop:
         ----------
         items : Union[Sized, Iterator]
             The items to iterate over. Must either be:
+
             - A sized iterable (with ``__len__``).
             - An explicit iterator (length must be given by ``items_length``).
         loop_name : str, optional

flowcept/version.py

@@ -4,4 +4,4 @@
 # The expected format is: <Major>.<Minor>.<Patch>
 # This file is supposed to be automatically modified by the CI Bot.
 # See .github/workflows/version_bumper.py
-__version__ = "0.9.9"
+__version__ = "0.9.11"

{flowcept-0.9.9.dist-info → flowcept-0.9.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.9
+Version: 0.9.11
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT
@@ -56,6 +56,7 @@ Requires-Dist: tensorflow; extra == 'all'
 Requires-Dist: tomli; extra == 'all'
 Requires-Dist: watchdog; extra == 'all'
 Provides-Extra: analytics
+Requires-Dist: matplotlib; extra == 'analytics'
 Requires-Dist: plotly; extra == 'analytics'
 Requires-Dist: scipy; extra == 'analytics'
 Requires-Dist: seaborn; extra == 'analytics'
@@ -219,7 +220,7 @@ def main():
 if __name__ == "__main__":
     main()
 
-    prov_messages = Flowcept.read_messages_file()
+    prov_messages = Flowcept.read_buffer_file()
     assert len(prov_messages) == 2
     print(json.dumps(prov_messages, indent=2))
 ```

{flowcept-0.9.9.dist-info → flowcept-0.9.11.dist-info}/RECORD

@@ -1,7 +1,7 @@
-flowcept/__init__.py,sha256=urpwIEJeikV0P6ORXKsM5Lq4o6wCwhySS9A487BYGy4,2241
-flowcept/cli.py,sha256=eVnUrmZtVhZ1ldRMGB1QsqBzNC1Pf2CX33efnlaZ4gs,22842
-flowcept/configs.py,sha256=aXgBkBpTs4_4MpvAe76aQ5lXl1gTmgk92bFiNqMQXPM,8382
-flowcept/version.py,sha256=ueMAMza92zxBIjD4eZdTMYArVaKDr6QL5eZ23IZjyd4,306
+flowcept/__init__.py,sha256=tvVZKyymdqv3qOsgpAyDppBlUiBc0ag4QF21IcS-mVk,2449
+flowcept/cli.py,sha256=AU6EuawCboCv933Ghb1xFL74UMIJeVTzNocPOoaJS0Q,25696
+flowcept/configs.py,sha256=DBkYx0CAaDSl8x2EJY1665PFY80eCp9PEriYH-BNwL4,8781
+flowcept/version.py,sha256=czeTBejrSGB7lfhNV-3pEINXGkwZD73Z8SRmBycnbvI,307
 flowcept/agents/__init__.py,sha256=8eeD2CiKBtHiDsWdrHK_UreIkKlTq4dUbhHDyzw372o,175
 flowcept/agents/agent_client.py,sha256=UiBQkC9WE2weLZR2OTkEOEQt9-zqQOkPwRA17HfI-jk,2027
 flowcept/agents/agents_utils.py,sha256=Az5lvWTsBHs_3sWWwy7jSdDjNn-PvZ7KmYd79wxvdyU,6666
@@ -9,7 +9,7 @@ flowcept/agents/dynamic_schema_tracker.py,sha256=TsmXRRkyUkqB-0bEgmeqSms8xj1tMMJ
 flowcept/agents/flowcept_agent.py,sha256=1sidjnNMdG0S6lUKBvml7ZfIb6o3u7zc6HNogsJbl9g,871
 flowcept/agents/flowcept_ctx_manager.py,sha256=-WmkddzzFY2dnU9LbZaoY4-5RcSAQH4FziEJgcC5LEI,7083
 flowcept/agents/gui/__init__.py,sha256=Qw9YKbAzgZqBjMQGnF7XWmfUo0fivtkDISQRK3LA3gU,113
-flowcept/agents/gui/agent_gui.py,sha256=44G6AXM9zJbVcJUziZcU5aA6mVDwtvgO8H58nTCFC0M,2855
+flowcept/agents/gui/agent_gui.py,sha256=VpwhQamzFKBfrmibxOIc-8wXtZnd2Cq7tbKahZZOp7c,2995
 flowcept/agents/gui/audio_utils.py,sha256=piA_dc36io1sYqLF6QArS4AMl-cfDa001jGhYz5LkB4,4279
 flowcept/agents/gui/gui_utils.py,sha256=cQVhOgnfxJNUVZyXyO8f40nB1yaKAKVtBrwQmJjL0B0,14933
 flowcept/agents/llms/__init__.py,sha256=kzOaJic5VhMBnGvy_Fr5C6sRKVrRntH1ZnYz7f5_4-s,23
@@ -19,7 +19,7 @@ flowcept/agents/prompts/__init__.py,sha256=7ICsNhLYzvPS1esG3Vg519s51b1c4yN0WegJU
 flowcept/agents/prompts/general_prompts.py,sha256=b0QhnF-ytIE1_WWrgpamC4VybjS8KuS051DgVVt8r2U,3961
 flowcept/agents/prompts/in_memory_query_prompts.py,sha256=iRaGySybNxZf5vuQ3n9cb14VNk6bMQ0z3tn2mVVke0E,19817
 flowcept/agents/tools/__init__.py,sha256=Xqz2E4-LL_7DDcm1XYJFx2f5RdAsjeTpOJb_DPC7xyc,27
-flowcept/agents/tools/general_tools.py,sha256=yOxNb7ZgrP93Ky85cdjiZkCZSH_1biB0pA2K1e9bzFQ,5038
+flowcept/agents/tools/general_tools.py,sha256=JSMG_UGdRKcQfC4_ixzDXDHW92UX5i0UsLTzFq0fmZg,4402
 flowcept/agents/tools/in_memory_queries/__init__.py,sha256=K8-JI_lXUgquKkgga8Nef8AntGg_logQtjjQjaEE7yI,39
 flowcept/agents/tools/in_memory_queries/in_memory_queries_tools.py,sha256=GcfAiUBhQ1DU3QKk0kAy9TSq8XmZw691Xs0beZoO76A,25984
 flowcept/agents/tools/in_memory_queries/pandas_agent_utils.py,sha256=xyrZupR86qoUptnnQ7PeF0LTzSOquEK2cjc0ghT1KBs,9018
@@ -33,8 +33,8 @@ flowcept/commons/flowcept_logger.py,sha256=0asRucrDMeRXvsdhuCmH6lWO7lAt_Z5o5uW7r
 flowcept/commons/query_utils.py,sha256=3tyK5VYA10iDtmtzNwa8OQGn93DBxsu6rTjHDphftSc,2208
 flowcept/commons/settings_factory.py,sha256=bMTjgXRfb5HsL2lPnLfem-9trqELbNWE04Ie7lSlxYM,1731
 flowcept/commons/task_data_preprocess.py,sha256=-ceLexv2ZfZOAYF43DPagGwQPgt_L_lNKuK8ZCpnzXs,13914
-flowcept/commons/utils.py,sha256=gF6ENWlTpR2ZSw3yVNPNBTVzSpcgy-WuzYzwWSXXsug,9252
-flowcept/commons/vocabulary.py,sha256=_GzHJ1wSYJlLsu_uu1Am6N3zvc59S4FCuT5yp7lynPw,713
+flowcept/commons/utils.py,sha256=okCShkcuWhzznBtADDDusTdfPXO0W041b2f4Aog-7SE,9831
+flowcept/commons/vocabulary.py,sha256=0psC4NulNFn88mjTcoT_aT4QxX8ljMFgTOF3FxzM40A,1118
 flowcept/commons/daos/__init__.py,sha256=RO51svfHOg9naN676zuQwbj_RQ6IFHu-RALeefvtwwk,23
 flowcept/commons/daos/keyvalue_dao.py,sha256=g7zgC9hVC1NTllwUAqGt44YqdqYUgAKgPlX8_G4BRGw,3599
 flowcept/commons/daos/redis_conn.py,sha256=gFyW-5yf6B8ExEYopCmbap8ki-iEwuIw-KH9f6o7UGQ,1495
@@ -43,7 +43,7 @@ flowcept/commons/daos/docdb_dao/docdb_dao_base.py,sha256=YbfSVJPwZGK2GBYkeapRC83
 flowcept/commons/daos/docdb_dao/lmdb_dao.py,sha256=ZuCsdEhI2wGAmjAf82j-1t3tbR6YMmDeaJ_C3HcsLYo,10461
 flowcept/commons/daos/docdb_dao/mongodb_dao.py,sha256=5x0un15uCDTcnuITOyOhvF9mKj_bUmF2du0AHQfjN9k,40055
 flowcept/commons/daos/mq_dao/__init__.py,sha256=Xxm4FmbBUZDQ7XIAmSFbeKE_AdHsbgFmSuftvMWSykQ,21
-flowcept/commons/daos/mq_dao/mq_dao_base.py,sha256=EL8eQedvNLsVLMz4oHemBAsR1S6xFZiezM8dIqKmmCA,9696
+flowcept/commons/daos/mq_dao/mq_dao_base.py,sha256=VXqXzesU01dCHE5i0urnYQppixUNGZbJMRmm4jSAcgM,9424
 flowcept/commons/daos/mq_dao/mq_dao_kafka.py,sha256=kjZqPLIu5PaNeM4IDvOxkDRVGTd5UWwq3zhDvVirqW8,5067
 flowcept/commons/daos/mq_dao/mq_dao_mofka.py,sha256=tRdMGYDzdeIJxad-B4-DE6u8Wzs61eTzOW4ojZrnTxs,4057
 flowcept/commons/daos/mq_dao/mq_dao_redis.py,sha256=ejBMxImA-h2KuMEAk3l7aU0chCcObCbUXEOXM6L4Zhc,5571
@@ -54,7 +54,7 @@ flowcept/commons/flowcept_dataclasses/telemetry.py,sha256=9_5ONCo-06r5nKHXmi5HfI
 flowcept/commons/flowcept_dataclasses/workflow_object.py,sha256=cauWtXHhBv9lHS-q6cb7yUsNiwQ6PkZPuSinR1TKcqU,6161
 flowcept/flowcept_api/__init__.py,sha256=T1ty86YlocQ5Z18l5fUqHj_CC6Unq_iBv0lFyiI7Ao8,22
 flowcept/flowcept_api/db_api.py,sha256=hKXep-n50rp9cAzV0ljk2QVEF8O64yxi3ujXv5_Ibac,9723
-flowcept/flowcept_api/flowcept_controller.py,sha256=jfssXUvG55RVXJBziq-lXekt7Dog3mAalo5Zsp_7_to,16060
+flowcept/flowcept_api/flowcept_controller.py,sha256=az1bktiL8_xs4pc97Zqgd1ezsg-cD0whf3XWA1ZN08Q,20652
 flowcept/flowcept_api/task_query_api.py,sha256=SrwB0OCVtbpvCPECkE2ySM10G_g8Wlk5PJ8h-0xEaNc,23821
 flowcept/flowcept_webserver/__init__.py,sha256=8411GIXGddKTKoHUvbo_Rq6svosNG7tG8VzvUEBd7WI,28
 flowcept/flowcept_webserver/app.py,sha256=VUV8_JZbIbx9u_1O7m7XtRdhZb_7uifUa-iNlPhmZws,658
@@ -64,7 +64,7 @@ flowcept/flowcept_webserver/resources/task_messages_rsrc.py,sha256=0u68it2W-9NzU
 flowcept/flowceptor/__init__.py,sha256=wVxRXUv07iNx6SMRRma2vqhR_GIcRl0re_WCYG65PUs,29
 flowcept/flowceptor/telemetry_capture.py,sha256=CWyR8E1rTAjFbUFI9BxaGfJyDd2UbiK0uLGt4m8BnSU,13932
 flowcept/flowceptor/adapters/__init__.py,sha256=SuZbSZVVQeBJ9zXW-M9jF09dw3XIjre3lSGrUO1Y8Po,27
-flowcept/flowceptor/adapters/base_interceptor.py,sha256=kbdYW6VuvmBibOVy7Pg3OzeD3OUaHc6jnAhRBpj9f14,6517
+flowcept/flowceptor/adapters/base_interceptor.py,sha256=oqnD19TNqi1FHlwAsbyEkiLgywpm5HYLHT1gYlsOHTk,6589
 flowcept/flowceptor/adapters/instrumentation_interceptor.py,sha256=DhK2bBnpghqPSeA62BUqRg6pl8zxuYrP33dK4x6PhRE,733
 flowcept/flowceptor/adapters/interceptor_state_manager.py,sha256=xRzmi5YFKBEqNtX8F5s6XlMTRe27ml4BmQtBO4WtG2c,919
 flowcept/flowceptor/adapters/brokers/__init__.py,sha256=mhQXVmh0JklvL93GUtJZLJnPRYX9Nmb8IqcyKJGQBzk,36
@@ -90,13 +90,13 @@ flowcept/flowceptor/consumers/agent/base_agent_context_manager.py,sha256=5fBPYs-
 flowcept/instrumentation/__init__.py,sha256=M5bTmg80E4QyN91gUX3qfw_nbtJSXwGWcKxdZP3vJz0,34
 flowcept/instrumentation/flowcept_agent_task.py,sha256=XN9JU4LODca0SgojUm4F5iU_V8tuWkOt1fAKcoOAG34,10757
 flowcept/instrumentation/flowcept_decorator.py,sha256=X4Lp_FSsoL08K8ZhRM4mC0OjKupbQtbMQR8zxy3ezDY,1350
-flowcept/instrumentation/flowcept_loop.py,sha256=jea_hYPuXg5_nOWf-nNb4vx8A__OBM4m96_92-J51o4,15670
+flowcept/instrumentation/flowcept_loop.py,sha256=nF7Sov-DCDapyYvS8zx-1ZFrnjc3CPg2VsjDaxFs0Cc,15667
 flowcept/instrumentation/flowcept_task.py,sha256=EmKODpjl8usNklKSVmsKYyCa6gC_QMqKhAr3DKaw44s,8199
 flowcept/instrumentation/flowcept_torch.py,sha256=kkZQRYq6cDBpdBU6J39_4oKRVkhyF3ODlz8ydV5WGKw,23455
 flowcept/instrumentation/task_capture.py,sha256=1g9EtLdqsTB0RHsF-eRmA2Xh9l_YqTd953d4v89IC24,8287
-resources/sample_settings.yaml,sha256=2IxbkbpFEymoUk-qcR70wsdXX2Y2cVP31cVWDVy0BNg,6779
-flowcept-0.9.9.dist-info/METADATA,sha256=NoHey7ehIO4kN0PrOCbz4z-TQ1Susn_yFOTsbd464uk,32439
-flowcept-0.9.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-flowcept-0.9.9.dist-info/entry_points.txt,sha256=i8q67WE0201rVxYI2lyBtS52shvgl93x2Szp4q8zMlw,47
-flowcept-0.9.9.dist-info/licenses/LICENSE,sha256=r5-2P6tFTuRGWT5TiX32s1y0tnp4cIqBEC1QjTaXe2k,1086
-flowcept-0.9.9.dist-info/RECORD,,
+resources/sample_settings.yaml,sha256=5insrkZdP-DHoB6rKxO9QkSqZUBVMaVlkTKWh_8TjIg,6881
+flowcept-0.9.11.dist-info/METADATA,sha256=CIU3PnotF8ynCtOLhTYofW7CYWz5aGZhOhOP1h-hb8c,32486
+flowcept-0.9.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+flowcept-0.9.11.dist-info/entry_points.txt,sha256=i8q67WE0201rVxYI2lyBtS52shvgl93x2Szp4q8zMlw,47
+flowcept-0.9.11.dist-info/licenses/LICENSE,sha256=r5-2P6tFTuRGWT5TiX32s1y0tnp4cIqBEC1QjTaXe2k,1086
+flowcept-0.9.11.dist-info/RECORD,,

resources/sample_settings.yaml

@@ -1,13 +1,15 @@
-flowcept_version: 0.9.9 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.11 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
   json_serializer: default # JSON serialization mode: default or complex. If "complex", Flowcept will deal with complex python dicts that may contain JSON unserializable values
   replace_non_json_serializable: true # Replace values that can't be JSON serialized
   performance_logging: false # Enable performance logging if true. Particularly useful for MQ flushes.
-  enrich_messages: true # Add extra metadata to task messages, such as IP addresses and UTC timestamps.
+  enrich_messages: true # Add extra metadata to task messages, such as IP addresses of the node that executed the task, UTC timestamps, GitHub repo metadata.
   db_flush_mode: online # Mode for flushing DB entries: "online" or "offline". If online, flushes to the DB will happen before the workflow ends.
-  # dump_buffer_path: flowcept_messages.jsonl # This is useful if you need to run completely offline. If you omit this, even offline, buffer data will not be persisted.
+  dump_buffer: # This is particularly useful if you need to run completely offline. If you omit this, even offline, buffer data will not be persisted.
+    enabled: false
+    path: flowcept_buffer.jsonl
 
 log:
   log_path: "default" # Path for log file output; "default" will write the log in the directory where the main executable is running from.
@@ -106,6 +108,7 @@ databases:
     db: flowcept
     create_collection_index: true  # Whether flowcept should create collection indices if they haven't been created yet. This is done only at the Flowcept start up.
 #    bin: /usr/bin/mongod
+#    db_path:
 #    log_path: /var/log/mongodb/mongod.log
 #    lock_file_path: /var/run/mongod.pid