flowcept 0.9.1__py3-none-any.whl → 0.9.2__py3-none-any.whl

flowcept/__init__.py

@@ -16,6 +16,11 @@ def __getattr__(name):
 
         return WorkflowObject
 
+    elif name == "TaskObject":
+        from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
+
+        return TaskObject
+
     elif name == "flowcept_task":
         from flowcept.instrumentation.flowcept_task import flowcept_task
 

flowcept/agents/prompts/general_prompts.py

@@ -24,8 +24,8 @@ ROUTING_PROMPT = (
     "Given the following user message, classify it into one of the following routes:\n"
     "- small_talk: if it's casual conversation or some random word (e.g., 'hausdn', 'a', hello, how are you, what can you do, what's your name)\n"
     "- plot: if user is requesting plots (e.g., plot, chart, visualize)\n"
+    "- in_context_query: if the user asks questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
     "- historical_prov_query: if the user wants to query historical provenance data\n"
-    "- in_context_query: if the user appears to ask questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
     "- in_chat_query: if the user appears to be asking about something that has said recently in this chat.\n"
     "- unknown: if you don't know.\n"
     "Respond with only the route label."

flowcept/cli.py

@@ -101,17 +101,17 @@ def version():
     print(f"Flowcept {__version__}")
 
 
-def stream_messages(print_messages: bool = False, messages_file_path: Optional[str] = None):
+def stream_messages(messages_file_path: Optional[str] = None, keys_to_show: List[str] = None):
     """
     Listen to Flowcept's message stream and optionally echo/save messages.
 
     Parameters.
     ----------
-    print_messages : bool, optional
-        If True, print each decoded message to stdout.
     messages_file_path : str, optional
         If provided, append each message as JSON (one per line) to this file.
         If the file already exists, a new timestamped file is created instead.
+    keys_to_show : List[str], optional
+        List of object keys to show in the prints. Use comma-separated list: --keys-to-show 'activity_id','workflow_id'
     """
     # Local imports to avoid changing module-level deps
     from flowcept.configs import MQ_TYPE
@@ -123,10 +123,7 @@ def stream_messages(print_messages: bool = False, messages_file_path: Optional[s
     import os
     import json
     from datetime import datetime
-    import redis
-    import msgpack
-    from flowcept.configs import MQ_HOST, MQ_PORT, MQ_CHANNEL, KVDB_URI
-    from flowcept.commons.daos.mq_dao.mq_dao_redis import MQDaoRedis
+    from flowcept.flowceptor.consumers.base_consumer import BaseConsumer
 
     def _timestamped_path_if_exists(path: Optional[str]) -> Optional[str]:
         if not path:
@@ -150,53 +147,53 @@ def stream_messages(print_messages: bool = False, messages_file_path: Optional[s
 
         return json.dumps(obj, ensure_ascii=False, separators=(",", ":"), default=_default)
 
-    # Prepare output file (JSONL)
     out_fh = None
     if messages_file_path:
         out_path = _timestamped_path_if_exists(messages_file_path)
         out_fh = open(out_path, "w", encoding="utf-8", buffering=1)  # line-buffered
 
-    # Connect & subscribe
-    redis_client = redis.from_url(KVDB_URI) if KVDB_URI else redis.Redis(host=MQ_HOST, port=MQ_PORT, db=0)
-    pubsub = redis_client.pubsub()
-    pubsub.subscribe(MQ_CHANNEL)
-
-    print(f"Listening for messages on channel '{MQ_CHANNEL}'... (Ctrl+C to exit)")
-
-    try:
-        for message in pubsub.listen():
-            if not message or message.get("type") in MQDaoRedis.MESSAGE_TYPES_IGNORE:
-                continue
-
-            data = message.get("data")
-            if not isinstance(data, (bytes, bytearray)):
-                print(f"Skipping message with unexpected data type: {type(data)} - {data}")
-                continue
+    class MyConsumer(BaseConsumer):
+        def __init__(self):
+            super().__init__()
 
+        def message_handler(self, msg_obj: Dict) -> bool:
             try:
-                msg_obj = msgpack.loads(data, strict_map_key=False)
-                msg_type = msg_obj.get("type", None)
-                print(f"\nReceived a message! type={msg_type}")
-
-                if print_messages:
-                    print(_json_dumps(msg_obj))
+                if keys_to_show is not None:
+                    obj_to_print = {}
+                    for k in keys_to_show:
+                        v = msg_obj.get(k, None)
+                        if v is not None:
+                            obj_to_print[k] = v
+                    if not obj_to_print:
+                        obj_to_print = msg_obj
+                else:
+                    obj_to_print = msg_obj
+
+                print(_json_dumps(obj_to_print))
 
                 if out_fh is not None:
-                    out_fh.write(_json_dumps(msg_obj))
+                    out_fh.write(_json_dumps(obj_to_print))
                     out_fh.write("\n")
-
+            except KeyboardInterrupt:
+                print("\nGracefully interrupted, shutting down...")
+                return False
             except Exception as e:
-                print(f"Error decoding message: {e}")
+                print(e)
+                return False
+            finally:
+                try:
+                    if out_fh:
+                        out_fh.close()
+                except Exception as e:
+                    print(e)
+                    return False
 
-    except KeyboardInterrupt:
-        print("\nInterrupted, shutting down...")
-    finally:
-        try:
-            if out_fh:
-                out_fh.close()
-            pubsub.close()
-        except Exception:
-            pass
+            return True
+
+    m = f"Printing only the keys {keys_to_show}" if keys_to_show is not None else ""
+    print(f"Listening for messages.{m} Ctrl+C to exit")
+    consumer = MyConsumer()
+    consumer.start(daemon=False)
 
 
 def start_consumption_services(bundle_exec_id: str = None, check_safe_stops: bool = False, consumers: List[str] = None):
@@ -700,3 +697,5 @@ def main():  # noqa: D103
 if __name__ == "__main__":
     main()
     # check_services()
+
+__doc__ = None

flowcept/commons/autoflush_buffer.py

@@ -47,6 +47,11 @@ class AutoflushBuffer:
         if len(buffer) >= self._max_size:
             self._swap_event.set()
 
+    @property
+    def current_buffer(self):
+        """Return the currently active buffer (read-only)."""
+        return self._buffers[self._current_buffer_index]
+
     def time_based_flush(self):
         """Time flush."""
         while not self._stop_event.is_set():

flowcept/commons/daos/docdb_dao/lmdb_dao.py

@@ -56,7 +56,10 @@ class LMDBDAO(DocumentDBDAO):
             t0 = 0
             if PERF_LOG:
                 t0 = time()
-            indexed_buffer = curate_dict_task_messages(docs, indexing_key, t0, convert_times=False)
+            indexed_buffer = curate_dict_task_messages(
+                docs, indexing_key, t0, convert_times=False, keys_to_drop=["data"]
+            )
+
             with self._env.begin(write=True, db=self._tasks_db) as txn:
                 for key, value in indexed_buffer.items():
                     k, v = key.encode(), json.dumps(value).encode()

flowcept/commons/daos/mq_dao/mq_dao_base.py

@@ -102,6 +102,7 @@ class MQDao(object):
 
                 with open(DUMP_BUFFER_PATH, "wb", buffering=1_048_576) as f:
                     for obj in buffer:
+                        obj.pop("data", None)  # We are not going to store data in the buffer file.
                         f.write(orjson.dumps(obj))
                         f.write(b"\n")
                 self.logger.info(f"Saved Flowcept messages into {DUMP_BUFFER_PATH}.")

flowcept/commons/flowcept_dataclasses/task_object.py

@@ -16,45 +16,125 @@ from flowcept.configs import (
 
 
 class TaskObject:
-    """Task class."""
+    """Task object class.
+
+    Represents a single provenance task in Flowcept, including inputs, outputs,
+    execution metadata, telemetry, and environment details.
+    """
 
     type = "task"
+    """Constant type label for this object ("task")."""
+
     subtype: AnyStr = None
-    task_id: AnyStr = None  # Any way to identify a task
+    """Optional subtype of the task (e.g., iteration, ML step, custom)."""
+
+    task_id: AnyStr = None
+    """Unique identifier of the task."""
+
     utc_timestamp: float = None
+    """UTC timestamp when the task object was created."""
+
     adapter_id: AnyStr = None
+    """Identifier of the adapter that produced this task (if any)."""
+
     user: AnyStr = None
+    """User who executed or triggered the task."""
+
     data: Any = None
-    used: Dict[AnyStr, Any] = None  # Used parameter and files
+    """Arbitrary raw data payload associated with the task. It is good practice to add custom_metadata associated with
+    `data`, especially if it contains file contents. 
+    In that case, `custom_metadata` should contain the keys "file_type", "file_content", "file_name", "extension".
+    """
+
+    used: Dict[AnyStr, Any] = None
+    """Inputs consumed by the task (parameters, files, resources)."""
+
     campaign_id: AnyStr = None
-    generated: Dict[AnyStr, Any] = None  # Generated results and files
+    """Campaign identifier grouping related tasks together."""
+
+    generated: Dict[AnyStr, Any] = None
+    """Outputs produced by the task (results, artifacts, files)."""
+
     submitted_at: float = None
+    """Timestamp when the task was submitted."""
+
     started_at: float = None
+    """Timestamp when the task execution started."""
+
     ended_at: float = None
-    registered_at: float = None  # Leave this for dates generated at the DocInserter
+    """Timestamp when the task execution ended."""
+
+    registered_at: float = None
+    """Timestamp when the task was registered by the DocInserter."""
+
     telemetry_at_start: Telemetry = None
+    """Telemetry snapshot captured at the start of the task."""
+
     telemetry_at_end: Telemetry = None
+    """Telemetry snapshot captured at the end of the task."""
+
     workflow_name: AnyStr = None
+    """Name of the workflow this task belongs to."""
+
     workflow_id: AnyStr = None
+    """Identifier of the workflow this task belongs to."""
+
     parent_task_id: AnyStr = None
+    """Identifier of the parent task, if this task is nested or dependent."""
+
     activity_id: AnyStr = None
-    group_id: AnyStr = None  # Utilized especially for loop iteration tasks, to group them.
+    """Activity name (usually the function name) associated with the task."""
+
+    group_id: AnyStr = None
+    """Grouping identifier, often used to link loop iterations together."""
+
     status: Status = None
+    """Execution status of the task (e.g., FINISHED, ERROR)."""
+
     stdout: Union[AnyStr, Dict] = None
+    """Captured standard output from the task, if available."""
+
     stderr: Union[AnyStr, Dict] = None
+    """Captured standard error from the task, if available."""
+
     custom_metadata: Dict[AnyStr, Any] = None
+    """Custom metadata dictionary provided by the developer/user."""
+
     mq_host: str = None
+    """Message queue host associated with the task."""
+
     environment_id: AnyStr = None
+    """Identifier of the environment where the task executed."""
+
     node_name: AnyStr = None
+    """Node name in a distributed system or HPC cluster."""
+
     login_name: AnyStr = None
+    """Login name of the user in the execution environment."""
+
     public_ip: AnyStr = None
+    """Public IP address of the machine executing the task."""
+
     private_ip: AnyStr = None
+    """Private IP address of the machine executing the task."""
+
     hostname: AnyStr = None
+    """Hostname of the machine executing the task."""
+
     address: AnyStr = None
+    """Optional network address associated with the task."""
+
     dependencies: List = None
+    """List of task IDs this task depends on."""
+
     dependents: List = None
+    """List of task IDs that depend on this task."""
+
     tags: List = None
+    """User-defined tags attached to the task."""
+
     agent_id: str = None
+    """Identifier of the agent responsible for executing this task (if any)."""
 
     _DEFAULT_ENRICH_VALUES = {
         "node_name": NODE_NAME,

flowcept/commons/flowcept_dataclasses/workflow_object.py

@@ -19,27 +19,67 @@ from flowcept.configs import (
 # Not a dataclass because a dataclass stores keys even when there's no value,
 # adding unnecessary overhead.
 class WorkflowObject:
-    """Workflow class."""
+    """Workflow object class.
+
+    Represents metadata and provenance details for a workflow execution.
+    """
 
     workflow_id: AnyStr = None
+    """Unique identifier for the workflow."""
+
     parent_workflow_id: AnyStr = None
+    """Identifier of the parent workflow, if this workflow is nested or derived."""
+
     machine_info: Dict = None
+    """System or hardware information where the workflow is executed."""
+
     conf: Dict = None
+    """Workflow configuration parameters, such as hyperparameters or runtime options."""
+
     flowcept_settings: Dict = None
+    """Snapshot of Flowcept’s active settings used for this workflow."""
+
     flowcept_version: AnyStr = None
+    """Version of Flowcept used during execution."""
+
     utc_timestamp: float = None
+    """Timestamp (UTC, in seconds) when the workflow object was created."""
+
     user: AnyStr = None
+    """User who launched or owns the workflow run."""
+
     campaign_id: AnyStr = None
+    """Identifier for grouping workflows into a campaign or experiment."""
+
     adapter_id: AnyStr = None
+    """Identifier of the adapter (e.g., Dask, MLflow) that triggered workflow capture."""
+
     interceptor_ids: List[AnyStr] = None
+    """List of interceptors applied to this workflow (e.g., instrumentation, telemetry)."""
+
     name: AnyStr = None
+    """Descriptive name for the workflow."""
+
     custom_metadata: Dict = None
+    """User-defined metadata dictionary with additional annotations."""
+
     environment_id: str = None
+    """Identifier for the runtime environment (e.g., conda env, container)."""
+
     sys_name: str = None
+    """Logical system or facility name (e.g., HPC system name, cluster identifier)."""
+
     extra_metadata: str = None
+    """Optional free-form metadata for extensions not covered by other fields."""
+
     used: Dict = None
+    """Inputs consumed by the workflow (datasets, arguments, or configuration values)."""
+
     code_repository: Dict = None
+    """Details of the code repository (URL, commit hash, branch) used to run the workflow."""
+
     generated: Dict = None
+    """Outputs generated by the workflow (artifacts, models, or results)."""
 
     def __init__(self, workflow_id=None, name=None, used=None, generated=None):
         self.workflow_id = workflow_id

flowcept/configs.py

@@ -158,7 +158,7 @@ PERF_LOG = settings["project"].get("performance_logging", False)
 JSON_SERIALIZER = settings["project"].get("json_serializer", "default")
 REPLACE_NON_JSON_SERIALIZABLE = settings["project"].get("replace_non_json_serializable", True)
 ENRICH_MESSAGES = settings["project"].get("enrich_messages", True)
-DUMP_BUFFER_PATH = settings["project"].get("dump_buffer_path", None)
+DUMP_BUFFER_PATH = settings["project"].get("dump_buffer_path", "flowcept_messages.jsonl")
 
 TELEMETRY_CAPTURE = settings.get("telemetry_capture", None)
 TELEMETRY_ENABLED = TELEMETRY_CAPTURE is not None and len(TELEMETRY_CAPTURE)

flowcept/flowcept_api/flowcept_controller.py

@@ -4,6 +4,7 @@ import os.path
 from typing import List, Dict
 from uuid import uuid4
 
+from flowcept.commons.autoflush_buffer import AutoflushBuffer
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
 from flowcept.commons.flowcept_dataclasses.workflow_object import (
     WorkflowObject,
@@ -30,6 +31,7 @@ class Flowcept(object):
     # TODO: rename current_workflow_id to workflow_id. This will be a major refactor
     current_workflow_id = None
     campaign_id = None
+    buffer = None
 
     @ClassProperty
     def db(cls):
@@ -155,7 +157,11 @@ class Flowcept(object):
                 interceptor_inst = BaseInterceptor.build(interceptor)
                 interceptor_inst.start(bundle_exec_id=self._bundle_exec_id, check_safe_stops=self._check_safe_stops)
                 self._interceptor_instances.append(interceptor_inst)
-                self.buffer = interceptor_inst._mq_dao.buffer
+                if isinstance(interceptor_inst._mq_dao.buffer, AutoflushBuffer):
+                    Flowcept.buffer = self.buffer = interceptor_inst._mq_dao.buffer.current_buffer
+                else:
+                    Flowcept.buffer = self.buffer = interceptor_inst._mq_dao.buffer
+
                 if self._should_save_workflow and not self._workflow_saved:
                     self.save_workflow(interceptor, interceptor_inst)
 
@@ -296,6 +302,8 @@ class Flowcept(object):
             self.logger.info("Stopping DB Inserters...")
             for db_inserter in self._db_inserters:
                 db_inserter.stop(bundle_exec_id=self._bundle_exec_id)
+
+        Flowcept.buffer = self.buffer = None
         self.is_started = False
         self.logger.debug("All stopped!")
 

flowcept/flowceptor/consumers/base_consumer.py

@@ -4,6 +4,7 @@ from typing import Callable, Dict, Tuple, Optional
 
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
 from flowcept.commons.flowcept_logger import FlowceptLogger
+from flowcept.configs import MQ_ENABLED
 
 
 class BaseConsumer(object):
@@ -16,7 +17,11 @@ class BaseConsumer(object):
 
     def __init__(self):
         """Initialize the message queue DAO and logger."""
+        if not MQ_ENABLED:
+            raise Exception("MQ is disabled in the settings. You cannot consume messages.")
+
         self._mq_dao = MQDao.build()
+
         self.logger = FlowceptLogger()
         self._main_thread: Optional[Thread] = None
 

flowcept/flowceptor/consumers/consumer_utils.py

@@ -11,7 +11,7 @@ from flowcept.commons.vocabulary import Status
 UTC_TZ = ZoneInfo("UTC")
 
 
-def curate_task_msg(task_msg_dict: dict, convert_times=True):
+def curate_task_msg(task_msg_dict: dict, convert_times=True, keys_to_drop: List = None):
     """Curate a task message."""
     # Converting any arg to kwarg in the form {"arg1": val1, "arg2: val2}
     for field in TaskObject.get_dict_field_names():
@@ -45,6 +45,10 @@ def curate_task_msg(task_msg_dict: dict, convert_times=True):
     if "used" in task_msg_dict and task_msg_dict["used"].get("workflow_id", None):
         task_msg_dict["workflow_id"] = task_msg_dict["used"].pop("workflow_id")
 
+    if keys_to_drop is not None:
+        for k in keys_to_drop:
+            task_msg_dict.pop(k, None)
+
     if convert_times:
         for time_field in TaskObject.get_time_field_names():
             if time_field in task_msg_dict:
@@ -97,7 +101,11 @@ def convert_keys_to_strings(obj):
 
 
 def curate_dict_task_messages(
-    doc_list: List[Dict], indexing_key: str, utc_time_at_insertion: float = 0, convert_times=True
+    doc_list: List[Dict],
+    indexing_key: str,
+    utc_time_at_insertion: float = 0,
+    convert_times=True,
+    keys_to_drop: List = None,
 ):
     """Remove duplicates.
 
@@ -134,7 +142,7 @@ def curate_dict_task_messages(
         if utc_time_at_insertion > 0:
             doc["utc_time_at_insertion"] = utc_time_at_insertion
 
-        curate_task_msg(doc, convert_times)
+        curate_task_msg(doc, convert_times, keys_to_drop)
         indexing_key_value = doc[indexing_key]
 
         if indexing_key_value not in indexed_buffer:

flowcept/flowceptor/telemetry_capture.py

@@ -258,7 +258,8 @@ class TelemetryCapture:
             disk.disk_usage = psutil.disk_usage("/")._asdict()
 
             platform_info = platform.uname()._asdict()
-            network_info = psutil.net_if_addrs()
+            network_info_raw = psutil.net_if_addrs()
+            network_info = {ifname: [addr._asdict() for addr in addrs] for ifname, addrs in network_info_raw.items()}
             processor_info = cpuinfo.get_cpu_info()
 
             gpu_info = None

flowcept/instrumentation/task_capture.py

@@ -22,21 +22,6 @@ class FlowceptTask(object):
     and metadata. It integrates with the Flowcept API and Instrumentation Interceptor to
     log task-specific details.
 
-    Parameters
-    ----------
-    task_id : str, optional
-        Unique identifier for the task. If not provided, it defaults to the current timestamp.
-    workflow_id : str, optional
-        ID of the workflow to which this task belongs. Defaults to the current workflow ID from
-        Flowcept.
-    campaign_id : str, optional
-        ID of the campaign to which this task belongs. Defaults to the current campaign ID from
-        Flowcept.
-    used : Dict, optional
-        Metadata about the resources or data used during the task execution.
-    custom_metadata : Dict, optional
-        User-defined metadata associated with the task.
-
     Methods
     -------
     __enter__()
@@ -65,6 +50,7 @@ class FlowceptTask(object):
         subtype: str = None,
         custom_metadata: Dict = None,
         generated: Dict = None,
+        started_at: float = None,
         ended_at: float = None,
         stdout: str = None,
         stderr: str = None,
@@ -90,12 +76,16 @@ class FlowceptTask(object):
             Describes the specific activity this task captures.
         used : Dict, optional
             Metadata about resources or data used during the task.
+        data: Any, optional
+            Any raw data associated to this task.
         subtype : str, optional
             Optional string categorizing the task subtype.
         custom_metadata : Dict, optional
             Additional user-defined metadata to associate with the task.
         generated : Dict, optional
             Output data generated during the task execution.
+        started_at : float, optional
+            Timestamp indicating when the task started.
         ended_at : float, optional
             Timestamp indicating when the task ended.
         stdout : str, optional
@@ -117,7 +107,7 @@ class FlowceptTask(object):
             self._task.telemetry_at_start = tel
 
         self._task.activity_id = activity_id
-        self._task.started_at = time()
+        self._task.started_at = started_at or time()
         self._task.task_id = task_id or self._gen_task_id()
         self._task.workflow_id = workflow_id or Flowcept.current_workflow_id
         self._task.campaign_id = campaign_id or Flowcept.campaign_id
@@ -159,6 +149,8 @@ class FlowceptTask(object):
         ended_at: float = None,
         stdout: str = None,
         stderr: str = None,
+        data: Any = None,
+        custom_metadata: Dict = None,
         status: Status = Status.FINISHED,
     ):
         """
@@ -172,6 +164,10 @@ class FlowceptTask(object):
         ----------
         generated : Dict, optional
             Metadata or data generated during the task's execution. Defaults to None.
+        data: Any, optional
+            Any raw data associated to this task.
+        custom_metadata : Dict, optional
+            Additional user-defined metadata to associate with the task.
         ended_at : float, optional
             Timestamp indicating when the task ended. If not provided, defaults to the current time.
         stdout : str, optional
@@ -191,6 +187,10 @@ class FlowceptTask(object):
         if TELEMETRY_ENABLED:
             tel = self._interceptor.telemetry_capture.capture()
             self._task.telemetry_at_end = tel
+        if data:
+            self._task.data = data
+        if custom_metadata:
+            self._task.custom_metadata = custom_metadata
         self._task.ended_at = ended_at or time()
         self._task.status = status
         self._task.stderr = stderr

flowcept/version.py

@@ -4,4 +4,4 @@
 # The expected format is: <Major>.<Minor>.<Patch>
 # This file is supposed to be automatically modified by the CI Bot.
 # See .github/workflows/version_bumper.py
-__version__ = "0.9.1"
+__version__ = "0.9.2"