flowcept 0.8.11__py3-none-any.whl → 0.9.1__py3-none-any.whl

flowcept/flowcept_api/flowcept_controller.py

@@ -1,6 +1,7 @@
 """Controller module."""
 
-from typing import List
+import os.path
+from typing import List, Dict
 from uuid import uuid4
 
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
@@ -16,6 +17,8 @@ from flowcept.configs import (
     SETTINGS_PATH,
     LMDB_ENABLED,
     KVDB_ENABLED,
+    MQ_ENABLED,
+    DUMP_BUFFER_PATH,
 )
 from flowcept.flowceptor.adapters.base_interceptor import BaseInterceptor
 
@@ -44,7 +47,7 @@ class Flowcept(object):
         campaign_id: str = None,
         workflow_id: str = None,
         workflow_name: str = None,
-        workflow_args: str = None,
+        workflow_args: Dict = None,
         start_persistence=True,
         check_safe_stops=True,  # TODO add to docstring
         save_workflow=True,
@@ -94,6 +97,7 @@ class Flowcept(object):
         self.logger.debug(f"Using settings file: {SETTINGS_PATH}")
         self._enable_persistence = start_persistence
         self._db_inserters: List = []
+        self.buffer = None
         self._check_safe_stops = check_safe_stops
         if bundle_exec_id is None:
             self._bundle_exec_id = id(self)
@@ -151,7 +155,7 @@ class Flowcept(object):
                 interceptor_inst = BaseInterceptor.build(interceptor)
                 interceptor_inst.start(bundle_exec_id=self._bundle_exec_id, check_safe_stops=self._check_safe_stops)
                 self._interceptor_instances.append(interceptor_inst)
-
+                self.buffer = interceptor_inst._mq_dao.buffer
                 if self._should_save_workflow and not self._workflow_saved:
                     self.save_workflow(interceptor, interceptor_inst)
 
@@ -161,6 +165,68 @@ class Flowcept(object):
         self.logger.debug("Flowcept started successfully.")
         return self
 
+    def _publish_buffer(self):
+        self._interceptor_instances[0]._mq_dao.bulk_publish(self.buffer)
+
+    @staticmethod
+    def read_messages_file(file_path: str = None) -> List[Dict]:
+        """
+        Read a JSON Lines (JSONL) file containing captured Flowcept messages.
+
+        This function loads a file where each line is a serialized JSON object.
+        It joins the lines into a single JSON array and parses them efficiently
+        with ``orjson``.
+
+        Parameters
+        ----------
+        file_path : str, optional
+            Path to the messages file. If not provided, defaults to the
+            value of ``DUMP_BUFFER_PATH`` from the configuration.
+            If neither is provided, an assertion error is raised.
+
+        Returns
+        -------
+        List[dict]
+            A list of message objects (dictionaries) parsed from the file.
+
+        Raises
+        ------
+        AssertionError
+            If no ``file_path`` is provided and ``DUMP_BUFFER_PATH`` is not set.
+        FileNotFoundError
+            If the specified file does not exist.
+        orjson.JSONDecodeError
+            If the file contents cannot be parsed as valid JSON.
+
+        Examples
+        --------
+        Read messages from a file explicitly:
+
+        >>> msgs = read_messages_file("offline_buffer.jsonl")
+        >>> print(len(msgs))
+        128
+
+        Use the default dump buffer path from config:
+
+        >>> msgs = read_messages_file()
+        >>> for m in msgs[:2]:
+        ...     print(m["type"], m.get("workflow_id"))
+        task_start wf_123
+        task_end wf_123
+        """
+        import orjson
+
+        _buffer = []
+        if file_path is None:
+            file_path = DUMP_BUFFER_PATH
+        assert file_path is not None, "Please indicate file_path either in the argument or in the config file."
+        if not os.path.exists(file_path):
+            raise f"File {file_path} has not been created. It will only be created if you run in fully offline mode."
+        with open(file_path, "rb") as f:
+            lines = [ln for ln in f.read().splitlines() if ln]
+        _buffer = orjson.loads(b"[" + b",".join(lines) + b"]")
+        return _buffer
+
     def save_workflow(self, interceptor: str, interceptor_instance: BaseInterceptor):
         """
         Save the current workflow and send its metadata using the provided interceptor.
@@ -270,9 +336,10 @@ class Flowcept(object):
         """
         logger = FlowceptLogger()
         mq = MQDao.build()
-        if not mq.liveness_test():
-            logger.error("MQ Not Ready!")
-            return False
+        if MQ_ENABLED:
+            if not mq.liveness_test():
+                logger.error("MQ Not Ready!")
+                return False
 
         if KVDB_ENABLED:
             if not mq._keyvalue_dao.liveness_test():

flowcept/flowceptor/adapters/base_interceptor.py

@@ -9,14 +9,13 @@ from flowcept.commons.flowcept_dataclasses.workflow_object import (
 )
 from flowcept.configs import (
     ENRICH_MESSAGES,
+    TELEMETRY_ENABLED,
 )
 from flowcept.commons.flowcept_logger import FlowceptLogger
 from flowcept.commons.daos.mq_dao.mq_dao_base import MQDao
 from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
 from flowcept.commons.settings_factory import get_settings
 
-from flowcept.flowceptor.telemetry_capture import TelemetryCapture
-
 
 # TODO :base-interceptor-refactor: :ml-refactor: :code-reorg: :usability:
 #  Consider creating a new concept for instrumentation-based 'interception'.
@@ -74,7 +73,14 @@ class BaseInterceptor(object):
         self._bundle_exec_id = None
         self.started = False
         self._interceptor_instance_id = str(id(self))
-        self.telemetry_capture = TelemetryCapture()
+
+        if TELEMETRY_ENABLED:
+            from flowcept.flowceptor.telemetry_capture import TelemetryCapture
+
+            self.telemetry_capture = TelemetryCapture()
+        else:
+            self.telemetry_capture = None
+
         self._saved_workflows = set()
         self._generated_workflow_id = False
         self.kind = kind
@@ -129,8 +135,8 @@ class BaseInterceptor(object):
             # TODO :base-interceptor-refactor: :code-reorg: :usability:
             raise Exception(f"This interceptor {id(self)} has never been started!")
         workflow_obj.interceptor_ids = [self._interceptor_instance_id]
-        machine_info = self.telemetry_capture.capture_machine_info()
-        if machine_info is not None:
+        if self.telemetry_capture:
+            machine_info = self.telemetry_capture.capture_machine_info()
             if workflow_obj.machine_info is None:
                 workflow_obj.machine_info = dict()
             # TODO :refactor-base-interceptor: we might want to register

flowcept/flowceptor/consumers/agent/base_agent_context_manager.py

@@ -1,7 +1,9 @@
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from typing import Dict, List
+from uuid import uuid4
 
+from flowcept.flowcept_api.flowcept_controller import Flowcept
 from flowcept.flowceptor.consumers.base_consumer import BaseConsumer
 
 
@@ -35,13 +37,17 @@ class BaseAgentContextManager(BaseConsumer):
     - Access shared state via `self.context` during execution
     """
 
+    agent_id = None
+
     def __init__(self):
         """
         Initializes the agent and resets its context state.
         """
+        self._started = False
         super().__init__()
         self.context = None
         self.reset_context()
+        self.agent_id = BaseAgentContextManager.agent_id
 
     def message_handler(self, msg_obj: Dict) -> bool:
         """
@@ -94,7 +100,25 @@ class BaseAgentContextManager(BaseConsumer):
         BaseAppContext
             The current application context, including collected tasks.
         """
-        self.start()
+        if not self._started:
+            self.agent_id = BaseAgentContextManager.agent_id = str(uuid4())
+            self.logger.info(f"Starting lifespan for agent {BaseAgentContextManager.agent_id}.")
+            self._started = True
+
+            f = Flowcept(
+                start_persistence=False,
+                save_workflow=True,
+                check_safe_stops=False,
+                workflow_name="agent_workflow",
+                workflow_args={"agent_id": self.agent_id},
+            )
+            self.agent_workflow_id = f.current_workflow_id
+            f.start()
+            f.logger.info(
+                f"This section's workflow_id={Flowcept.current_workflow_id}, campaign_id={Flowcept.campaign_id}"
+            )
+            self.start()
+
         try:
             yield self.context
         finally:

flowcept/flowceptor/consumers/base_consumer.py

@@ -82,6 +82,10 @@ class BaseConsumer(object):
         """
         self.logger.debug("Going to wait for new messages!")
         self._mq_dao.message_listener(self.message_handler)
+        self.logger.debug("Broke main message listening loop!")
+        # self._mq_dao.stop(check_safe_stops=False) # TODO Do we need to stop mq_dao here?
+        self.stop_consumption()
+        self.logger.debug("MQ stopped.")
 
     def stop_consumption(self):
         """

flowcept/flowceptor/consumers/consumer_utils.py

@@ -1,14 +1,15 @@
 """Consumer utilities module."""
 
 from datetime import datetime
+from zoneinfo import ZoneInfo
 from time import time
 from typing import List, Dict
 
-import pytz
-
 from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
 from flowcept.commons.vocabulary import Status
 
+UTC_TZ = ZoneInfo("UTC")
+
 
 def curate_task_msg(task_msg_dict: dict, convert_times=True):
     """Curate a task message."""
@@ -47,10 +48,10 @@ def curate_task_msg(task_msg_dict: dict, convert_times=True):
     if convert_times:
         for time_field in TaskObject.get_time_field_names():
             if time_field in task_msg_dict:
-                task_msg_dict[time_field] = datetime.fromtimestamp(task_msg_dict[time_field], pytz.utc)
+                task_msg_dict[time_field] = datetime.fromtimestamp(task_msg_dict[time_field], UTC_TZ)
 
         if "registered_at" not in task_msg_dict:
-            task_msg_dict["registered_at"] = datetime.fromtimestamp(time(), pytz.utc)
+            task_msg_dict["registered_at"] = datetime.fromtimestamp(time(), UTC_TZ)
 
 
 def remove_empty_fields_from_dict(obj: dict):

flowcept/flowceptor/consumers/document_inserter.py

@@ -150,9 +150,9 @@ class DocumentInserter(BaseConsumer):
                 and message["telemetry_at_end"]
             ):
                 try:
-                    telemetry_summary = summarize_telemetry(message)
+                    telemetry_summary = summarize_telemetry(message, self.logger)
                     message["telemetry_summary"] = telemetry_summary
-                    # TODO: make this dynamic
+                    # TODO: make this configurable
                     tags = tag_critical_task(
                         generated=message.get("generated", {}), telemetry_summary=telemetry_summary, thresholds=None
                     )

flowcept/flowceptor/telemetry_capture.py

@@ -2,9 +2,8 @@
 
 from typing import Callable, Set, List
 
-import psutil
 import platform
-import cpuinfo
+
 import os
 
 from flowcept.commons.flowcept_logger import FlowceptLogger
@@ -15,6 +14,10 @@ from flowcept.configs import (
 )
 from flowcept.commons.flowcept_dataclasses.telemetry import Telemetry
 
+if TELEMETRY_CAPTURE is not None and len(TELEMETRY_CAPTURE):
+    import psutil
+    import cpuinfo
+
 
 class GPUCapture:
     """GPU Capture class."""

flowcept/instrumentation/flowcept_agent_task.py

@@ -0,0 +1,294 @@
+"""Flowcept Agent Task module."""
+
+import argparse
+import threading
+from functools import wraps
+from time import time
+from typing import Any, Dict, List, Optional, Union
+
+from langchain_core.language_models import LLM
+from langchain_core.language_models.base import BaseLanguageModel
+from langchain_core.messages import BaseMessage
+from langchain_core.runnables import Runnable
+
+from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
+from flowcept.commons.flowcept_logger import FlowceptLogger
+from flowcept.commons.utils import replace_non_serializable
+from flowcept.commons.vocabulary import Status
+from flowcept.configs import (
+    INSTRUMENTATION_ENABLED,
+    REPLACE_NON_JSON_SERIALIZABLE,
+    TELEMETRY_ENABLED,
+)
+from flowcept.flowcept_api.flowcept_controller import Flowcept
+from flowcept.flowceptor.adapters.instrumentation_interceptor import InstrumentationInterceptor
+from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
+from flowcept.instrumentation.task_capture import FlowceptTask
+
+
+_thread_local = threading.local()
+
+
+# TODO: :code-reorg: consider moving it to utils and reusing it in dask interceptor
+def default_args_handler(*args, **kwargs):
+    """Get default arguments."""
+    args_handled = {}
+    if args is not None and len(args):
+        if isinstance(args[0], argparse.Namespace):
+            args_handled.update(args[0].__dict__)
+            args = args[1:]
+        for i in range(len(args)):
+            args_handled[f"arg_{i}"] = args[i]
+    if kwargs is not None and len(kwargs):
+        args_handled.update(kwargs)
+    if REPLACE_NON_JSON_SERIALIZABLE:
+        args_handled = replace_non_serializable(args_handled)
+    return args_handled
+
+
+def agent_flowcept_task(func=None, **decorator_kwargs):
+    """Get flowcept task."""
+    if INSTRUMENTATION_ENABLED:
+        interceptor = InstrumentationInterceptor.get_instance()
+        logger = FlowceptLogger()
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if not INSTRUMENTATION_ENABLED:
+                return func(*args, **kwargs)
+
+            args_handler = decorator_kwargs.get("args_handler", default_args_handler)
+            custom_metadata = decorator_kwargs.get("custom_metadata", None)
+            tags = decorator_kwargs.get("tags", None)
+
+            task_obj = TaskObject()
+            task_obj.subtype = decorator_kwargs.get("subtype", "agent_task")
+            task_obj.activity_id = func.__name__
+            handled_args = args_handler(*args, **kwargs)
+            task_obj.workflow_id = handled_args.pop("workflow_id", Flowcept.current_workflow_id)
+            task_obj.campaign_id = handled_args.pop("campaign_id", Flowcept.campaign_id)
+            task_obj.used = handled_args
+            task_obj.tags = tags
+            task_obj.started_at = time()
+            task_obj.custom_metadata = custom_metadata or {}
+            task_obj.task_id = str(task_obj.started_at)
+            _thread_local._flowcept_current_context_task = task_obj
+            if TELEMETRY_ENABLED:
+                task_obj.telemetry_at_start = interceptor.telemetry_capture.capture()
+            task_obj.agent_id = BaseAgentContextManager.agent_id
+
+            try:
+                result = func(*args, **kwargs)
+                task_obj.status = Status.FINISHED
+            except Exception as e:
+                task_obj.status = Status.ERROR
+                result = None
+                logger.exception(e)
+                task_obj.stderr = str(e)
+            task_obj.ended_at = time()
+
+            if TELEMETRY_ENABLED:
+                task_obj.telemetry_at_end = interceptor.telemetry_capture.capture()
+            try:
+                if result is not None:
+                    if isinstance(result, dict):
+                        task_obj.generated = args_handler(**result)
+                    else:
+                        task_obj.generated = args_handler(result)
+            except Exception as e:
+                logger.exception(e)
+
+            interceptor.intercept(task_obj.to_dict())
+            return result
+
+        return wrapper
+
+    if func is None:
+        return decorator
+    else:
+        return decorator(func)
+
+
+def get_current_context_task() -> TaskObject | None:
+    """Retrieve the current task object from thread-local storage."""
+    return getattr(_thread_local, "_flowcept_current_context_task", None)
+
+
+def _extract_llm_metadata(llm: LLM) -> Dict:
+    """
+    Extract metadata from a LangChain LLM instance.
+
+    Parameters
+    ----------
+    llm : LLM
+        The language model instance.
+
+    Returns
+    -------
+    dict
+        Dictionary containing class name, module, model name, and configuration if available.
+    """
+    llm_metadata = {
+        "class_name": llm.__class__.__name__,
+        "module": llm.__class__.__module__,
+        "config": llm.dict() if hasattr(llm, "dict") else {},
+    }
+    return llm_metadata
+
+
+class FlowceptLLM(Runnable):
+    """
+    Flowcept wrapper for language models to capture provenance of LLM interactions.
+
+    This class wraps a LangChain-compatible LLM (any subclass of
+    ``langchain_core.language_models.base.BaseLanguageModel``) so that
+    prompts and responses are automatically captured as provenance tasks
+    in Flowcept. It ensures that both inputs (prompts) and outputs
+    (responses) are recorded, along with metadata about the underlying LLM.
+
+    Parameters
+    ----------
+    llm : BaseLanguageModel
+        The underlying LangChain-compatible LLM instance to wrap.
+    agent_id : str, optional
+        Identifier of the agent that owns this LLM. Used to correlate
+        tasks across agents.
+    parent_task_id : str, optional
+        Identifier of the parent task, if this LLM interaction is part
+        of a larger workflow task.
+    workflow_id : str, optional
+        Identifier of the workflow execution associated with this task.
+    campaign_id : str, optional
+        Identifier of the campaign or experiment associated with this task.
+
+    Attributes
+    ----------
+    llm : BaseLanguageModel
+        The underlying LLM object.
+    agent_id : str
+        The agent identifier, if provided.
+    parent_task_id : str
+        Parent task identifier, if provided.
+    worflow_id : str
+        Workflow identifier, if provided.
+    campaign_id : str
+        Campaign identifier, if provided.
+    metadata : dict
+        Extracted metadata about the underlying LLM, such as class name,
+        module, and configuration.
+
+    Methods
+    -------
+    call(messages, tools=None, callbacks=None, available_functions=None)
+        Generic call method for compatibility with some LLM APIs.
+    invoke(input, **kwargs)
+        Standard LangChain entrypoint for invoking the LLM.
+    __call__(*args, **kwargs)
+        Syntactic sugar for calling the wrapper like a function.
+    _format_messages(messages)
+        Utility method to render messages (string or list of role/content dicts)
+        into a human-readable string.
+
+    Notes
+    -----
+    Every call is wrapped in a :class:`flowcept.instrumentation.task_capture.FlowceptTask`
+    context. This ensures the provenance database records:
+
+    - Used: the input prompt/messages
+    - Generated: the LLM response
+    - Metadata: model configuration and optional response metadata
+
+    Examples
+    --------
+    Wrap an OpenAI model and capture provenance automatically:
+
+    >>> from langchain_openai import ChatOpenAI
+    >>> from flowcept.flowceptor.adapters.flowcept_llm import FlowceptLLM
+    >>>
+    >>> llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
+    >>> wrapped_llm = FlowceptLLM(llm, agent_id="agent_123", workflow_id="wf_456")
+    >>>
+    >>> # Example with a single string prompt
+    >>> response = wrapped_llm("What is the capital of France?")
+    >>> print(response)
+    "Paris"
+
+    Example with a list of role/content messages:
+
+    >>> messages = [
+    ...     {"role": "system", "content": "You are a helpful assistant."},
+    ...     {"role": "user", "content": "Tell me a joke about computers."}
+    ... ]
+    >>> response = wrapped_llm.invoke(messages)
+    >>> print(response)
+    "Why did the computer show up at work late? It had a hard drive!"
+
+    In both cases, provenance is captured automatically and can be
+    queried via the Flowcept API.
+    """
+
+    def __init__(
+        self,
+        llm: BaseLanguageModel,
+        agent_id: str = None,
+        parent_task_id: str = None,
+        workflow_id=None,
+        campaign_id=None,
+    ):
+        self.llm = llm
+        self.agent_id = agent_id
+        self.worflow_id = workflow_id
+        self.campaign_id = campaign_id
+        self.metadata = _extract_llm_metadata(llm)
+        self.parent_task_id = parent_task_id
+
+    def _our_call(self, messages, **kwargs):
+        messages_str = FlowceptLLM._format_messages(messages)
+        used = {"prompt": messages_str}
+        with FlowceptTask(
+            used=used,
+            subtype="llm_task",
+            custom_metadata=self.metadata,
+            agent_id=self.agent_id,
+            activity_id="llm_interaction",
+            campaign_id=self.campaign_id,
+            workflow_id=self.worflow_id,
+            parent_task_id=self.parent_task_id,
+        ) as task:
+            response = self.llm.invoke(messages, **kwargs)
+            response_str = response.content if isinstance(response, BaseMessage) else str(response)
+            generated = {"response": response_str}
+
+            if hasattr(response, "response_metadata"):
+                task._task.custom_metadata["response_metadata"] = response.response_metadata
+
+            task.end(generated=generated)
+            return response_str
+
+    def call(
+        self,
+        messages: Union[str, List[Dict[str, str]]],
+        tools: Optional[List[dict]] = None,
+        callbacks: Optional[List[Any]] = None,
+        available_functions: Optional[Dict[str, Any]] = None,
+    ) -> Union[str, Any]:
+        """Invoke method used by some other LLMs."""
+        return self._our_call(messages)
+
+    def invoke(self, input: Union[str, List[Dict[str, str]]], **kwargs) -> Any:
+        """Invoke method used by LangChain."""
+        return self._our_call(input, **kwargs)
+
+    def __call__(self, *args, **kwargs):
+        """Default call method, to be used like llm("string")."""
+        return self.invoke(*args, **kwargs)
+
+    @staticmethod
+    def _format_messages(messages: Union[str, List[Dict[str, str]]]) -> str:
+        if isinstance(messages, str):
+            return messages
+        elif isinstance(messages, list):
+            return "\n".join(f"{m.get('role', '').capitalize()}: {m.get('content', '')}" for m in messages)
+        else:
+            raise ValueError(f"Invalid message format: {messages}")

flowcept/instrumentation/flowcept_decorator.py

@@ -0,0 +1,43 @@
+import inspect
+from functools import wraps
+
+from flowcept.flowcept_api.flowcept_controller import Flowcept
+
+
+def flowcept(func=None, **flowcept_constructor_kwargs):
+    """
+    Usage:
+        @flowcept
+        def main(): ...
+
+        @flowcept(project="X", campaign_id="C123", verbose=True)
+        def main(): ...
+    """
+
+    def _decorate(f):
+        is_async = inspect.iscoroutinefunction(f)
+        flowcept_args = flowcept_constructor_kwargs.copy()
+        flowcept_args["start_persistence"] = flowcept_constructor_kwargs.get("start_persistence", False)
+        flowcept_args["save_workflow"] = flowcept_constructor_kwargs.get("save_workflow", False)
+        flowcept_args["check_safe_stops"] = flowcept_constructor_kwargs.get("check_safe_stops", False)
+
+        if is_async:
+
+            @wraps(f)
+            async def _aw(*args, **kwargs):
+                # Flowcept used as a context manager around the coroutine call
+                with Flowcept(**flowcept_args):
+                    return await f(*args, **kwargs)
+
+            return _aw
+        else:
+
+            @wraps(f)
+            def _w(*args, **kwargs):
+                with Flowcept(**flowcept_args):
+                    return f(*args, **kwargs)
+
+            return _w
+
+    # Support bare @flowcept vs @flowcept(...)
+    return _decorate if func is None else _decorate(func)

flowcept/instrumentation/flowcept_loop.py

@@ -6,7 +6,7 @@ from typing import Union, Sized, Iterator, Dict
 
 from flowcept import Flowcept
 from flowcept.commons.vocabulary import Status
-from flowcept.configs import INSTRUMENTATION_ENABLED
+from flowcept.configs import INSTRUMENTATION_ENABLED, TELEMETRY_ENABLED
 from flowcept.flowceptor.adapters.instrumentation_interceptor import InstrumentationInterceptor
 
 
@@ -160,8 +160,8 @@ class FlowceptLoop:
 
     def _end_iteration_task(self, _):
         self._last_iteration_task["status"] = Status.FINISHED.value
-        tel = FlowceptLoop._interceptor.telemetry_capture.capture()
-        if tel:
+        if TELEMETRY_ENABLED:
+            tel = FlowceptLoop._interceptor.telemetry_capture.capture()
             self._last_iteration_task["telemetry_at_end"] = tel.to_dict()
         FlowceptLoop._interceptor.intercept(self._last_iteration_task)