flowcept 0.9.3__tar.gz → 0.9.5__tar.gz

{flowcept-0.9.3 → flowcept-0.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.3
+Version: 0.9.5
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT
@@ -25,6 +25,7 @@ Requires-Dist: gitpython; extra == 'all'
 Requires-Dist: google-genai; extra == 'all'
 Requires-Dist: jupyterlab; extra == 'all'
 Requires-Dist: langchain-community; extra == 'all'
+Requires-Dist: langchain-openai; extra == 'all'
 Requires-Dist: lmdb; extra == 'all'
 Requires-Dist: mcp[cli]; extra == 'all'
 Requires-Dist: mlflow-skinny; extra == 'all'
@@ -85,11 +86,13 @@ Provides-Extra: kafka
 Requires-Dist: confluent-kafka<=2.8.0; extra == 'kafka'
 Provides-Extra: llm-agent
 Requires-Dist: langchain-community; extra == 'llm-agent'
+Requires-Dist: langchain-openai; extra == 'llm-agent'
 Requires-Dist: mcp[cli]; extra == 'llm-agent'
 Requires-Dist: streamlit; extra == 'llm-agent'
 Provides-Extra: llm-google
 Requires-Dist: google-genai; extra == 'llm-google'
 Requires-Dist: langchain-community; extra == 'llm-google'
+Requires-Dist: langchain-openai; extra == 'llm-google'
 Requires-Dist: mcp[cli]; extra == 'llm-google'
 Requires-Dist: streamlit; extra == 'llm-google'
 Provides-Extra: lmdb

{flowcept-0.9.3 → flowcept-0.9.5}/docs/api-reference.rst

@@ -1,37 +1,27 @@
 API Reference
 =============
 
-.. toctree::
-   :maxdepth: 3
-   :caption: Contents:
+.. autosummary::
+   :toctree: generated/
+   :recursive:
 
+   flowcept.Flowcept
+   flowcept.flowcept_api.db_api.DBAPI
+   flowcept.TaskObject
+   flowcept.WorkflowObject
+   flowcept.FlowceptTask
+   flowcept.FlowceptLoop
+   flowcept.FlowceptLightweightLoop
 
-Public API documentation.
-
-Core components
----------------
+Main Flowcept Object
+--------------------
 
 .. autoclass:: flowcept.Flowcept
    :members:
+   :special-members: __init__
+   :exclude-members: __weakref__, __dict__, __module__
 
 
-Main Message Objects
----------------------
-
-.. autoclass:: flowcept.TaskObject
-   :members:
-
-.. autoclass:: flowcept.WorkflowObject
-   :members:
-
-FlowceptTask object
--------------------
-
-.. autoclass:: flowcept.FlowceptTask
-   :members:
-   :special-members: __init__
-   :undoc-members:
-   :show-inheritance:
 
 Flowcept.db: Querying the Database
 ----------------------------------
@@ -58,4 +48,42 @@ Typical usage:
 .. autoclass:: flowcept.flowcept_api.db_api.DBAPI
    :members:
    :undoc-members:
+   :show-inheritance:
+
+
+Main Message Objects
+---------------------
+
+.. autoclass:: flowcept.TaskObject
+   :members:
+
+.. autoclass:: flowcept.WorkflowObject
+   :members:
+
+FlowceptTask object
+-------------------
+
+.. autoclass:: flowcept.FlowceptTask
+   :members:
+   :special-members: __init__
+   :undoc-members:
+   :show-inheritance:
+
+FlowceptLoop object
+-------------------
+
+.. autoclass:: flowcept.FlowceptLoop
+   :members:
+   :special-members: __init__
+   :undoc-members:
+   :show-inheritance:
+
+
+FlowceptLightweightLoop object
+------------------------------
+
+.. autoclass:: flowcept.FlowceptLightweightLoop
+   :members:
+   :special-members: __init__
+   :undoc-members:
    :show-inheritance:

{flowcept-0.9.3 → flowcept-0.9.5}/docs/architecture.rst

@@ -1,5 +1,5 @@
 Flowcept Architecture
-===================
+======================
 
 .. toctree::
    :maxdepth: 2

{flowcept-0.9.3 → flowcept-0.9.5}/docs/prov_capture.rst

@@ -435,18 +435,20 @@ Requires an active workflow (``with Flowcept(...)`` or ``Flowcept().start()``).
    with Flowcept(workflow_name="custom_tasks"):
        # Context-managed publish
        with FlowceptTask(activity_id="download", used={"url": "https://..."}) as t:
-           data = b"..."
-           t.add_generated({"bytes": len(data)})
+           data = b"..." # Some binary data
+           t.end(data=data, generated={"bytes": len(data)})
 
        # Or publish explicitly
        task = FlowceptTask(activity_id="parse", used={"bytes": len(data)})
-       task.add_generated({"records": 42})
+       task.end({"records": 42})
        task.send()  # publishes to MQ
 
 **Notes**:
 
 - Use **context** (``with FlowceptTask(...)``) *or* call ``send()`` explicitly.
 - Flows publish to the MQ; persistence/queries require a DB (e.g., MongoDB).
+- See also: `Consumer example <https://flowcept.readthedocs.io/en/latest/prov_storage.html#example-extending-the-base-consumer>`_
+- See also: `Ping pong example via PubSub with Flowcept <https://github.com/ORNL/flowcept/blob/main/examples/consumers/ping_pong_example.py>`_
 
 References & Examples
 ---------------------

{flowcept-0.9.3 → flowcept-0.9.5}/docs/prov_storage.rst

@@ -84,6 +84,12 @@ This can serve as a template for building custom provenance consumers.
        consumer = MyConsumer()
        consumer.start(daemon=False)
 
+**Notes**:
+
+- See also: `Explicit publish example <file:///Users/rsr/Documents/GDrive/ORNL/dev/flowcept/docs/_build/html/prov_capture.html#custom-task-creation-fully-customizable>`_
+- See also: `Ping pong example via PubSub with Flowcept <https://github.com/ORNL/flowcept/blob/main/examples/consumers/ping_pong_example.py>`_
+
+
 
 Document Inserter
 -----------------

{flowcept-0.9.3 → flowcept-0.9.5}/pyproject.toml

@@ -65,7 +65,7 @@ mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog", "cryptography"]
 nvidia = ["nvidia-ml-py"]
 mqtt = ["paho-mqtt"]
 tensorboard = ["tensorboard", "tensorflow", "tbparse"]
-llm_agent = ["mcp[cli]", "langchain_community", "streamlit"]
+llm_agent = ["mcp[cli]", "langchain_community", "streamlit", "langchain_openai"]
 llm_google = ["flowcept[llm_agent]", "google-genai"]
 
 dev = [

{flowcept-0.9.3 → flowcept-0.9.5}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.9.3 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.5 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/agents/flowcept_ctx_manager.py

@@ -37,6 +37,7 @@ class FlowceptAppContext(BaseAppContext):
     tasks_schema: Dict | None  # TODO: we dont need to keep the tasks_schema in context, just in the manager's memory.
     value_examples: Dict | None
     tracker_config: Dict | None
+    custom_guidance: List[str] | None
 
 
 class FlowceptAgentContextManager(BaseAgentContextManager):
@@ -156,6 +157,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
             df=pd.DataFrame(),
             tasks_schema={},
             value_examples={},
+            custom_guidance=[],
             tracker_config=self.tracker_config,
         )
         DEBUG = True  # TODO debugging!

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/agents/prompts/general_prompts.py

@@ -24,8 +24,9 @@ ROUTING_PROMPT = (
     "Given the following user message, classify it into one of the following routes:\n"
     "- small_talk: if it's casual conversation or some random word (e.g., 'hausdn', 'a', hello, how are you, what can you do, what's your name)\n"
     "- plot: if user is requesting plots (e.g., plot, chart, visualize)\n"
-    "- in_context_query: if the user asks questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
-    "- historical_prov_query: if the user wants to query historical provenance data\n"
+    #"- in_context_query: if the user asks questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
+    "- in_context_query: if the user is querying the provenance data questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
+    #"- historical_prov_query: if the user wants to query historical provenance data\n"
     "- in_chat_query: if the user appears to be asking about something that has said recently in this chat.\n"
     "- unknown: if you don't know.\n"
     "Respond with only the route label."

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/agents/prompts/in_memory_query_prompts.py

@@ -220,7 +220,15 @@ OUTPUT_FORMATTING = """
 """
 
 
-def generate_pandas_code_prompt(query: str, dynamic_schema, example_values):
+def generate_pandas_code_prompt(query: str, dynamic_schema, example_values, custom_user_guidances):
+    if custom_user_guidances is not None and isinstance(custom_user_guidances, list) and len(custom_user_guidances):
+        concatenated_guidance = "\n".join(f"- {msg}" for msg in custom_user_guidances)
+        custom_user_guidance_prompt = (f"You MUST consider the following guidance from the user:\n"
+                                       f"{concatenated_guidance}"
+                                       "------------------------------------------------------"
+                                       )
+    else:
+        custom_user_guidance_prompt = ""
     prompt = (
         f"{ROLE}"
         f"{JOB}"
@@ -228,6 +236,7 @@ def generate_pandas_code_prompt(query: str, dynamic_schema, example_values):
         f"{get_df_schema_prompt(dynamic_schema, example_values)}"  # main tester
         f"{QUERY_GUIDELINES}"  # main tester
         f"{FEW_SHOTS}"  # main tester
+        f"{custom_user_guidance_prompt}"
         f"{OUTPUT_FORMATTING}"
         "User Query:"
         f"{query}"

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/agents/tools/general_tools.py

@@ -1,4 +1,5 @@
 import json
+from typing import List
 
 from flowcept.agents.agents_utils import build_llm_model, ToolResult
 from flowcept.agents.flowcept_ctx_manager import mcp_flowcept
@@ -59,6 +60,16 @@ def check_llm() -> str:
     return response
 
 
+@mcp_flowcept.tool()
+def record_guidance(message: str) -> ToolResult:
+    ctx = mcp_flowcept.get_context()
+    message = message.replace("@record", "")
+    custom_guidance: List = ctx.request_context.lifespan_context.custom_guidance
+    custom_guidance.append(message)
+
+    return ToolResult(code=201, result=f"Ok. I recorded in my memory: {message}")
+
+
 @mcp_flowcept.tool()
 def prompt_handler(message: str) -> ToolResult:
     """
@@ -79,6 +90,10 @@ def prompt_handler(message: str) -> ToolResult:
         if key in message:
             return run_df_query(llm=None, query=message, plot=False)
 
+    if "@record" in message:
+        return record_guidance(message)
+
+
     llm = build_llm_model()
 
     prompt = ROUTING_PROMPT + message

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/agents/tools/in_memory_queries/in_memory_queries_tools.py

@@ -87,6 +87,7 @@ def run_df_query(llm, query: str, plot=False) -> ToolResult:
     df: pd.DataFrame = ctx.request_context.lifespan_context.df
     schema = ctx.request_context.lifespan_context.tasks_schema
     value_examples = ctx.request_context.lifespan_context.value_examples
+    custom_user_guidance = ctx.request_context.lifespan_context.custom_guidance
     if df is None or not len(df):
         return ToolResult(code=404, result="Current df is empty or null.")
 
@@ -99,13 +100,13 @@ def run_df_query(llm, query: str, plot=False) -> ToolResult:
         return run_df_code(user_code=query, df=df)
 
     if plot:
-        return generate_plot_code(llm, query, schema, value_examples, df)
+        return generate_plot_code(llm, query, schema, value_examples, df, custom_user_guidance=custom_user_guidance)
     else:
-        return generate_result_df(llm, query, schema, value_examples, df)
+        return generate_result_df(llm, query, schema, value_examples, df, custom_user_guidance=custom_user_guidance)
 
 
 @mcp_flowcept.tool()
-def generate_plot_code(llm, query, dynamic_schema, value_examples, df) -> ToolResult:
+def generate_plot_code(llm, query, dynamic_schema, value_examples, df, custom_user_guidance=None) -> ToolResult:
     """
     Generate DataFrame and plotting code from a natural language query using an LLM.
 
@@ -221,7 +222,7 @@ def generate_plot_code(llm, query, dynamic_schema, value_examples, df) -> ToolRe
 
 
 @mcp_flowcept.tool()
-def generate_result_df(llm, query: str, dynamic_schema, example_values, df, attempt_fix=True, summarize=True):
+def generate_result_df(llm, query: str, dynamic_schema, example_values, df, custom_user_guidance=None, attempt_fix=True, summarize=True):
     """
     Generate a result DataFrame from a natural language query using an LLM.
 
@@ -297,7 +298,7 @@ def generate_result_df(llm, query: str, dynamic_schema, example_values, df, atte
     if llm is None:
         llm = build_llm_model()
     try:
-        prompt = generate_pandas_code_prompt(query, dynamic_schema, example_values)
+        prompt = generate_pandas_code_prompt(query, dynamic_schema, example_values, custom_user_guidance)
         response = llm(prompt)
     except Exception as e:
         return ToolResult(code=400, result=str(e), extra=prompt)

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/agents/tools/in_memory_queries/pandas_agent_utils.py

@@ -77,7 +77,7 @@ def normalize_output(result):
         raise TypeError(f"Unsupported result type: {type(result)}")
 
     if not len(_df):
-        raise ValueError("Result DataFrame is Empty.")
+        return pd.DataFrame()
 
     return _df
 

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/flowcept_api/flowcept_controller.py

@@ -25,7 +25,7 @@ from flowcept.flowceptor.adapters.base_interceptor import BaseInterceptor
 
 
 class Flowcept(object):
-    """Flowcept Controller class."""
+    """Main Flowcept controller class."""
 
     _db = None
     # TODO: rename current_workflow_id to workflow_id. This will be a major refactor

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/instrumentation/flowcept_loop.py

@@ -18,26 +18,6 @@ class FlowceptLoop:
     while capturing metadata for each iteration and for the loop as a whole. This is particularly
     useful in scenarios where tracking and instrumentation of loop executions is required.
 
-    Parameters
-    ----------
-    items : typing.Union[Sized, int, Iterator]
-        The items to iterate over. Must either be an iterable with a `__len__` method or an integer
-        representing the range of iteration.
-    loop_name : str, optional
-        A descriptive name for the loop (default is "loop").
-    item_name : str, optional
-        The name used for each item in the telemetry (default is "item").
-    parent_task_id : str, optional
-        The ID of the parent task associated with the loop, if applicable (default is None).
-    workflow_id : str, optional
-        The workflow ID to associate with this loop. If not provided, it will be generated or
-        inferred from the current workflow context.
-
-    Raises
-    ------
-    Exception
-        If `items` is not an iterable with a `__len__` method or an integer.
-
     Notes
     -----
     This class integrates with the `Flowcept` system for telemetry and tracking, ensuring
@@ -57,6 +37,52 @@ class FlowceptLoop:
         items_length=0,
         capture_enabled=True,
     ):
+        """
+        Initialize a FlowceptLoop instance for tracking iterations.
+
+        This constructor wraps an iterable, numeric range, or explicit iterator into a
+        loop context where each iteration is instrumented with provenance and optional
+        telemetry. If instrumentation is disabled, the loop behaves like a normal
+        Python iterator with minimal overhead.
+
+        Parameters
+        ----------
+        items : Union[Sized, Iterator, int]
+            The items to iterate over. Can be:
+            - A sized iterable (e.g., list, range).
+            - An integer (interpreted as ``range(items)``).
+            - An iterator (requires ``items_length`` if length cannot be inferred).
+        loop_name : str, optional
+            A descriptive name for the loop. Used in provenance as the loop's activity
+            identifier. Default is ``"loop"``.
+        item_name : str, optional
+            The key name under which each iteration's item is recorded in provenance.
+            Default is ``"item"``.
+        parent_task_id : str, optional
+            The identifier of a parent task, if this loop is nested within another task.
+            Default is ``None``.
+        workflow_id : str, optional
+            Identifier for the workflow this loop belongs to. If not provided, it is
+            inherited from the current Flowcept context or generated as a UUID.
+        items_length : int, optional
+            Explicit number of items if ``items`` is an iterator without a defined length.
+            Default is ``0``.
+        capture_enabled : bool, optional
+            Whether to enable provenance/telemetry capture. If ``False``, the loop runs
+            without instrumentation. Default is ``True``.
+
+        Raises
+        ------
+        Exception
+            If ``items`` is not a supported type (sized iterable, integer, or iterator).
+
+        Notes
+        -----
+        - Each iteration is recorded with ``used`` (inputs) and optional ``generated``
+          values, plus telemetry if enabled.
+        - Iteration metadata is finalized at the end of each iteration and sent to the
+          active Flowcept interceptor.
+        """
         self._current_iteration_task = {}
         if not (INSTRUMENTATION_ENABLED and capture_enabled):
             # These do_nothing functions help reduce overhead if no instrumentation is needed
@@ -232,6 +258,51 @@ class FlowceptLightweightLoop:
         items_length=0,
         capture_enabled=True,
     ):
+        """
+        Initialize a FlowceptLightweightLoop instance for tracking iterations.
+
+        This constructor provides a lower-overhead loop wrapper compared to
+        ``FlowceptLoop``. Iterations are pre-registered as task objects, and capture
+        primarily updates ``used`` and ``generated`` fields as the loop progresses.
+
+        Parameters
+        ----------
+        items : Union[Sized, Iterator]
+            The items to iterate over. Must either be:
+            - A sized iterable (with ``__len__``).
+            - An explicit iterator (length must be given by ``items_length``).
+        loop_name : str, optional
+            A descriptive name for the loop. Used in provenance as the loop's activity
+            identifier. Default is ``"loop"``.
+        item_name : str, optional
+            The key name under which each iteration's item is recorded in provenance.
+            Default is ``"item"``.
+        parent_task_id : str, optional
+            The identifier of a parent task, if this loop is nested within another task.
+            Default is ``None``.
+        workflow_id : str, optional
+            Identifier for the workflow this loop belongs to. If not provided, it is
+            inherited from the current Flowcept context or generated as a UUID.
+        items_length : int, optional
+            Explicit number of items if ``items`` is an iterator without a defined length.
+            Default is ``0``.
+        capture_enabled : bool, optional
+            Whether to enable provenance/telemetry capture. If ``False``, the loop runs
+            without instrumentation. Default is ``True``.
+
+        Raises
+        ------
+        Exception
+            If ``items`` is neither a sized iterable nor an iterator.
+
+        Notes
+        -----
+        - This class is designed for high-performance scenarios with many iterations.
+        - Iteration tasks are pre-allocated, and provenance capture is batched via
+          the Flowcept interceptor.
+        - Compared to ``FlowceptLoop``, this class avoids per-iteration telemetry
+          overhead unless explicitly enabled.
+        """
         if isinstance(items, Iterator):
             self._iterator = items
         else:

{flowcept-0.9.3 → flowcept-0.9.5}/src/flowcept/version.py

@@ -4,4 +4,4 @@
 # The expected format is: <Major>.<Minor>.<Patch>
 # This file is supposed to be automatically modified by the CI Bot.
 # See .github/workflows/version_bumper.py
-__version__ = "0.9.3"
+__version__ = "0.9.5"