flowcept 0.9.2__tar.gz → 0.9.4__tar.gz

{flowcept-0.9.2 → flowcept-0.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.2
+Version: 0.9.4
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT
@@ -25,6 +25,7 @@ Requires-Dist: gitpython; extra == 'all'
 Requires-Dist: google-genai; extra == 'all'
 Requires-Dist: jupyterlab; extra == 'all'
 Requires-Dist: langchain-community; extra == 'all'
+Requires-Dist: langchain-openai; extra == 'all'
 Requires-Dist: lmdb; extra == 'all'
 Requires-Dist: mcp[cli]; extra == 'all'
 Requires-Dist: mlflow-skinny; extra == 'all'
@@ -85,11 +86,13 @@ Provides-Extra: kafka
 Requires-Dist: confluent-kafka<=2.8.0; extra == 'kafka'
 Provides-Extra: llm-agent
 Requires-Dist: langchain-community; extra == 'llm-agent'
+Requires-Dist: langchain-openai; extra == 'llm-agent'
 Requires-Dist: mcp[cli]; extra == 'llm-agent'
 Requires-Dist: streamlit; extra == 'llm-agent'
 Provides-Extra: llm-google
 Requires-Dist: google-genai; extra == 'llm-google'
 Requires-Dist: langchain-community; extra == 'llm-google'
+Requires-Dist: langchain-openai; extra == 'llm-google'
 Requires-Dist: mcp[cli]; extra == 'llm-google'
 Requires-Dist: streamlit; extra == 'llm-google'
 Provides-Extra: lmdb

{flowcept-0.9.2 → flowcept-0.9.4}/docs/api-reference.rst

@@ -1,37 +1,27 @@
 API Reference
 =============
 
-.. toctree::
-   :maxdepth: 3
-   :caption: Contents:
+.. autosummary::
+   :toctree: generated/
+   :recursive:
 
+   flowcept.Flowcept
+   flowcept.flowcept_api.db_api.DBAPI
+   flowcept.TaskObject
+   flowcept.WorkflowObject
+   flowcept.FlowceptTask
+   flowcept.FlowceptLoop
+   flowcept.FlowceptLightweightLoop
 
-Public API documentation.
-
-Core components
----------------
+Main Flowcept Object
+--------------------
 
 .. autoclass:: flowcept.Flowcept
    :members:
+   :special-members: __init__
+   :exclude-members: __weakref__, __dict__, __module__
 
 
-Main Message Objects
----------------------
-
-.. autoclass:: flowcept.TaskObject
-   :members:
-
-.. autoclass:: flowcept.WorkflowObject
-   :members:
-
-FlowceptTask object
--------------------
-
-.. autoclass:: flowcept.FlowceptTask
-   :members:
-   :special-members: __init__
-   :undoc-members:
-   :show-inheritance:
 
 Flowcept.db: Querying the Database
 ----------------------------------
@@ -58,4 +48,42 @@ Typical usage:
 .. autoclass:: flowcept.flowcept_api.db_api.DBAPI
    :members:
    :undoc-members:
+   :show-inheritance:
+
+
+Main Message Objects
+---------------------
+
+.. autoclass:: flowcept.TaskObject
+   :members:
+
+.. autoclass:: flowcept.WorkflowObject
+   :members:
+
+FlowceptTask object
+-------------------
+
+.. autoclass:: flowcept.FlowceptTask
+   :members:
+   :special-members: __init__
+   :undoc-members:
+   :show-inheritance:
+
+FlowceptLoop object
+-------------------
+
+.. autoclass:: flowcept.FlowceptLoop
+   :members:
+   :special-members: __init__
+   :undoc-members:
+   :show-inheritance:
+
+
+FlowceptLightweightLoop object
+------------------------------
+
+.. autoclass:: flowcept.FlowceptLightweightLoop
+   :members:
+   :special-members: __init__
+   :undoc-members:
    :show-inheritance:

{flowcept-0.9.2 → flowcept-0.9.4}/docs/architecture.rst

@@ -1,5 +1,5 @@
 Flowcept Architecture
-===================
+======================
 
 .. toctree::
    :maxdepth: 2

{flowcept-0.9.2 → flowcept-0.9.4}/docs/prov_capture.rst

@@ -25,6 +25,10 @@ Decorators
 --------------------------
 
 Use decorators to mark functions as **workflows** or **tasks** with almost no code changes.
+If using the decorators, we expect that `instrumentation` is enabled in your settings file.
+If it is not, the provenance capture will be simply ignored and the decorated function
+will run as if without any Flowcept instrumentation.
+
 
 ``@flowcept`` (wrap a “main” function as a workflow)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -431,18 +435,20 @@ Requires an active workflow (``with Flowcept(...)`` or ``Flowcept().start()``).
    with Flowcept(workflow_name="custom_tasks"):
        # Context-managed publish
        with FlowceptTask(activity_id="download", used={"url": "https://..."}) as t:
-           data = b"..."
-           t.add_generated({"bytes": len(data)})
+           data = b"..." # Some binary data
+           t.end(data=data, generated={"bytes": len(data)})
 
        # Or publish explicitly
        task = FlowceptTask(activity_id="parse", used={"bytes": len(data)})
-       task.add_generated({"records": 42})
+       task.end({"records": 42})
        task.send()  # publishes to MQ
 
 **Notes**:
 
 - Use **context** (``with FlowceptTask(...)``) *or* call ``send()`` explicitly.
 - Flows publish to the MQ; persistence/queries require a DB (e.g., MongoDB).
+- See also: `Consumer example <https://flowcept.readthedocs.io/en/latest/prov_storage.html#example-extending-the-base-consumer>`_
+- See also: `Ping pong example via PubSub with Flowcept <https://github.com/ORNL/flowcept/blob/main/examples/consumers/ping_pong_example.py>`_
 
 References & Examples
 ---------------------

{flowcept-0.9.2 → flowcept-0.9.4}/docs/prov_storage.rst

@@ -84,6 +84,12 @@ This can serve as a template for building custom provenance consumers.
        consumer = MyConsumer()
        consumer.start(daemon=False)
 
+**Notes**:
+
+- See also: `Explicit publish example <file:///Users/rsr/Documents/GDrive/ORNL/dev/flowcept/docs/_build/html/prov_capture.html#custom-task-creation-fully-customizable>`_
+- See also: `Ping pong example via PubSub with Flowcept <https://github.com/ORNL/flowcept/blob/main/examples/consumers/ping_pong_example.py>`_
+
+
 
 Document Inserter
 -----------------

flowcept-0.9.4/examples/convergence_loop_example.py

@@ -0,0 +1,44 @@
+from flowcept import Flowcept, FlowceptLoop
+
+THRESHOLD = 0.05
+MAX_ITERS = 50
+
+
+def update_error(e: float) -> float:
+    """Dummy update rule that shrinks error each step."""
+    return e * 0.7  # replace with your real update
+
+
+def finalize_loop(loop):
+    """Flush the last iteration so it’s captured even if we break early."""
+    if getattr(loop, "enabled", False) and getattr(loop, "_last_iteration_task", None) is not None:
+        try:
+            loop._end_iteration_task(loop._last_iteration_task)  # uses FlowceptLoop internals
+        except Exception:
+            pass
+
+
+# --- Convergence with FlowceptLoop ---
+error = 1.0
+loop = FlowceptLoop(
+    items=range(MAX_ITERS),     # upper bound; we’ll break early on convergence
+    loop_name="convergence",
+    item_name="iter",
+)
+
+for it in loop:
+    # --- your loop body ---
+    error = update_error(error)
+    print(f"iter={it}  error={error:.4f}")
+
+    # Record per-iteration outputs (only once per iteration)
+    loop.end_iter({
+        "iter": it,
+        "error": error,
+        "status": "converged" if error <= THRESHOLD else "continuing",
+    })
+
+    # Convergence check — emulate: while error > THRESHOLD: update error
+    if error <= THRESHOLD:
+        finalize_loop(loop)   # ensure the last iteration is closed before breaking
+        break

flowcept-0.9.4/examples/instrumented_loop_example.py

@@ -0,0 +1,77 @@
+import random
+from time import sleep
+
+from flowcept import Flowcept, FlowceptLoop
+
+# EXAMPLE 1:
+
+
+iterations = range(1, 5)
+
+with Flowcept():
+
+    loop = FlowceptLoop(iterations)         # See also: FlowceptLightweightLoop
+    for item in loop:
+        loss = random.random()
+        sleep(0.05)
+        print(item, loss)
+        # The following is optional, in case you want to capture values generated inside the loop.
+        loop.end_iter({"item": item, "loss": loss})
+
+docs = Flowcept.db.get_tasks_from_current_workflow()
+assert len(docs) == len(iterations)
+
+
+# EXAMPLE 2: WHILE LOOP EXAMPLE
+# The following code is equivalent to:
+# while layer_for_control_update <= num_layers - 1:
+#     ... loop code ...
+#    layer_for_control_update += 1
+
+layer_for_control_update = 2
+num_layers = 6
+
+
+def do_work(layer_ix: int) -> dict:
+    """Dummy per-iteration work that returns something to log."""
+    # simulate some computation
+    value = layer_ix * 10
+    return {"value": value}
+
+
+# Wrap the loop with FlowceptLoop
+iterations = range(layer_for_control_update, num_layers)
+
+# You can create with_flowcept flags
+with_flowcept = True
+
+if with_flowcept:
+    loop = FlowceptLoop(
+        items=iterations,
+        loop_name="control_loop",
+        item_name="layer_ix",
+        # parent_task_id="optional-parent-id",
+        # workflow_id="optional-workflow-id",
+    )
+    iterations = loop
+
+
+for layer_ix in iterations:
+    # Your loop body
+    result = do_work(layer_ix)
+    print(f"Processed layer {layer_ix}, result={result}")
+
+    if with_flowcept:
+        # Record per-iteration outputs
+        iterations.end_iter({
+            "layer_ix": layer_ix,
+            "status": "updated",
+            **result,                 # attach any metrics you computed
+        })
+
+if with_flowcept:
+    docs = Flowcept.db.get_tasks_from_current_workflow()
+    assert len(docs) == len(iterations)
+
+
+# EXAMPLE 3: WHILE LOOP EXAMPLE

flowcept-0.9.4/examples/mqtt_example.py

@@ -0,0 +1,101 @@
+# broker_minimal_example.py
+import json
+import time
+import uuid
+import paho.mqtt.client as mqtt
+
+from flowcept import Flowcept
+from flowcept.configs import settings
+
+
+def configure_flowcept_for_mqtt():
+    """
+    Mirror your settings.yaml exactly under adapters.broker_mqtt so the
+    MQTTBrokerInterceptor picks it up via self.settings.
+    """
+    settings.setdefault("adapters", {})
+    settings["adapters"]["broker_mqtt"] = {
+        "kind": "broker",
+        "host": "localhost",
+        "port": 30011,
+        "protocol": "mqtt3.1.1",
+        "queues": ["#"],
+        "username": "postman",
+        "password": "p",
+        "qos": 2,
+        "task_subtype": "intersect_msg",
+        "tracked_keys": {
+            "used": "payload",
+            "generated": None,          # ~ in YAML
+            "custom_metadata": ["headers", "msgId"],
+            "activity_id": "operationId",
+            "submitted_at": None,
+            "started_at": None,
+            "ended_at": None,
+            "registered_at": None,
+        },
+    }
+
+
+def publish_one_message(
+    host="localhost",
+    port=30011,
+    username="postman",
+    password="p",
+    topic="flowcept/demo/test",
+):
+    """
+    Publish a single JSON payload with the fields your tracked_keys expect:
+    - payload            -> mapped to Task.used
+    - headers, msgId     -> copied into Task.custom_metadata
+    - operationId        -> mapped to Task.activity_id
+    """
+    client = mqtt.Client(protocol=mqtt.MQTTv311)
+    client.username_pw_set(username, password)
+    client.connect(host, port, keepalive=60)
+
+    payload = {
+        "payload": {"greeting": "hello from mqtt"},   # becomes Task.used
+        "headers": {"x-source": "unit-test", "x-env": "local"},
+        "msgId": str(uuid.uuid4()),
+        "operationId": "op-xyz-123",                  # becomes Task.activity_id
+        # optional extras also okay
+        "status": "FINISHED",
+    }
+
+    client.publish(topic, json.dumps(payload), qos=2, retain=False)
+    client.disconnect()
+    return payload
+
+
+def main():
+    # 1) Apply settings that match your YAML
+    configure_flowcept_for_mqtt()
+
+    # 2) Start Flowcept in context for the broker_mqtt adapter
+    with Flowcept("broker_mqtt"):
+        # 3) Start the interceptor (runs MQTT loop in a background thread)
+        # Give it a moment to connect/subscribe to "#"
+        time.sleep(1.0)
+
+        # 4) Publish one message
+        sent = publish_one_message()
+        print("Published:", sent)
+
+        # 5) Allow time for ingest
+        time.sleep(2.0)
+
+    # 6) (Optional) Query Flowcept DB for the ingested task
+    try:
+        # Query by nested field in used (payload.greeting)
+        tasks = Flowcept.db.query(filter={"used.greeting": "hello from mqtt"})
+        if tasks:
+            print("Ingested task example:\n", json.dumps(tasks[0], indent=2))
+        else:
+            print("No tasks found for filter used.greeting == 'hello from mqtt'")
+    except Exception as e:
+        print(f"DB query failed/skipped: {e}")
+
+
+if __name__ == "__main__":
+    main()

{flowcept-0.9.2 → flowcept-0.9.4}/pyproject.toml

@@ -65,7 +65,7 @@ mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog", "cryptography"]
 nvidia = ["nvidia-ml-py"]
 mqtt = ["paho-mqtt"]
 tensorboard = ["tensorboard", "tensorflow", "tbparse"]
-llm_agent = ["mcp[cli]", "langchain_community", "streamlit"]
+llm_agent = ["mcp[cli]", "langchain_community", "streamlit", "langchain_openai"]
 llm_google = ["flowcept[llm_agent]", "google-genai"]
 
 dev = [

{flowcept-0.9.2 → flowcept-0.9.4}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.9.2 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.4 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.

{flowcept-0.9.2 → flowcept-0.9.4}/src/flowcept/agents/flowcept_ctx_manager.py

@@ -37,6 +37,7 @@ class FlowceptAppContext(BaseAppContext):
     tasks_schema: Dict | None  # TODO: we dont need to keep the tasks_schema in context, just in the manager's memory.
     value_examples: Dict | None
     tracker_config: Dict | None
+    custom_guidance: List[str] | None
 
 
 class FlowceptAgentContextManager(BaseAgentContextManager):
@@ -53,7 +54,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         Current application context holding task state and QA components.
     msgs_counter : int
         Counter tracking how many task messages have been processed.
-    context_size : int
+    context_chunk_size : int
         Number of task messages to collect before triggering QA index building and LLM analysis.
     qa_manager : FlowceptQAManager
         Utility for constructing QA chains from task summaries.
@@ -64,7 +65,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         self.tracker_config = dict(max_examples=3, max_str_len=50)
         self.schema_tracker = DynamicSchemaTracker(**self.tracker_config)
         self.msgs_counter = 0
-        self.context_size = 1
+        self.context_chunk_size = 1  # Should be in the settings
         super().__init__()
 
     def message_handler(self, msg_obj: Dict):
@@ -98,18 +99,22 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
             if len(task_summary.get("tags", [])):
                 self.context.critical_tasks.append(task_summary)
 
-            if self.msgs_counter > 0 and self.msgs_counter % self.context_size == 0:
+            if self.msgs_counter > 0 and self.msgs_counter % self.context_chunk_size == 0:
                 self.logger.debug(
-                    f"Going to add to index! {(self.msgs_counter - self.context_size, self.msgs_counter)}"
+                    f"Going to add to index! {(self.msgs_counter - self.context_chunk_size, self.msgs_counter)}"
                 )
                 try:
                     self.update_schema_and_add_to_df(
-                        tasks=self.context.task_summaries[self.msgs_counter - self.context_size : self.msgs_counter]
+                        tasks=self.context.task_summaries[
+                            self.msgs_counter - self.context_chunk_size : self.msgs_counter
+                        ]
                     )
                 except Exception as e:
                     self.logger.error(
                         f"Could not add these tasks to buffer!\n"
-                        f"{self.context.task_summaries[self.msgs_counter - self.context_size : self.msgs_counter]}"
+                        f"{
+                            self.context.task_summaries[self.msgs_counter - self.context_chunk_size : self.msgs_counter]
+                        }"
                     )
                     self.logger.exception(e)
 
@@ -152,6 +157,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
             df=pd.DataFrame(),
             tasks_schema={},
             value_examples={},
+            custom_guidance=[],
             tracker_config=self.tracker_config,
         )
         DEBUG = True  # TODO debugging!

{flowcept-0.9.2 → flowcept-0.9.4}/src/flowcept/agents/gui/gui_utils.py

@@ -1,3 +1,5 @@
+import base64
+import ast
 import io
 import json
 
@@ -122,6 +124,53 @@ def display_ai_msg_from_tool(tool_result: ToolResult):
     return agent_reply
 
 
+def _sniff_mime(b: bytes) -> str:
+    if b.startswith(b"\x89PNG\r\n\x1a\n"):
+        return "image/png"
+    if b.startswith(b"\xff\xd8\xff"):
+        return "image/jpeg"
+    if b.startswith(b"GIF87a") or b.startswith(b"GIF89a"):
+        return "image/gif"
+    if b.startswith(b"BM"):
+        return "image/bmp"
+    if b.startswith(b"RIFF") and b[8:12] == b"WEBP":
+        return "image/webp"
+    return "application/octet-stream"
+
+
+def ensure_data_uri(val):
+    r"""Accepts bytes/bytearray/memoryview or a repr like \"b'\\x89PNG...'\" and returns a data URL."""
+    if isinstance(val, str) and val.startswith("data:"):
+        return val
+    if isinstance(val, str) and val.startswith("b'"):
+        try:
+            val = ast.literal_eval(val)  # turn repr into bytes
+        except Exception:
+            return None
+    if isinstance(val, memoryview):
+        val = val.tobytes()
+    if isinstance(val, bytearray):
+        val = bytes(val)
+    if isinstance(val, bytes):
+        mime = _sniff_mime(val)
+        return f"data:{mime};base64,{base64.b64encode(val).decode('ascii')}"
+    return val  # path/URL, etc.
+
+
+def _render_df(df: pd.DataFrame, image_width: int = 90, row_height: int = 90):
+    if "image" in df.columns:
+        df = df.copy()
+        df["image"] = df["image"].apply(ensure_data_uri)
+        st.dataframe(
+            df,
+            column_config={"image": st.column_config.ImageColumn("Preview", width=image_width)},
+            hide_index=True,
+            row_height=row_height,  # make thumbnails visible
+        )
+    else:
+        st.dataframe(df, hide_index=True)
+
+
 def display_df_tool_response(tool_result: ToolResult):
     r"""
     Display the DataFrame contained in a ToolResult.
@@ -170,7 +219,8 @@ def display_df_tool_response(tool_result: ToolResult):
             df = pd.read_csv(io.StringIO(result_df_str))
             print("The result is a df")
             if not df.empty:
-                st.dataframe(df, hide_index=False)
+                _render_df(df)
+
                 print("Columns", str(df.columns))
                 print("Number of columns", len(df.columns))
             else:
@@ -190,6 +240,7 @@ def display_df_tool_response(tool_result: ToolResult):
 
         if summary:
             st.markdown("📝 Summary:")
+            print(f"THIS IS THE SUMMARY\n{summary}")
             st.markdown(summary)
         elif summary_error:
             st.markdown(f"⚠️ Encountered this error when summarizing the result dataframe:\n```text\n{summary_error}")

{flowcept-0.9.2 → flowcept-0.9.4}/src/flowcept/agents/prompts/general_prompts.py

@@ -24,8 +24,9 @@ ROUTING_PROMPT = (
     "Given the following user message, classify it into one of the following routes:\n"
     "- small_talk: if it's casual conversation or some random word (e.g., 'hausdn', 'a', hello, how are you, what can you do, what's your name)\n"
     "- plot: if user is requesting plots (e.g., plot, chart, visualize)\n"
-    "- in_context_query: if the user asks questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
-    "- historical_prov_query: if the user wants to query historical provenance data\n"
+    #"- in_context_query: if the user asks questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
+    "- in_context_query: if the user is querying the provenance data questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
+    #"- historical_prov_query: if the user wants to query historical provenance data\n"
     "- in_chat_query: if the user appears to be asking about something that has said recently in this chat.\n"
     "- unknown: if you don't know.\n"
     "Respond with only the route label."