flowcept 0.9.2__tar.gz → 0.9.3__tar.gz

{flowcept-0.9.2 → flowcept-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.2
+Version: 0.9.3
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT

{flowcept-0.9.2 → flowcept-0.9.3}/docs/prov_capture.rst

@@ -25,6 +25,10 @@ Decorators
 --------------------------
 
 Use decorators to mark functions as **workflows** or **tasks** with almost no code changes.
+If using the decorators, we expect that `instrumentation` is enabled in your settings file.
+If it is not, the provenance capture will be simply ignored and the decorated function
+will run as if without any Flowcept instrumentation.
+
 
 ``@flowcept`` (wrap a “main” function as a workflow)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

flowcept-0.9.3/examples/convergence_loop_example.py

@@ -0,0 +1,44 @@
+from flowcept import Flowcept, FlowceptLoop
+
+THRESHOLD = 0.05
+MAX_ITERS = 50
+
+
+def update_error(e: float) -> float:
+    """Dummy update rule that shrinks error each step."""
+    return e * 0.7  # replace with your real update
+
+
+def finalize_loop(loop):
+    """Flush the last iteration so it’s captured even if we break early."""
+    if getattr(loop, "enabled", False) and getattr(loop, "_last_iteration_task", None) is not None:
+        try:
+            loop._end_iteration_task(loop._last_iteration_task)  # uses FlowceptLoop internals
+        except Exception:
+            pass
+
+
+# --- Convergence with FlowceptLoop ---
+error = 1.0
+loop = FlowceptLoop(
+    items=range(MAX_ITERS),     # upper bound; we’ll break early on convergence
+    loop_name="convergence",
+    item_name="iter",
+)
+
+for it in loop:
+    # --- your loop body ---
+    error = update_error(error)
+    print(f"iter={it}  error={error:.4f}")
+
+    # Record per-iteration outputs (only once per iteration)
+    loop.end_iter({
+        "iter": it,
+        "error": error,
+        "status": "converged" if error <= THRESHOLD else "continuing",
+    })
+
+    # Convergence check — emulate: while error > THRESHOLD: update error
+    if error <= THRESHOLD:
+        finalize_loop(loop)   # ensure the last iteration is closed before breaking
+        break

flowcept-0.9.3/examples/instrumented_loop_example.py

@@ -0,0 +1,77 @@
+import random
+from time import sleep
+
+from flowcept import Flowcept, FlowceptLoop
+
+# EXAMPLE 1:
+
+
+iterations = range(1, 5)
+
+with Flowcept():
+
+    loop = FlowceptLoop(iterations)         # See also: FlowceptLightweightLoop
+    for item in loop:
+        loss = random.random()
+        sleep(0.05)
+        print(item, loss)
+        # The following is optional, in case you want to capture values generated inside the loop.
+        loop.end_iter({"item": item, "loss": loss})
+
+docs = Flowcept.db.get_tasks_from_current_workflow()
+assert len(docs) == len(iterations)
+
+
+# EXAMPLE 2: WHILE LOOP EXAMPLE
+# The following code is equivalent to:
+# while layer_for_control_update <= num_layers - 1:
+#     ... loop code ...
+#    layer_for_control_update += 1
+
+layer_for_control_update = 2
+num_layers = 6
+
+
+def do_work(layer_ix: int) -> dict:
+    """Dummy per-iteration work that returns something to log."""
+    # simulate some computation
+    value = layer_ix * 10
+    return {"value": value}
+
+
+# Wrap the loop with FlowceptLoop
+iterations = range(layer_for_control_update, num_layers)
+
+# You can create with_flowcept flags
+with_flowcept = True
+
+if with_flowcept:
+    loop = FlowceptLoop(
+        items=iterations,
+        loop_name="control_loop",
+        item_name="layer_ix",
+        # parent_task_id="optional-parent-id",
+        # workflow_id="optional-workflow-id",
+    )
+    iterations = loop
+
+
+for layer_ix in iterations:
+    # Your loop body
+    result = do_work(layer_ix)
+    print(f"Processed layer {layer_ix}, result={result}")
+
+    if with_flowcept:
+        # Record per-iteration outputs
+        iterations.end_iter({
+            "layer_ix": layer_ix,
+            "status": "updated",
+            **result,                 # attach any metrics you computed
+        })
+
+if with_flowcept:
+    docs = Flowcept.db.get_tasks_from_current_workflow()
+    assert len(docs) == len(iterations)
+
+
+# EXAMPLE 3: WHILE LOOP EXAMPLE

flowcept-0.9.3/examples/mqtt_example.py

@@ -0,0 +1,101 @@
+# broker_minimal_example.py
+import json
+import time
+import uuid
+import paho.mqtt.client as mqtt
+
+from flowcept import Flowcept
+from flowcept.configs import settings
+
+
+def configure_flowcept_for_mqtt():
+    """
+    Mirror your settings.yaml exactly under adapters.broker_mqtt so the
+    MQTTBrokerInterceptor picks it up via self.settings.
+    """
+    settings.setdefault("adapters", {})
+    settings["adapters"]["broker_mqtt"] = {
+        "kind": "broker",
+        "host": "localhost",
+        "port": 30011,
+        "protocol": "mqtt3.1.1",
+        "queues": ["#"],
+        "username": "postman",
+        "password": "p",
+        "qos": 2,
+        "task_subtype": "intersect_msg",
+        "tracked_keys": {
+            "used": "payload",
+            "generated": None,          # ~ in YAML
+            "custom_metadata": ["headers", "msgId"],
+            "activity_id": "operationId",
+            "submitted_at": None,
+            "started_at": None,
+            "ended_at": None,
+            "registered_at": None,
+        },
+    }
+
+
+def publish_one_message(
+    host="localhost",
+    port=30011,
+    username="postman",
+    password="p",
+    topic="flowcept/demo/test",
+):
+    """
+    Publish a single JSON payload with the fields your tracked_keys expect:
+    - payload            -> mapped to Task.used
+    - headers, msgId     -> copied into Task.custom_metadata
+    - operationId        -> mapped to Task.activity_id
+    """
+    client = mqtt.Client(protocol=mqtt.MQTTv311)
+    client.username_pw_set(username, password)
+    client.connect(host, port, keepalive=60)
+
+    payload = {
+        "payload": {"greeting": "hello from mqtt"},   # becomes Task.used
+        "headers": {"x-source": "unit-test", "x-env": "local"},
+        "msgId": str(uuid.uuid4()),
+        "operationId": "op-xyz-123",                  # becomes Task.activity_id
+        # optional extras also okay
+        "status": "FINISHED",
+    }
+
+    client.publish(topic, json.dumps(payload), qos=2, retain=False)
+    client.disconnect()
+    return payload
+
+
+def main():
+    # 1) Apply settings that match your YAML
+    configure_flowcept_for_mqtt()
+
+    # 2) Start Flowcept in context for the broker_mqtt adapter
+    with Flowcept("broker_mqtt"):
+        # 3) Start the interceptor (runs MQTT loop in a background thread)
+        # Give it a moment to connect/subscribe to "#"
+        time.sleep(1.0)
+
+        # 4) Publish one message
+        sent = publish_one_message()
+        print("Published:", sent)
+
+        # 5) Allow time for ingest
+        time.sleep(2.0)
+
+    # 6) (Optional) Query Flowcept DB for the ingested task
+    try:
+        # Query by nested field in used (payload.greeting)
+        tasks = Flowcept.db.query(filter={"used.greeting": "hello from mqtt"})
+        if tasks:
+            print("Ingested task example:\n", json.dumps(tasks[0], indent=2))
+        else:
+            print("No tasks found for filter used.greeting == 'hello from mqtt'")
+    except Exception as e:
+        print(f"DB query failed/skipped: {e}")
+
+
+if __name__ == "__main__":
+    main()

{flowcept-0.9.2 → flowcept-0.9.3}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.9.2 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.3 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.

{flowcept-0.9.2 → flowcept-0.9.3}/src/flowcept/agents/flowcept_ctx_manager.py

@@ -53,7 +53,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         Current application context holding task state and QA components.
     msgs_counter : int
         Counter tracking how many task messages have been processed.
-    context_size : int
+    context_chunk_size : int
         Number of task messages to collect before triggering QA index building and LLM analysis.
     qa_manager : FlowceptQAManager
         Utility for constructing QA chains from task summaries.
@@ -64,7 +64,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         self.tracker_config = dict(max_examples=3, max_str_len=50)
         self.schema_tracker = DynamicSchemaTracker(**self.tracker_config)
         self.msgs_counter = 0
-        self.context_size = 1
+        self.context_chunk_size = 1  # Should be in the settings
         super().__init__()
 
     def message_handler(self, msg_obj: Dict):
@@ -98,18 +98,22 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
             if len(task_summary.get("tags", [])):
                 self.context.critical_tasks.append(task_summary)
 
-            if self.msgs_counter > 0 and self.msgs_counter % self.context_size == 0:
+            if self.msgs_counter > 0 and self.msgs_counter % self.context_chunk_size == 0:
                 self.logger.debug(
-                    f"Going to add to index! {(self.msgs_counter - self.context_size, self.msgs_counter)}"
+                    f"Going to add to index! {(self.msgs_counter - self.context_chunk_size, self.msgs_counter)}"
                 )
                 try:
                     self.update_schema_and_add_to_df(
-                        tasks=self.context.task_summaries[self.msgs_counter - self.context_size : self.msgs_counter]
+                        tasks=self.context.task_summaries[
+                            self.msgs_counter - self.context_chunk_size : self.msgs_counter
+                        ]
                     )
                 except Exception as e:
                     self.logger.error(
                         f"Could not add these tasks to buffer!\n"
-                        f"{self.context.task_summaries[self.msgs_counter - self.context_size : self.msgs_counter]}"
+                        f"{
+                            self.context.task_summaries[self.msgs_counter - self.context_chunk_size : self.msgs_counter]
+                        }"
                     )
                     self.logger.exception(e)
 

{flowcept-0.9.2 → flowcept-0.9.3}/src/flowcept/agents/gui/gui_utils.py

@@ -1,3 +1,5 @@
+import base64
+import ast
 import io
 import json
 
@@ -122,6 +124,53 @@ def display_ai_msg_from_tool(tool_result: ToolResult):
     return agent_reply
 
 
+def _sniff_mime(b: bytes) -> str:
+    if b.startswith(b"\x89PNG\r\n\x1a\n"):
+        return "image/png"
+    if b.startswith(b"\xff\xd8\xff"):
+        return "image/jpeg"
+    if b.startswith(b"GIF87a") or b.startswith(b"GIF89a"):
+        return "image/gif"
+    if b.startswith(b"BM"):
+        return "image/bmp"
+    if b.startswith(b"RIFF") and b[8:12] == b"WEBP":
+        return "image/webp"
+    return "application/octet-stream"
+
+
+def ensure_data_uri(val):
+    r"""Accepts bytes/bytearray/memoryview or a repr like \"b'\\x89PNG...'\" and returns a data URL."""
+    if isinstance(val, str) and val.startswith("data:"):
+        return val
+    if isinstance(val, str) and val.startswith("b'"):
+        try:
+            val = ast.literal_eval(val)  # turn repr into bytes
+        except Exception:
+            return None
+    if isinstance(val, memoryview):
+        val = val.tobytes()
+    if isinstance(val, bytearray):
+        val = bytes(val)
+    if isinstance(val, bytes):
+        mime = _sniff_mime(val)
+        return f"data:{mime};base64,{base64.b64encode(val).decode('ascii')}"
+    return val  # path/URL, etc.
+
+
+def _render_df(df: pd.DataFrame, image_width: int = 90, row_height: int = 90):
+    if "image" in df.columns:
+        df = df.copy()
+        df["image"] = df["image"].apply(ensure_data_uri)
+        st.dataframe(
+            df,
+            column_config={"image": st.column_config.ImageColumn("Preview", width=image_width)},
+            hide_index=True,
+            row_height=row_height,  # make thumbnails visible
+        )
+    else:
+        st.dataframe(df, hide_index=True)
+
+
 def display_df_tool_response(tool_result: ToolResult):
     r"""
     Display the DataFrame contained in a ToolResult.
@@ -170,7 +219,8 @@ def display_df_tool_response(tool_result: ToolResult):
             df = pd.read_csv(io.StringIO(result_df_str))
             print("The result is a df")
             if not df.empty:
-                st.dataframe(df, hide_index=False)
+                _render_df(df)
+
                 print("Columns", str(df.columns))
                 print("Number of columns", len(df.columns))
             else:
@@ -190,6 +240,7 @@ def display_df_tool_response(tool_result: ToolResult):
 
         if summary:
             st.markdown("📝 Summary:")
+            print(f"THIS IS THE SUMMARY\n{summary}")
             st.markdown(summary)
         elif summary_error:
             st.markdown(f"⚠️ Encountered this error when summarizing the result dataframe:\n```text\n{summary_error}")

{flowcept-0.9.2 → flowcept-0.9.3}/src/flowcept/agents/prompts/in_memory_query_prompts.py

@@ -15,6 +15,7 @@ COMMON_TASK_FIELDS = """
     | `ended_at`                    | datetime64[ns, UTC] | End time of a task. | 
     | `subtype`                     | string | Subtype of a task. |
     | `tags`                        | List[str] | List of descriptive tags. |
+    | `image`                        | blob | Raw binary data related to an image. |
     | `telemetry_summary.duration_sec` | float | Task duration (seconds). |
     | `telemetry_summary.cpu.percent_all_diff` | float | Difference in overall CPU utilization percentage across all cores between task end and start.|
     | `telemetry_summary.cpu.user_time_diff`   | float |  Difference average per core CPU user time ( seconds ) between task start and end times.|
@@ -27,6 +28,17 @@ COMMON_TASK_FIELDS = """
 DF_FORM = "The user has a pandas DataFrame called `df`, created from flattened task objects using `pd.json_normalize`."
 
 
+def get_example_values_prompt(example_values):
+    values_prompt = f"""    
+           Now, this other dictionary below provides type (t), up to 3 example values (v), and, for lists, shape (s) and element type (et) for each field.
+           Field names do not include `used.` or `generated.` They represent the unprefixed form shared across roles. String values may be truncated if they exceed the length limit.
+           ```python
+           {example_values}
+           ```
+       """
+    return values_prompt
+
+
 def get_df_schema_prompt(dynamic_schema, example_values):
     schema_prompt = f"""
      ## DATAFRAME STRUCTURE
@@ -52,14 +64,7 @@ def get_df_schema_prompt(dynamic_schema, example_values):
         ---
     """
 
-    values_prompt = f"""    
-        Now, this other dictionary below provides type (t), up to 3 example values (v), and, for lists, shape (s) and element type (et) for each field.
-        Field names do not include `used.` or `generated.` They represent the unprefixed form shared across roles. String values may be truncated if they exceed the length limit.
-        ```python
-        {example_values}
-        ```
-    """
-
+    values_prompt = get_example_values_prompt(example_values)
     # values_prompt = ""
     prompt = schema_prompt + values_prompt
     return prompt
@@ -221,7 +226,7 @@ def generate_pandas_code_prompt(query: str, dynamic_schema, example_values):
         f"{JOB}"
         f"{DF_FORM}"
         f"{get_df_schema_prompt(dynamic_schema, example_values)}"  # main tester
-        # f"{QUERY_GUIDELINES}" # main tester
+        f"{QUERY_GUIDELINES}"  # main tester
         f"{FEW_SHOTS}"  # main tester
         f"{OUTPUT_FORMATTING}"
         "User Query:"
@@ -230,9 +235,16 @@ def generate_pandas_code_prompt(query: str, dynamic_schema, example_values):
     return prompt
 
 
-def dataframe_summarizer_context(code, reduced_df, query) -> str:
+def dataframe_summarizer_context(code, reduced_df, dynamic_schema, example_values, query) -> str:
+    job = "You are a Workflow Provenance Specialist analyzing a DataFrame that was obtained to answer a query."
+
+    if "image" in reduced_df.columns:
+        reduced_df = reduced_df.drop(columns=["image"])
+
     prompt = f"""
-    You are a Workflow Provenance Specialist analyzing a DataFrame that was obtained to answer a query. Given:
+    {job}
+    
+     Given:
     
     **User Query**:  
     {query}
@@ -240,19 +252,26 @@ def dataframe_summarizer_context(code, reduced_df, query) -> str:
     **Query_Code**:  
     {code}
     
-    **Reduced DataFrame** (rows sampled from full result):  
+    **Reduced DataFrame `df` contents** (rows sampled from full result):  
     {reduced_df}
     
-    Your task is to:
-    1. Analyze the DataFrame values and columns for any meaningful or notable information.
-    2. Compare the query_code with the data content to understand what the result represents. THIS IS A REDUCED DATAFRAME, the original dataframe, used to answer the query, may be much bigger. IT IS ALREADY KNOWN! Do not need to restate this.
-    3. Provide a concise and direct answer to the user query. Your final response to the query should be  within ```text .
+    **Original df (before reduction) had this schema:
+    {get_df_schema_prompt(dynamic_schema, example_values)}
+    
+    Your task is to find a concise and direct answer as an English sentence to the user query.
+        
+    Only if the answer to the query is complex, provide more explanation by: 
+        1. Analyzing the DataFrame values and columns for any meaningful or notable information. 
+        2. Comparing the query_code with the data content to understand what the result represents. THIS IS A REDUCED DATAFRAME, the original dataframe, used to answer the query, may be much bigger. IT IS ALREADY KNOWN! Do not need to restate this.
+        3. If it makes sense, provide information beyond the recorded provenance, but state it clearly that you are inferring it.
+    
+    In the end, conclude by giving your concise answer as follows: **Response**: <YOUR ANSWER>
 
     Note that the user should not know that this is a reduced dataframe. 
-    
     Keep your response short and focused.
 
     """
+
     return prompt
 
 

{flowcept-0.9.2 → flowcept-0.9.3}/src/flowcept/agents/tools/general_tools.py

@@ -74,7 +74,7 @@ def prompt_handler(message: str) -> ToolResult:
     TextContent
         The AI response or routing feedback.
     """
-    df_key_words = {"save", "result = df", "reset context"}
+    df_key_words = ["df", "save", "result = df", "reset context"]
     for key in df_key_words:
         if key in message:
             return run_df_query(llm=None, query=message, plot=False)

{flowcept-0.9.2 → flowcept-0.9.3}/src/flowcept/agents/tools/in_memory_queries/in_memory_queries_tools.py

@@ -294,6 +294,8 @@ def generate_result_df(llm, query: str, dynamic_schema, example_values, df, atte
     >>> generate_result_df(llm, "bad query", schema, examples, df, attempt_fix=False)
     ToolResult(code=405, result="Failed to parse this as Python code: ...")
     """
+    if llm is None:
+        llm = build_llm_model()
     try:
         prompt = generate_pandas_code_prompt(query, dynamic_schema, example_values)
         response = llm(prompt)
@@ -351,7 +353,14 @@ def generate_result_df(llm, query: str, dynamic_schema, example_values, df, atte
     summary, summary_error = None, None
     if summarize:
         try:
-            tool_result = summarize_result(llm, result_code, result_df, query)
+            tool_result = summarize_result(
+                llm,
+                result_code,
+                result_df,
+                query,
+                dynamic_schema,
+                example_values,
+            )
             if tool_result.is_success():
                 return_code = 301
                 summary = tool_result.result
@@ -570,7 +579,14 @@ def extract_or_fix_json_code(llm, raw_text) -> ToolResult:
 
 
 @mcp_flowcept.tool()
-def summarize_result(llm, code, result, query: str) -> ToolResult:
+def summarize_result(
+    llm,
+    code,
+    result,
+    query: str,
+    dynamic_schema,
+    example_values,
+) -> ToolResult:
     """
     Summarize the pandas result with local reduction for large DataFrames.
     - For wide DataFrames, selects top columns based on variance and uniqueness.
@@ -578,7 +594,7 @@ def summarize_result(llm, code, result, query: str) -> ToolResult:
     - Constructs a detailed prompt for the LLM with original column context.
     """
     summarized_df = summarize_df(result, code)
-    prompt = dataframe_summarizer_context(code, summarized_df, query)
+    prompt = dataframe_summarizer_context(code, summarized_df, dynamic_schema, example_values, query)
     try:
         response = llm(prompt)
         return ToolResult(code=201, result=response)

{flowcept-0.9.2 → flowcept-0.9.3}/src/flowcept/commons/task_data_preprocess.py

@@ -137,7 +137,6 @@ def summarize_task(task: Dict, thresholds: Dict = None, logger=None) -> Dict:
         "agent_id",
         "campaign_id",
         "subtype",
-        "custom_metadata",
     ]:
         value = _safe_get(task, key)
         if value is not None:
@@ -146,7 +145,14 @@ def summarize_task(task: Dict, thresholds: Dict = None, logger=None) -> Dict:
             else:
                 task_summary[key] = value
 
-        # Special handling for timestamp field
+    # Adding image column if data is image. This is to handle special cases when there is an image associated to
+    # a provenance task.
+    if "data" in task:
+        if "custom_metadata" in task:
+            if "image" in task["custom_metadata"].get("mime_type", ""):
+                task_summary["image"] = task["data"]
+
+    # Special handling for timestamp field
     try:
         time_keys = ["started_at", "ended_at"]
         for time_key in time_keys:

{flowcept-0.9.2 → flowcept-0.9.3}/src/flowcept/version.py

@@ -4,4 +4,4 @@
 # The expected format is: <Major>.<Minor>.<Patch>
 # This file is supposed to be automatically modified by the CI Bot.
 # See .github/workflows/version_bumper.py
-__version__ = "0.9.2"
+__version__ = "0.9.3"

flowcept-0.9.2/examples/instrumented_loop_example.py

@@ -1,19 +0,0 @@
-import random
-from time import sleep
-
-from flowcept import Flowcept, FlowceptLoop
-
-iterations = range(1, 5)
-
-with Flowcept():
-
-    loop = FlowceptLoop(iterations)         # See also: FlowceptLightweightLoop
-    for item in loop:
-        loss = random.random()
-        sleep(0.05)
-        print(item, loss)
-        # The following is optional, in case you want to capture values generated inside the loop.
-        loop.end_iter({"item": item, "loss": loss})
-
-docs = Flowcept.db.get_tasks_from_current_workflow()
-assert len(docs) == len(iterations)