flowcept 0.9.17__tar.gz → 0.9.18__tar.gz

{flowcept-0.9.17 → flowcept-0.9.18}/.gitignore

@@ -34,3 +34,4 @@ core.*
 
 uv.lock
 examples/flowcept_messages.jsonl
+agents.md

{flowcept-0.9.17 → flowcept-0.9.18}/Makefile

@@ -115,7 +115,7 @@ services-stop-mofka:
 # Run unit tests using pytest
 .PHONY: tests
 tests:
-	pytest --ignore=tests/adapters/test_tensorboard.py
+	pytest --timeout=300 --ignore=tests/adapters/test_tensorboard.py
 
 .PHONY: tests-notebooks
 tests-notebooks:

{flowcept-0.9.17 → flowcept-0.9.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.17
+Version: 0.9.18
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT
@@ -41,6 +41,7 @@ Requires-Dist: pyarrow; extra == 'all'
 Requires-Dist: pymongo; extra == 'all'
 Requires-Dist: pymupdf; extra == 'all'
 Requires-Dist: pytest; extra == 'all'
+Requires-Dist: pytest-timeout; extra == 'all'
 Requires-Dist: pyyaml; extra == 'all'
 Requires-Dist: redis; extra == 'all'
 Requires-Dist: requests; extra == 'all'
@@ -50,6 +51,7 @@ Requires-Dist: seaborn; extra == 'all'
 Requires-Dist: sphinx; extra == 'all'
 Requires-Dist: sqlalchemy; extra == 'all'
 Requires-Dist: streamlit; extra == 'all'
+Requires-Dist: tabulate; extra == 'all'
 Requires-Dist: tbparse; extra == 'all'
 Requires-Dist: tensorboard; extra == 'all'
 Requires-Dist: tensorflow; extra == 'all'
@@ -69,6 +71,7 @@ Requires-Dist: jupyterlab; extra == 'dev'
 Requires-Dist: nbmake; extra == 'dev'
 Requires-Dist: pika; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-timeout; extra == 'dev'
 Requires-Dist: pyyaml; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
 Requires-Dist: sphinx; extra == 'dev'
@@ -94,6 +97,7 @@ Requires-Dist: matplotlib; extra == 'llm-agent'
 Requires-Dist: mcp[cli]; extra == 'llm-agent'
 Requires-Dist: pymupdf; extra == 'llm-agent'
 Requires-Dist: streamlit; extra == 'llm-agent'
+Requires-Dist: tabulate; extra == 'llm-agent'
 Provides-Extra: llm-agent-audio
 Requires-Dist: gtts; extra == 'llm-agent-audio'
 Requires-Dist: langchain-community; extra == 'llm-agent-audio'
@@ -105,6 +109,7 @@ Requires-Dist: pymupdf; extra == 'llm-agent-audio'
 Requires-Dist: speechrecognition; extra == 'llm-agent-audio'
 Requires-Dist: streamlit; extra == 'llm-agent-audio'
 Requires-Dist: streamlit-mic-recorder; extra == 'llm-agent-audio'
+Requires-Dist: tabulate; extra == 'llm-agent-audio'
 Provides-Extra: llm-google
 Requires-Dist: google-genai; extra == 'llm-google'
 Requires-Dist: langchain-community; extra == 'llm-google'
@@ -113,6 +118,7 @@ Requires-Dist: matplotlib; extra == 'llm-google'
 Requires-Dist: mcp[cli]; extra == 'llm-google'
 Requires-Dist: pymupdf; extra == 'llm-google'
 Requires-Dist: streamlit; extra == 'llm-google'
+Requires-Dist: tabulate; extra == 'llm-google'
 Provides-Extra: lmdb
 Requires-Dist: lmdb; extra == 'lmdb'
 Provides-Extra: ml-dev
@@ -180,6 +186,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 <h4 align="center">
+  <a href="https://flowcept.org">Website</a> &#8226;
   <a href="https://flowcept.readthedocs.io/">Documentation</a> &#8226; 
   <a href="./docs/publications">Publications</a>
 </h4>

{flowcept-0.9.17 → flowcept-0.9.18}/README.md

@@ -31,6 +31,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 <h4 align="center">
+  <a href="https://flowcept.org">Website</a> &#8226;
   <a href="https://flowcept.readthedocs.io/">Documentation</a> &#8226; 
   <a href="./docs/publications">Publications</a>
 </h4>

{flowcept-0.9.17 → flowcept-0.9.18}/pyproject.toml

@@ -65,7 +65,7 @@ mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog", "cryptography"]
 nvidia = ["nvidia-ml-py"]
 mqtt = ["paho-mqtt"]
 tensorboard = ["tensorboard", "tensorflow", "tbparse"]
-llm_agent = ["mcp[cli]", "langchain_community", "langchain_openai", "streamlit", "PyMuPDF", "matplotlib"]
+llm_agent = ["mcp[cli]", "langchain_community", "langchain_openai", "streamlit", "PyMuPDF", "matplotlib", "tabulate"]
 llm_google = ["flowcept[llm_agent]", "google-genai"]
 llm_agent_audio = ["flowcept[llm_agent]", "streamlit-mic-recorder", "SpeechRecognition", "pydub", "gTTS"]
 # System dependency (required for pydub)
@@ -80,8 +80,9 @@ dev = [
     "nbmake",
     "pika",
     "pytest",
+    "pytest-timeout",
     "ruff",
-    "pyyaml",
+    "pyyaml"
 ]
 # Torch and some other ml-specific libs, only used for dev purposes, require the following specific versions.
 ml_dev = [

{flowcept-0.9.17 → flowcept-0.9.18}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.9.17 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.18 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
@@ -94,6 +94,7 @@ agent:
   service_provider: '?'
   model_kwargs: {}
   audio_enabled: false
+  debug: true
 
 databases:
 

{flowcept-0.9.17 → flowcept-0.9.18}/src/flowcept/agents/agents_utils.py

@@ -1,4 +1,6 @@
 import os
+import re
+import unicodedata
 from typing import Union, Dict
 
 from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
@@ -194,3 +196,43 @@ def build_llm_model(
             if tool_task:
                 llm.parent_task_id = tool_task.task_id
     return llm
+
+
+def normalize_message(user_msg: str) -> str:
+    """
+    Normalize a user message into a canonical, comparison-friendly form.
+
+    The function standardizes text by trimming whitespace, applying Unicode
+    normalization, normalizing dash characters, collapsing repeated whitespace,
+    removing trailing punctuation that does not affect semantics, and converting
+    the result to lowercase.
+
+    Parameters
+    ----------
+    user_msg : str
+        Raw user input message.
+
+    Returns
+    -------
+    str
+        Normalized message suitable for matching, comparison, or hashing.
+    """
+    # 1) Strip leading/trailing whitespace
+    user_msg = user_msg.strip()
+
+    # 2) Unicode normalize to avoid weird characters (like fancy quotes, dashes)
+    user_msg = unicodedata.normalize("NFKC", user_msg)
+
+    # 3) Normalize dashes commonly used in chemistry (C–H, C—H, etc.)
+    user_msg = user_msg.replace("–", "-").replace("—", "-")
+
+    # 4) Collapse multiple spaces / newlines into a single space
+    user_msg = re.sub(r"\s+", " ", user_msg)
+
+    # 5) Remove trailing punctuation that doesn't change semantics
+    #    e.g., "?", "!", "." at the VERY end
+    user_msg = re.sub(r"[?!.\s]+$", "", user_msg)
+
+    user_msg = user_msg.lower()
+
+    return user_msg

{flowcept-0.9.17 → flowcept-0.9.18}/src/flowcept/agents/flowcept_agent.py

@@ -20,11 +20,14 @@ def main():
     def run():
         uvicorn.run(mcp_flowcept.streamable_http_app, host=AGENT_HOST, port=AGENT_PORT, lifespan="on")
 
-    Thread(target=run).start()
+    server_thread = Thread(target=run, daemon=False)
+    server_thread.start()
     sleep(2)
     # Wake up tool call
     print(run_tool(check_liveness, host=AGENT_HOST, port=AGENT_PORT)[0])
 
+    server_thread.join()
+
 
 if __name__ == "__main__":
     main()

{flowcept-0.9.17 → flowcept-0.9.18}/src/flowcept/agents/flowcept_ctx_manager.py

@@ -1,6 +1,9 @@
 from flowcept.agents.dynamic_schema_tracker import DynamicSchemaTracker
 from flowcept.agents.tools.in_memory_queries.pandas_agent_utils import load_saved_df
 from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
+from flowcept.commons.flowcept_logger import FlowceptLogger
+from flowcept.commons.vocabulary import Status
+from flowcept.configs import AGENT
 from mcp.server.fastmcp import FastMCP
 
 import json
@@ -12,11 +15,12 @@ import pandas as pd
 
 from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager, BaseAppContext
 
-
-from flowcept.agents import agent_client
 from flowcept.commons.task_data_preprocess import summarize_task
 
 
+AGENT_DEBUG = AGENT.get("debug", False)
+
+
 @dataclass
 class FlowceptAppContext(BaseAppContext):
     """
@@ -39,6 +43,39 @@ class FlowceptAppContext(BaseAppContext):
     tracker_config: Dict | None
     custom_guidance: List[str] | None
 
+    def __init__(self):
+        self.logger = FlowceptLogger()
+        self.reset_context()
+
+    def reset_context(self):
+        """
+        Reset the agent's context to a clean state, initializing a new QA setup.
+        """
+        self.tasks = []
+        self.task_summaries = []
+        self.critical_tasks = []
+        self.df = pd.DataFrame()
+        self.tasks_schema = {}
+        self.value_examples = {}
+        self.custom_guidance = []
+        self.tracker_config = {}
+
+        if AGENT_DEBUG:
+            from flowcept.commons.flowcept_logger import FlowceptLogger
+
+            FlowceptLogger().warning("Running agent in DEBUG mode!")
+            df_path = "/tmp/current_agent_df.csv"
+            if os.path.exists(df_path):
+                self.logger.warning("Going to load df into context")
+                df = load_saved_df(df_path)
+                self.df = df
+            if os.path.exists("/tmp/current_tasks_schema.json"):
+                with open("/tmp/current_tasks_schema.json") as f:
+                    self.tasks_schema = json.load(f)
+            if os.path.exists("/tmp/value_examples.json"):
+                with open("/tmp/value_examples.json") as f:
+                    self.value_examples = json.load(f)
+
 
 class FlowceptAgentContextManager(BaseAgentContextManager):
     """
@@ -61,7 +98,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
     """
 
     def __init__(self):
-        self.context: FlowceptAppContext = None
+        self.context = FlowceptAppContext()
         self.tracker_config = dict(max_examples=3, max_str_len=50)
         self.schema_tracker = DynamicSchemaTracker(**self.tracker_config)
         self.msgs_counter = 0
@@ -82,7 +119,6 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         bool
             True if the message was handled successfully.
         """
-        print("Received:", msg_obj)
         msg_type = msg_obj.get("type", None)
         if msg_type == "task":
             task_msg = TaskObject.from_dict(msg_obj)
@@ -90,8 +126,62 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
                 self.logger.info(f"Going to ignore our own LLM messages: {task_msg}")
                 return True
 
-            self.msgs_counter += 1
             self.logger.debug("Received task msg!")
+            if task_msg.subtype == "call_agent_task":
+                from flowcept.instrumentation.task_capture import FlowceptTask
+
+                if task_msg.activity_id == "reset_user_context":
+                    self.context.reset_context()
+                    self.msgs_counter = 0
+                    FlowceptTask(
+                        agent_id=self.agent_id,
+                        generated={"msg": "Provenance Agent reset context."},
+                        subtype="agent_task",
+                        activity_id="reset_user_context",
+                    ).send()
+                    return True
+                elif task_msg.activity_id == "provenance_query":
+                    self.logger.info("Received a prov query message!")
+                    query_text = task_msg.used.get("query")
+                    from flowcept.agents import ToolResult
+                    from flowcept.agents.tools.general_tools import prompt_handler
+                    from flowcept.agents.agent_client import run_tool
+
+                    resp = run_tool(tool_name=prompt_handler, kwargs={"message": query_text})[0]
+
+                    try:
+                        error = None
+                        status = Status.FINISHED
+                        tool_result = ToolResult(**json.loads(resp))
+                        if tool_result.result_is_str():
+                            generated = {"text": tool_result.result}
+                        else:
+                            generated = tool_result.result
+                    except Exception as e:
+                        status = Status.ERROR
+                        error = f"Could not convert the following into a ToolResult:\n{resp}\nException: {e}"
+                        generated = {"text": str(resp)}
+                    FlowceptTask(
+                        agent_id=self.agent_id,
+                        generated=generated,
+                        stderr=error,
+                        status=status,
+                        subtype="agent_task",
+                        activity_id="provenance_query_response",
+                    ).send()
+
+                    return True
+
+            elif (
+                task_msg.subtype == "agent_task"
+                and task_msg.agent_id is not None
+                and task_msg.agent_id == self.agent_id
+            ):
+                self.logger.info(f"Ignoring agent tasks from myself: {task_msg}")
+                return True
+
+            self.msgs_counter += 1
+
             self.context.tasks.append(msg_obj)
 
             task_summary = summarize_task(msg_obj, logger=self.logger)
@@ -136,7 +226,9 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         Perform LLM-based analysis on the current chunk of task messages and send the results.
         """
         self.logger.debug(f"Going to begin LLM job! {self.msgs_counter}")
-        result = agent_client.run_tool("analyze_task_chunk")
+        from flowcept.agents.agent_client import run_tool
+
+        result = run_tool("analyze_task_chunk")
         if len(result):
             content = result[0].text
             if content != "Error executing tool":
@@ -146,36 +238,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
             else:
                 self.logger.error(content)
 
-    def reset_context(self):
-        """
-        Reset the agent's context to a clean state, initializing a new QA setup.
-        """
-        self.context = FlowceptAppContext(
-            tasks=[],
-            task_summaries=[],
-            critical_tasks=[],
-            df=pd.DataFrame(),
-            tasks_schema={},
-            value_examples={},
-            custom_guidance=[],
-            tracker_config=self.tracker_config,
-        )
-        DEBUG = True  # TODO debugging!
-        if DEBUG:
-            self.logger.warning("Running agent in DEBUG mode!")
-            df_path = "/tmp/current_agent_df.csv"
-            if os.path.exists(df_path):
-                self.logger.warning("Going to load df into context")
-                df = load_saved_df(df_path)
-                self.context.df = df
-            if os.path.exists("/tmp/current_tasks_schema.json"):
-                with open("/tmp/current_tasks_schema.json") as f:
-                    self.context.tasks_schema = json.load(f)
-            if os.path.exists("/tmp/value_examples.json"):
-                with open("/tmp/value_examples.json") as f:
-                    self.context.value_examples = json.load(f)
-
 
 # Exporting the ctx_manager and the mcp_flowcept
 ctx_manager = FlowceptAgentContextManager()
-mcp_flowcept = FastMCP("FlowceptAgent", require_session=False, lifespan=ctx_manager.lifespan, stateless_http=True)
+mcp_flowcept = FastMCP("FlowceptAgent", lifespan=ctx_manager.lifespan, stateless_http=True)

{flowcept-0.9.17 → flowcept-0.9.18}/src/flowcept/agents/gui/gui_utils.py

@@ -351,10 +351,28 @@ def exec_st_plot_code(code, result_df, st_module):
     >>> code = "st.line_chart(result)"
     >>> exec_st_plot_code(code, df, st)
     """
-    print("Plot code \n", code)
+    # 1) Make a copy of result_df and rename columns with dots
+    plot_df = result_df.copy()
+    col_map = {}
+
+    for col in plot_df.columns:
+        if "." in col:
+            new_col = col.replace(".", "_")
+            col_map[col] = new_col
+            plot_df.rename(columns={col: new_col}, inplace=True)
+
+    # 2) Rewrite the code so column names match the renamed columns
+    sanitized_code = code
+    for old, new in col_map.items():
+        # replace only inside quotes: 'generated.bd_enthalpy' → 'generated_bd_enthalpy'
+        sanitized_code = sanitized_code.replace(f"'{old}'", f"'{new}'")
+        sanitized_code = sanitized_code.replace(f'"{old}"', f'"{new}"')
+
+    print("SANITIZED CODE:\n", sanitized_code)
+    print(f"Renamed DF columms: {plot_df}")
     exec(
-        code,
-        {"result": result_df, "st": st_module, "plt": __import__("matplotlib.pyplot"), "alt": __import__("altair")},
+        sanitized_code,
+        {"result": plot_df, "st": st_module, "plt": __import__("matplotlib.pyplot"), "alt": __import__("altair")},
     )
 
 

{flowcept-0.9.17 → flowcept-0.9.18}/src/flowcept/agents/prompts/general_prompts.py

@@ -28,7 +28,7 @@ ROUTING_PROMPT = (
     # "- in_context_query: if the user asks questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
     # "- historical_prov_query: if the user wants to query historical provenance data\n"
     "- in_chat_query: if the user appears to be asking about something that has said recently in this chat.\n"
-    "- unknown: if you don't know.\n"
+    "- in_context_query: if you don't know.\n"
     "Respond with only the route label."
     "User message is below:\n "
 )