flowcept 0.9.17__tar.gz → 0.9.19__tar.gz

{flowcept-0.9.17 → flowcept-0.9.19}/.gitignore

@@ -34,3 +34,4 @@ core.*
 
 uv.lock
 examples/flowcept_messages.jsonl
+agents.md

{flowcept-0.9.17 → flowcept-0.9.19}/Makefile

@@ -77,13 +77,13 @@ run:
 	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_HOST=flowcept_mongo --network flowcept_default -it flowcept
 
 tests-in-container-mongo:
-	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_HOST=flowcept_mongo -e MONGO_ENABLED=true -e LMDB_ENABLED=false --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest --ignore=tests/instrumentation_tests/ml_tests
+	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_HOST=flowcept_mongo -e MONGO_ENABLED=true -e LMDB_ENABLED=false --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest tests --ignore=tests/instrumentation_tests/ml_tests
 
 tests-in-container:
-	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_ENABLED=false -e LMDB_ENABLED=true --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest --ignore=tests/instrumentation_tests/ml_tests
+	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=flowcept_redis -e MONGO_ENABLED=false -e LMDB_ENABLED=true --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest tests --ignore=tests/instrumentation_tests/ml_tests
 
 tests-in-container-kafka:
-	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=kafka -e MONGO_HOST=flowcept_mongo  -e MQ_PORT=29092 -e MQ_TYPE=kafka -e MONGO_ENABLED=true -e LMDB_ENABLED=false --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest --ignore=tests/instrumentation_tests/ml_tests
+	docker run --rm -v $(shell pwd):/flowcept -e KVDB_HOST=flowcept_redis -e MQ_HOST=kafka -e MONGO_HOST=flowcept_mongo  -e MQ_PORT=29092 -e MQ_TYPE=kafka -e MONGO_ENABLED=true -e LMDB_ENABLED=false --network flowcept_default flowcept /opt/conda/envs/flowcept/bin/pytest tests --ignore=tests/instrumentation_tests/ml_tests
 
 # This command can be removed once we have our CLI
 liveness:
@@ -115,7 +115,7 @@ services-stop-mofka:
 # Run unit tests using pytest
 .PHONY: tests
 tests:
-	pytest --ignore=tests/adapters/test_tensorboard.py
+	pytest --timeout=300 --ignore=tests/adapters/test_tensorboard.py
 
 .PHONY: tests-notebooks
 tests-notebooks:

{flowcept-0.9.17 → flowcept-0.9.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.17
+Version: 0.9.19
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT
@@ -41,6 +41,7 @@ Requires-Dist: pyarrow; extra == 'all'
 Requires-Dist: pymongo; extra == 'all'
 Requires-Dist: pymupdf; extra == 'all'
 Requires-Dist: pytest; extra == 'all'
+Requires-Dist: pytest-timeout; extra == 'all'
 Requires-Dist: pyyaml; extra == 'all'
 Requires-Dist: redis; extra == 'all'
 Requires-Dist: requests; extra == 'all'
@@ -50,6 +51,7 @@ Requires-Dist: seaborn; extra == 'all'
 Requires-Dist: sphinx; extra == 'all'
 Requires-Dist: sqlalchemy; extra == 'all'
 Requires-Dist: streamlit; extra == 'all'
+Requires-Dist: tabulate; extra == 'all'
 Requires-Dist: tbparse; extra == 'all'
 Requires-Dist: tensorboard; extra == 'all'
 Requires-Dist: tensorflow; extra == 'all'
@@ -69,6 +71,7 @@ Requires-Dist: jupyterlab; extra == 'dev'
 Requires-Dist: nbmake; extra == 'dev'
 Requires-Dist: pika; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-timeout; extra == 'dev'
 Requires-Dist: pyyaml; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
 Requires-Dist: sphinx; extra == 'dev'
@@ -94,6 +97,7 @@ Requires-Dist: matplotlib; extra == 'llm-agent'
 Requires-Dist: mcp[cli]; extra == 'llm-agent'
 Requires-Dist: pymupdf; extra == 'llm-agent'
 Requires-Dist: streamlit; extra == 'llm-agent'
+Requires-Dist: tabulate; extra == 'llm-agent'
 Provides-Extra: llm-agent-audio
 Requires-Dist: gtts; extra == 'llm-agent-audio'
 Requires-Dist: langchain-community; extra == 'llm-agent-audio'
@@ -105,6 +109,7 @@ Requires-Dist: pymupdf; extra == 'llm-agent-audio'
 Requires-Dist: speechrecognition; extra == 'llm-agent-audio'
 Requires-Dist: streamlit; extra == 'llm-agent-audio'
 Requires-Dist: streamlit-mic-recorder; extra == 'llm-agent-audio'
+Requires-Dist: tabulate; extra == 'llm-agent-audio'
 Provides-Extra: llm-google
 Requires-Dist: google-genai; extra == 'llm-google'
 Requires-Dist: langchain-community; extra == 'llm-google'
@@ -113,6 +118,7 @@ Requires-Dist: matplotlib; extra == 'llm-google'
 Requires-Dist: mcp[cli]; extra == 'llm-google'
 Requires-Dist: pymupdf; extra == 'llm-google'
 Requires-Dist: streamlit; extra == 'llm-google'
+Requires-Dist: tabulate; extra == 'llm-google'
 Provides-Extra: lmdb
 Requires-Dist: lmdb; extra == 'lmdb'
 Provides-Extra: ml-dev
@@ -180,6 +186,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 <h4 align="center">
+  <a href="https://flowcept.org">Website</a> &#8226;
   <a href="https://flowcept.readthedocs.io/">Documentation</a> &#8226; 
   <a href="./docs/publications">Publications</a>
 </h4>

{flowcept-0.9.17 → flowcept-0.9.19}/README.md

@@ -31,6 +31,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 <h4 align="center">
+  <a href="https://flowcept.org">Website</a> &#8226;
   <a href="https://flowcept.readthedocs.io/">Documentation</a> &#8226; 
   <a href="./docs/publications">Publications</a>
 </h4>

{flowcept-0.9.17 → flowcept-0.9.19}/deployment/Dockerfile

@@ -1,9 +1,6 @@
 # Use the command `make build` to build this image.
-FROM miniconda:local
-
-RUN apt-get update && \
-    apt-get install -y vim curl wget make patch gcc \
-    && rm -rf /var/lib/apt/lists/*
+# Use conda-forge base image consistently across local/CI.
+FROM condaforge/miniforge3:23.11.0-0
 
 WORKDIR /flowcept
 

flowcept-0.9.19/docs/agent.rst

@@ -0,0 +1,43 @@
+Flowcept Agent
+==============
+
+The Flowcept Agent is an MCP-powered LLM interface for querying provenance data. It exposes a small set of tools
+(e.g., ``prompt_handler``) that route natural-language questions to in-memory queries over captured task summaries.
+
+Online-first design
+------------------
+Like Flowcept as a whole, the agent is designed to run **while a workflow is still executing**. In online mode,
+it consumes messages from the MQ (typically Redis) so it can respond to queries in near real time. This is the
+recommended setup for interactive RAG/MCP analysis during live runs.
+
+Offline (file-based) queries
+----------------------------
+For simple tests or disconnected environments, the agent can also be initialized from a **JSONL buffer file**.
+In this mode, Flowcept writes messages to disk (``dump_buffer``), and the agent loads the file once at startup
+before serving queries.
+
+This is a minimal offline example:
+
+.. code-block:: python
+
+   import json
+   from flowcept import Flowcept, flowcept_task
+   from flowcept.agents.flowcept_agent import FlowceptAgent
+
+   @flowcept_task
+   def sum_one(x):
+       return x + 1
+
+   # Run a small workflow and dump the buffer to disk
+   with Flowcept(start_persistence=False, save_workflow=False, check_safe_stops=False) as f:
+       sum_one(1)
+       f.dump_buffer("flowcept_buffer.jsonl")
+
+   # Start the agent from the buffer file and query it
+   agent = FlowceptAgent(buffer_path="flowcept_buffer.jsonl")
+   agent.start()
+   resp = agent.query("how many tasks?")
+   print(json.loads(resp))
+   agent.stop()
+
+In the future, this page will include a full **online** example (live MQ + Redis) and deployment guidance.

{flowcept-0.9.17 → flowcept-0.9.19}/docs/index.rst

@@ -57,6 +57,7 @@ Flowcept
    quick_start
    architecture
    setup
+   agent
    prov_capture
    telemetry_capture
    prov_storage

{flowcept-0.9.17 → flowcept-0.9.19}/docs/prov_capture.rst

@@ -124,7 +124,7 @@ Optional Arguments
 
 When creating a ``Flowcept`` instance (with or without a context manager), you can pass:
 
-- **interceptors**: list of interceptors (e.g., ``"instrumentation"``, ``"dask"``, ``"mlflow"``). Defaults to ``["instrumentation"]`` if enabled.
+- **interceptors**: list of interceptors (e.g., ``"instrumentation"``, ``"dask"``, ``"mlflow"``). Defaults to ``["instrumentation"]`` if enabled. Instrumentation defaults to enabled unless explicitly set to ``false`` in settings.
 - **bundle_exec_id**: identifier for grouping interceptors. Defaults to ``id(self)``.
 - **campaign_id**: unique identifier for the campaign. Defaults to a generated UUID.
 - **workflow_id**: unique identifier for the workflow. Defaults to a generated UUID.

{flowcept-0.9.17 → flowcept-0.9.19}/pyproject.toml

@@ -65,7 +65,7 @@ mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog", "cryptography"]
 nvidia = ["nvidia-ml-py"]
 mqtt = ["paho-mqtt"]
 tensorboard = ["tensorboard", "tensorflow", "tbparse"]
-llm_agent = ["mcp[cli]", "langchain_community", "langchain_openai", "streamlit", "PyMuPDF", "matplotlib"]
+llm_agent = ["mcp[cli]", "langchain_community", "langchain_openai", "streamlit", "PyMuPDF", "matplotlib", "tabulate"]
 llm_google = ["flowcept[llm_agent]", "google-genai"]
 llm_agent_audio = ["flowcept[llm_agent]", "streamlit-mic-recorder", "SpeechRecognition", "pydub", "gTTS"]
 # System dependency (required for pydub)
@@ -80,8 +80,9 @@ dev = [
     "nbmake",
     "pika",
     "pytest",
+    "pytest-timeout",
     "ruff",
-    "pyyaml",
+    "pyyaml"
 ]
 # Torch and some other ml-specific libs, only used for dev purposes, require the following specific versions.
 ml_dev = [
@@ -129,5 +130,12 @@ packages = ["src/flowcept"]
 [tool.hatch.build.targets.wheel.force-include]
 "resources/sample_settings.yaml" = "resources/sample_settings.yaml"
 
+[tool.pytest.ini_options]
+filterwarnings = [
+    "ignore:websockets\\.legacy is deprecated:DeprecationWarning",
+    "ignore:websockets\\.server\\.WebSocketServerProtocol is deprecated:DeprecationWarning",
+    "ignore:Use `streamable_http_client` instead\\.:DeprecationWarning",
+]
+
 [project.scripts]
-flowcept = "flowcept.cli:main"
+flowcept = "flowcept.cli:main"

{flowcept-0.9.17 → flowcept-0.9.19}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.9.17 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.19 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
@@ -94,6 +94,7 @@ agent:
   service_provider: '?'
   model_kwargs: {}
   audio_enabled: false
+  debug: true
 
 databases:
 

{flowcept-0.9.17 → flowcept-0.9.19}/src/flowcept/agents/agent_client.py

@@ -1,4 +1,6 @@
 import asyncio
+import json
+import re
 from typing import Dict, List, Callable
 
 from flowcept.configs import AGENT_HOST, AGENT_PORT
@@ -48,10 +50,14 @@ def run_tool(
                 result: List[TextContent] = await session.call_tool(tool_name, arguments=kwargs)
                 actual_result = []
                 for r in result.content:
-                    if isinstance(r, str):
-                        actual_result.append(r)
-                    else:
-                        actual_result.append(r.text)
+                    text = r if isinstance(r, str) else r.text
+                    try:
+                        json.loads(text)
+                        actual_result.append(text)
+                    except Exception:
+                        match = re.search(r"Error code:\\s*(\\d+)", text)
+                        code = int(match.group(1)) if match else 400
+                        actual_result.append(json.dumps({"code": code, "result": text, "tool_name": tool_name}))
 
                 return actual_result
 

{flowcept-0.9.17 → flowcept-0.9.19}/src/flowcept/agents/agents_utils.py

@@ -1,4 +1,6 @@
 import os
+import re
+import unicodedata
 from typing import Union, Dict
 
 from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
@@ -137,8 +139,8 @@ def build_llm_model(
     if _service_provider == "sambanova":
         from langchain_community.llms.sambanova import SambaStudio
 
-        os.environ["SAMBASTUDIO_URL"] = AGENT.get("llm_server_url")
-        os.environ["SAMBASTUDIO_API_KEY"] = AGENT.get("api_key")
+        os.environ["SAMBASTUDIO_URL"] = os.environ.get("SAMBASTUDIO_URL", AGENT.get("llm_server_url"))
+        os.environ["SAMBASTUDIO_API_KEY"] = os.environ.get("SAMBASTUDIO_API_KEY", AGENT.get("api_key"))
 
         llm = SambaStudio(model_kwargs=_model_kwargs)
     elif _service_provider == "azure":
@@ -153,7 +155,16 @@ def build_llm_model(
         from langchain_openai import ChatOpenAI
 
         api_key = os.environ.get("OPENAI_API_KEY", AGENT.get("api_key", None))
-        llm = ChatOpenAI(openai_api_key=api_key, **model_kwargs)
+        base_url = os.environ.get("OPENAI_BASE_URL", AGENT.get("llm_server_url") or None)
+        org = os.environ.get("OPENAI_ORG_ID", AGENT.get("organization", None))
+
+        init_kwargs = {"api_key": api_key}
+        if base_url:
+            init_kwargs["base_url"] = base_url
+        if org:
+            init_kwargs["organization"] = org
+
+        llm = ChatOpenAI(**init_kwargs, **_model_kwargs)
     elif _service_provider == "google":
         if "claude" in _model_kwargs["model"]:
             api_key = os.environ.get("GOOGLE_API_KEY", AGENT.get("api_key", None))
@@ -166,22 +177,6 @@ def build_llm_model(
             from flowcept.agents.llms.gemini25 import Gemini25LLM
 
             llm = Gemini25LLM(**_model_kwargs)
-    elif _service_provider == "openai":
-        from langchain_openai import ChatOpenAI
-
-        api_key = os.environ.get("OPENAI_API_KEY", AGENT.get("api_key"))
-        base_url = os.environ.get("OPENAI_BASE_URL", AGENT.get("llm_server_url") or None)  # optional
-        org = os.environ.get("OPENAI_ORG_ID", AGENT.get("organization", None))  # optional
-
-        init_kwargs = {"api_key": api_key}
-        if base_url:
-            init_kwargs["base_url"] = base_url
-        if org:
-            init_kwargs["organization"] = org
-
-        # IMPORTANT: use the merged kwargs so `model` and temps flow through
-        llm = ChatOpenAI(**init_kwargs, **_model_kwargs)
-
     else:
         raise Exception("Currently supported providers are sambanova, openai, azure, and google.")
     if track_tools:
@@ -194,3 +189,43 @@ def build_llm_model(
             if tool_task:
                 llm.parent_task_id = tool_task.task_id
     return llm
+
+
+def normalize_message(user_msg: str) -> str:
+    """
+    Normalize a user message into a canonical, comparison-friendly form.
+
+    The function standardizes text by trimming whitespace, applying Unicode
+    normalization, normalizing dash characters, collapsing repeated whitespace,
+    removing trailing punctuation that does not affect semantics, and converting
+    the result to lowercase.
+
+    Parameters
+    ----------
+    user_msg : str
+        Raw user input message.
+
+    Returns
+    -------
+    str
+        Normalized message suitable for matching, comparison, or hashing.
+    """
+    # 1) Strip leading/trailing whitespace
+    user_msg = user_msg.strip()
+
+    # 2) Unicode normalize to avoid weird characters (like fancy quotes, dashes)
+    user_msg = unicodedata.normalize("NFKC", user_msg)
+
+    # 3) Normalize dashes commonly used in chemistry (C–H, C—H, etc.)
+    user_msg = user_msg.replace("–", "-").replace("—", "-")
+
+    # 4) Collapse multiple spaces / newlines into a single space
+    user_msg = re.sub(r"\s+", " ", user_msg)
+
+    # 5) Remove trailing punctuation that doesn't change semantics
+    #    e.g., "?", "!", "." at the VERY end
+    user_msg = re.sub(r"[?!.\s]+$", "", user_msg)
+
+    user_msg = user_msg.lower()
+
+    return user_msg

flowcept-0.9.19/src/flowcept/agents/flowcept_agent.py

@@ -0,0 +1,134 @@
+import json
+import os
+from threading import Thread
+
+from flowcept.agents import check_liveness
+from flowcept.agents.agents_utils import ToolResult
+from flowcept.agents.tools.general_tools import prompt_handler
+from flowcept.agents.agent_client import run_tool
+from flowcept.agents.flowcept_ctx_manager import mcp_flowcept, ctx_manager
+from flowcept.commons.flowcept_logger import FlowceptLogger
+from flowcept.configs import AGENT_HOST, AGENT_PORT, DUMP_BUFFER_PATH, MQ_ENABLED
+from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
+from uuid import uuid4
+
+import uvicorn
+
+
+class FlowceptAgent:
+    """
+    Flowcept agent server wrapper with optional offline buffer loading.
+    """
+
+    def __init__(self, buffer_path: str | None = None):
+        """
+        Initialize a FlowceptAgent.
+
+        Parameters
+        ----------
+        buffer_path : str or None
+            Optional path to a JSONL buffer file. When MQ is disabled, the agent
+            loads this file once at startup.
+        """
+        self.buffer_path = buffer_path
+        self.logger = FlowceptLogger()
+        self._server_thread: Thread | None = None
+        self._server = None
+
+    def _load_buffer_once(self) -> int:
+        """
+        Load messages from a JSONL buffer file into the agent context.
+
+        Returns
+        -------
+        int
+            Number of messages loaded.
+        """
+        path = self.buffer_path or DUMP_BUFFER_PATH
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"Buffer file not found: {path}")
+
+        count = 0
+        self.logger.info(f"Loading agent buffer from {path}")
+        if ctx_manager.agent_id is None:
+            agent_id = str(uuid4())
+            BaseAgentContextManager.agent_id = agent_id
+            ctx_manager.agent_id = agent_id
+        with open(path, "r") as handle:
+            for line in handle:
+                line = line.strip()
+                if not line:
+                    continue
+                msg_obj = json.loads(line)
+                ctx_manager.message_handler(msg_obj)
+                count += 1
+        self.logger.info(f"Loaded {count} messages from buffer.")
+        return count
+
+    def _run_server(self):
+        """Run the MCP server (blocking call)."""
+        config = uvicorn.Config(mcp_flowcept.streamable_http_app, host=AGENT_HOST, port=AGENT_PORT, lifespan="on")
+        self._server = uvicorn.Server(config)
+        self._server.run()
+
+    def start(self):
+        """
+        Start the agent server in a background thread.
+
+        Returns
+        -------
+        FlowceptAgent
+            The current instance.
+        """
+        if not MQ_ENABLED:
+            self._load_buffer_once()
+
+        self._server_thread = Thread(target=self._run_server, daemon=False)
+        self._server_thread.start()
+        self.logger.info(f"Flowcept agent server started on {AGENT_HOST}:{AGENT_PORT}")
+        return self
+
+    def stop(self):
+        """Stop the agent server and wait briefly for shutdown."""
+        if self._server is not None:
+            self._server.should_exit = True
+        if self._server_thread is not None:
+            self._server_thread.join(timeout=5)
+
+    def wait(self):
+        """Block until the server thread exits."""
+        if self._server_thread is not None:
+            self._server_thread.join()
+
+    def query(self, message: str) -> ToolResult:
+        """
+        Send a prompt to the agent's main router tool and return the response.
+        """
+        try:
+            resp = run_tool(tool_name=prompt_handler, kwargs={"message": message})[0]
+        except Exception as e:
+            return ToolResult(code=400, result=f"Error executing tool prompt_handler: {e}", tool_name="prompt_handler")
+
+        try:
+            return ToolResult(**json.loads(resp))
+        except Exception as e:
+            return ToolResult(
+                code=499,
+                result=f"Could not parse tool response as JSON: {resp}",
+                extra=str(e),
+                tool_name="prompt_handler",
+            )
+
+
+def main():
+    """
+    Start the MCP server.
+    """
+    agent = FlowceptAgent().start()
+    # Wake up tool call
+    print(run_tool(check_liveness, host=AGENT_HOST, port=AGENT_PORT)[0])
+    agent.wait()
+
+
+if __name__ == "__main__":
+    main()