flowcept 0.9.16__tar.gz → 0.9.18__tar.gz

{flowcept-0.9.16 → flowcept-0.9.18}/.github/workflows/create-release-n-publish.yml

@@ -18,10 +18,10 @@ jobs:
         with:
           fetch-depth: 1
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
 
       - name: Update version.py
         run: |

{flowcept-0.9.16 → flowcept-0.9.18}/.github/workflows/run-llm-tests.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: false

{flowcept-0.9.16 → flowcept-0.9.18}/.github/workflows/run-tests-all-dbs.yml

@@ -8,7 +8,7 @@ jobs:
     timeout-minutes: 40
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: true

{flowcept-0.9.16 → flowcept-0.9.18}/.github/workflows/run-tests-kafka.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: false

{flowcept-0.9.16 → flowcept-0.9.18}/.github/workflows/run-tests-py313.yml

@@ -1,4 +1,4 @@
-name: Tests on Py313
+name: Tests on Py_gt_312
 on: [pull_request]
 
 jobs:

{flowcept-0.9.16 → flowcept-0.9.18}/.github/workflows/run-tests-simple.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: false
       LMDB_ENABLED: true

{flowcept-0.9.16 → flowcept-0.9.18}/.github/workflows/run-tests.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: false

{flowcept-0.9.16 → flowcept-0.9.18}/.github/workflows/run_examples.sh

@@ -68,7 +68,9 @@ run_test() {
   echo "Ok, ran $test_path."
   if grep -iq "error" output.log; then
     echo "Test $test_path failed! See output.log for details."
+    echo "[BEGIN] Content of output.log"
     cat output.log
+    echo "[END] Content of output.log"
     exit 1
   fi
 

{flowcept-0.9.16 → flowcept-0.9.18}/.gitignore

@@ -34,3 +34,4 @@ core.*
 
 uv.lock
 examples/flowcept_messages.jsonl
+agents.md

{flowcept-0.9.16 → flowcept-0.9.18}/Makefile

@@ -115,7 +115,7 @@ services-stop-mofka:
 # Run unit tests using pytest
 .PHONY: tests
 tests:
-	pytest --ignore=tests/adapters/test_tensorboard.py
+	pytest --timeout=300 --ignore=tests/adapters/test_tensorboard.py
 
 .PHONY: tests-notebooks
 tests-notebooks:

{flowcept-0.9.16 → flowcept-0.9.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.16
+Version: 0.9.18
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT
@@ -9,7 +9,7 @@ Keywords: agentic-ai,agentic-workflows,ai,big-data,dask,data-analytics,data-inte
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: msgpack
 Requires-Dist: numpy
 Requires-Dist: omegaconf
@@ -41,6 +41,7 @@ Requires-Dist: pyarrow; extra == 'all'
 Requires-Dist: pymongo; extra == 'all'
 Requires-Dist: pymupdf; extra == 'all'
 Requires-Dist: pytest; extra == 'all'
+Requires-Dist: pytest-timeout; extra == 'all'
 Requires-Dist: pyyaml; extra == 'all'
 Requires-Dist: redis; extra == 'all'
 Requires-Dist: requests; extra == 'all'
@@ -50,6 +51,7 @@ Requires-Dist: seaborn; extra == 'all'
 Requires-Dist: sphinx; extra == 'all'
 Requires-Dist: sqlalchemy; extra == 'all'
 Requires-Dist: streamlit; extra == 'all'
+Requires-Dist: tabulate; extra == 'all'
 Requires-Dist: tbparse; extra == 'all'
 Requires-Dist: tensorboard; extra == 'all'
 Requires-Dist: tensorflow; extra == 'all'
@@ -69,6 +71,7 @@ Requires-Dist: jupyterlab; extra == 'dev'
 Requires-Dist: nbmake; extra == 'dev'
 Requires-Dist: pika; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-timeout; extra == 'dev'
 Requires-Dist: pyyaml; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
 Requires-Dist: sphinx; extra == 'dev'
@@ -94,6 +97,7 @@ Requires-Dist: matplotlib; extra == 'llm-agent'
 Requires-Dist: mcp[cli]; extra == 'llm-agent'
 Requires-Dist: pymupdf; extra == 'llm-agent'
 Requires-Dist: streamlit; extra == 'llm-agent'
+Requires-Dist: tabulate; extra == 'llm-agent'
 Provides-Extra: llm-agent-audio
 Requires-Dist: gtts; extra == 'llm-agent-audio'
 Requires-Dist: langchain-community; extra == 'llm-agent-audio'
@@ -105,6 +109,7 @@ Requires-Dist: pymupdf; extra == 'llm-agent-audio'
 Requires-Dist: speechrecognition; extra == 'llm-agent-audio'
 Requires-Dist: streamlit; extra == 'llm-agent-audio'
 Requires-Dist: streamlit-mic-recorder; extra == 'llm-agent-audio'
+Requires-Dist: tabulate; extra == 'llm-agent-audio'
 Provides-Extra: llm-google
 Requires-Dist: google-genai; extra == 'llm-google'
 Requires-Dist: langchain-community; extra == 'llm-google'
@@ -113,6 +118,7 @@ Requires-Dist: matplotlib; extra == 'llm-google'
 Requires-Dist: mcp[cli]; extra == 'llm-google'
 Requires-Dist: pymupdf; extra == 'llm-google'
 Requires-Dist: streamlit; extra == 'llm-google'
+Requires-Dist: tabulate; extra == 'llm-google'
 Provides-Extra: lmdb
 Requires-Dist: lmdb; extra == 'lmdb'
 Provides-Extra: ml-dev
@@ -169,6 +175,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 [![Documentation](https://img.shields.io/badge/docs-readthedocs.io-green.svg)](https://flowcept.readthedocs.io/)
+[![Slack](https://img.shields.io/badge/Slack-%23flowcept%40Workflows%20Community-4A154B?logo=slack)](https://workflowscommunity.slack.com/archives/C06L5GYJKQS)
 [![Build](https://github.com/ORNL/flowcept/actions/workflows/create-release-n-publish.yml/badge.svg)](https://github.com/ORNL/flowcept/actions/workflows/create-release-n-publish.yml)
 [![PyPI](https://badge.fury.io/py/flowcept.svg)](https://pypi.org/project/flowcept)
 [![Tests](https://github.com/ORNL/flowcept/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ORNL/flowcept/actions/workflows/run-tests.yml)
@@ -179,6 +186,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 <h4 align="center">
+  <a href="https://flowcept.org">Website</a> &#8226;
   <a href="https://flowcept.readthedocs.io/">Documentation</a> &#8226; 
   <a href="./docs/publications">Publications</a>
 </h4>

{flowcept-0.9.16 → flowcept-0.9.18}/README.md

@@ -20,6 +20,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 [![Documentation](https://img.shields.io/badge/docs-readthedocs.io-green.svg)](https://flowcept.readthedocs.io/)
+[![Slack](https://img.shields.io/badge/Slack-%23flowcept%40Workflows%20Community-4A154B?logo=slack)](https://workflowscommunity.slack.com/archives/C06L5GYJKQS)
 [![Build](https://github.com/ORNL/flowcept/actions/workflows/create-release-n-publish.yml/badge.svg)](https://github.com/ORNL/flowcept/actions/workflows/create-release-n-publish.yml)
 [![PyPI](https://badge.fury.io/py/flowcept.svg)](https://pypi.org/project/flowcept)
 [![Tests](https://github.com/ORNL/flowcept/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ORNL/flowcept/actions/workflows/run-tests.yml)
@@ -30,6 +31,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 <h4 align="center">
+  <a href="https://flowcept.org">Website</a> &#8226;
   <a href="https://flowcept.readthedocs.io/">Documentation</a> &#8226; 
   <a href="./docs/publications">Publications</a>
 </h4>

{flowcept-0.9.16 → flowcept-0.9.18}/docs/index.rst

@@ -18,6 +18,12 @@ Flowcept
        html[data-theme="auto"] .logo-light { display: none; }
        html[data-theme="auto"] .logo-dark  { display: inline; }
      }
+
+     .hero { text-align: center; margin: 1.25rem 0 0.25rem; }
+     .hero h1 { font-size: 2rem; line-height: 1.2; margin: 0; }
+     .tagline { text-align: center; font-size: 1.05rem; max-width: 60rem; margin: 0.25rem auto 1.25rem; }
+
+
    </style>
 
    <p align="center">
@@ -32,6 +38,18 @@ Flowcept
    :align: center
    :width: 120px
 
+.. raw:: html
+
+   <div class="hero">
+     <h4>Lightweight Distributed Workflow Provenance</h4>
+   </div>
+   <p class=\\"tagline\\">
+     Flowcept captures and queries workflow provenance at runtime with minimal code changes and low overhead.
+     It unifies data from diverse tools and workflows across the Edge–Cloud–HPC continuum and provides ML-aware capture,
+     MCP agents provenance, telemetry, extensible adapters, and flexible storage.
+   </p>
+
+
 .. toctree::
    :maxdepth: 2
    :caption: Contents:

{flowcept-0.9.16 → flowcept-0.9.18}/docs/prov_query.rst

@@ -142,8 +142,8 @@ Deleting a buffer file
 .. code-block:: python
 
    from flowcept import Flowcept
-   Flowcept.delete_buffer_file()                 # deletes default path from settings
-   Flowcept.delete_buffer_file(\"my_buffer.jsonl\")
+   Flowcept.delete_buffer_file()                     # deletes default path from settings
+   Flowcept.delete_buffer_file("my_buffer.jsonl")
 
 Notes
 ^^^^^

{flowcept-0.9.16 → flowcept-0.9.18}/docs/publications/README.md

@@ -7,48 +7,66 @@ Bibtex is available below.
 
 **Towards Lightweight Data Integration using Multi-workflow Provenance and Data Observability**  
 R. Souza, T. J. Skluzacek, S. R. Wilkinson, M. Ziatdinov, and R. Ferreira da Silva,  
-*IEEE International Conference on e-Science*, Limassol, Cyprus, 2023.  
-doi: [10.1109/e-Science58273.2023.10254822](https://doi.org/10.1109/e-Science58273.2023.10254822)  
-PDF: https://arxiv.org/pdf/2308.09004.pdf  
-
+*IEEE International Conference on e-Science*, Limassol, Cyprus, 2023.  <br/>
+**About**: Introduces Flowcept’s lightweight runtime provenance and data observability architecture and shows minimal-intrusion capture across heterogeneous workflows. <br/>
+[[doi]](https://doi.org/10.1109/e-Science58273.2023.10254822)
+[[pdf]](https://arxiv.org/pdf/2308.09004.pdf)  
 ---
 
 **PROV-AGENT: Unified Provenance for Tracking AI Agent Interactions in Agentic Workflows**  
 R. Souza, A. Gueroudji, S. DeWitt, D. Rosendo, T. Ghosal, R. Ross, P. Balaprakash, and R. Ferreira da Silva,  
-*IEEE International Conference on e-Science*, Chicago, USA., 2025.
-PDF: https://arxiv.org/pdf/2508.02866  
+*IEEE International Conference on e-Science*, Chicago, USA., 2025. <br/>
+**About**: Defines agentic provenance and a unified provenance model and tooling to capture, link, and query AI-agent interactions within agentic workflows. </br>
+[[doi]](https://doi.org/10.1109/eScience65000.2025.00093) 
+[[pdf]](https://arxiv.org/pdf/2508.02866)
+[[html]](https://arxiv.org/html/2508.02866v3) <br/>
+
 
 ---
 
 **Workflow Provenance in the Computing Continuum for Responsible, Trustworthy, and Energy-Efficient AI**  
 R. Souza, S. Caino-Lores, M. Coletti, T. J. Skluzacek, A. Costan, F. Suter, M. Mattoso, and R. Ferreira da Silva,  
-*IEEE International Conference on e-Science*, Osaka, Japan, 2024.  
-doi: [10.1109/e-Science62913.2024.10678731](https://doi.org/10.1109/e-Science62913.2024.10678731)  
-PDF: https://hal.science/hal-04902079v1/document  
+*IEEE International Conference on e-Science*, Osaka, Japan, 2024. <br/>
+**About**: Explains how end-to-end provenance across edge, cloud, and HPC supports responsible, trustworthy, and energy-aware AI workflows. <br/>
+[[doi]](https://doi.org/10.1109/e-Science62913.2024.10678731) 
+[[pdf]](https://hal.science/hal-04902079v1/document) <br/>
+
 
 ---
 
 **LLM Agents for Interactive Workflow Provenance: Reference Architecture and Evaluation Methodology**  
 R. Souza, T. Poteet, B. Etz, D. Rosendo, A. Gueroudji, W. Shin, P. Balaprakash, and R. Ferreira da Silva,  
 *Workflows in Support of Large-Scale Science (WORKS) co-located with the ACM/IEEE International Conference for High Performance Computing, Networking, Storage, and Analysis (SC)*, St. Louis, USA, 2025.  
-doi: [10.1145/3731599.3767582](https://doi.org/10.1145/3731599.3767582)  
+**About**: Presents a reference architecture and evaluation method for LLM agents that query and act on large-scale provenance databases. </br>
+[[doi]](https://doi.org/10.1145/3731599.3767582) 
+[[pdf]](https://arxiv.org/pdf/2509.13978)
+[[html]](https://arxiv.org/html/2509.13978)
 
 
+--- 
+
 ### Papers that used Flowcept
 
 
 **Toward a Persistent Event-Streaming System for High-Performance Computing Applications**
 M. Dorier, A. Gueroudji, V. Hayot-Sasson, H. Nguyen, S. Ockerman, R. Souza, T. Bicer, H. Pan, P. Carns, K. Chard, and others
-*Frontiers in High Performance Computing*, 2025.
-doi: [10.3389/fhpcp.2025.1638203](https://doi.org/10.3389/fhpcp.2025.1638203)
-PDF: https://web.cels.anl.gov/~woz/papers/Mofka_2025.pdf
+*Frontiers in High Performance Computing*, 2025. <br/>
+**About**: Demonstrates Flowcept generating high-volume provenance that is persistently streamed with Mofka for HPC applications. <br/>
+[[doi]](https://doi.org/10.3389/fhpcp.2025.1638203) 
+[[pdf]](https://web.cels.anl.gov/~woz/papers/Mofka_2025.pdf) 
+[[html]](https://www.frontiersin.org/journals/high-performance-computing/articles/10.3389/fhpcp.2025.1638203/full)
+<br/>
+
 
 ---
 
 **AI Agents for Enabling Autonomous Experiments at ORNL’s HPC and Manufacturing User Facilities**
 D. Rosendo, S. DeWitt, R. Souza, P. Austria, T. Ghosal, M. McDonnell, R. Miller, T. Skluzacek, J. Haley, B. Turcksin, and others
-*Extreme-Scale Experiment-in-the-Loop Computing (XLOOP) co-located with the ACM/IEEE International Conference for High Performance Computing, Networking, Storage, and Analysis (SC)*, 2025.
-PDF: https://rafaelsilva.com/files/publications/rosendo2025xloop.pdf
+*Extreme-Scale Experiment-in-the-Loop Computing (XLOOP) co-located with the ACM/IEEE International Conference for High Performance Computing, Networking, Storage, and Analysis (SC)*, 2025. <br/>
+**About**: Leverages Flowcept’s agentic provenance to coordinate multi-agent experiments and connect agents with HPC simulations through a shared provenance stream. <br/>
+[[doi]](https://doi.org/10.1145/3731599.3767592)
+[[pdf]](https://rafaelsilva.com/files/publications/rosendo2025xloop.pdf)
+[[html]](https://camps.aptaracorp.com/ACM_PMS/PMS/ACM/SCWORKSHOPS25/253/0b09762d-8e84-11f0-957d-16ffd757ba29/OUT/scworkshops25-253.html)
 
 --- 
 

{flowcept-0.9.16 → flowcept-0.9.18}/examples/llm_complex/llm_main_example.py

@@ -123,6 +123,13 @@ def search_workflow(ntokens, dataset_ref, train_data_path, val_data_path, test_d
 
 def start_dask(scheduler_file=None, start_dask_cluster=False, with_flowcept=True):
     from distributed import Client
+    try:
+        # Downgrading eventual dask comm errors in the logs
+        import logging
+        logging.getLogger("distributed.worker").setLevel(logging.WARNING)
+        logging.getLogger("distributed.comm").setLevel(logging.WARNING)
+    except:
+        pass
 
     if start_dask_cluster:
         import subprocess
@@ -196,9 +203,17 @@ def close_dask(client, cluster, scheduler_file=None, start_dask_cluster=False, _
                 print("Flowcept consumer closed.")
         else:
             print("Closing dask...")
-            client.close()
-            cluster.close()
-            print("Dask closed.")
+            try:
+                client.close()
+                cluster.close()
+                print("Dask closed.")
+            except Exception as e:
+                print(f"Some exception when closing dask: {e}")
+                try:
+                    import logging
+                    logging.getLogger("distributed.worker").setLevel(logging.WARNING)
+                except:
+                    pass
             if _flowcept:
                 print("Now closing flowcept consumer...")
                 _flowcept.stop()

{flowcept-0.9.16 → flowcept-0.9.18}/examples/llm_complex/llm_test_runner.py

@@ -67,7 +67,7 @@ def all():
         try:
             success = run_test(max_runs=3, config=config)
             if not success:
-                write_log(str(i) + "--> " + str(config))
+                write_log("GOT ERROR! " + str(i) + "--> " + str(config))
                 sys.exit(1)
         except Exception as e:
             write_exception(f"\n\n!!!!!!##### ERROR for combination {config}\n", e)

{flowcept-0.9.16 → flowcept-0.9.18}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "flowcept"
 dynamic = ["version"]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 
 authors = [{name = "Oak Ridge National Laboratory"}]
 description = "Capture and query workflow provenance data using data observability"
@@ -65,7 +65,7 @@ mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog", "cryptography"]
 nvidia = ["nvidia-ml-py"]
 mqtt = ["paho-mqtt"]
 tensorboard = ["tensorboard", "tensorflow", "tbparse"]
-llm_agent = ["mcp[cli]", "langchain_community", "langchain_openai", "streamlit", "PyMuPDF", "matplotlib"]
+llm_agent = ["mcp[cli]", "langchain_community", "langchain_openai", "streamlit", "PyMuPDF", "matplotlib", "tabulate"]
 llm_google = ["flowcept[llm_agent]", "google-genai"]
 llm_agent_audio = ["flowcept[llm_agent]", "streamlit-mic-recorder", "SpeechRecognition", "pydub", "gTTS"]
 # System dependency (required for pydub)
@@ -80,8 +80,9 @@ dev = [
     "nbmake",
     "pika",
     "pytest",
+    "pytest-timeout",
     "ruff",
-    "pyyaml",
+    "pyyaml"
 ]
 # Torch and some other ml-specific libs, only used for dev purposes, require the following specific versions.
 ml_dev = [

{flowcept-0.9.16 → flowcept-0.9.18}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.9.16 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.18 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
@@ -94,6 +94,7 @@ agent:
   service_provider: '?'
   model_kwargs: {}
   audio_enabled: false
+  debug: true
 
 databases:
 

{flowcept-0.9.16 → flowcept-0.9.18}/src/flowcept/agents/agents_utils.py

@@ -1,4 +1,6 @@
 import os
+import re
+import unicodedata
 from typing import Union, Dict
 
 from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
@@ -194,3 +196,43 @@ def build_llm_model(
             if tool_task:
                 llm.parent_task_id = tool_task.task_id
     return llm
+
+
+def normalize_message(user_msg: str) -> str:
+    """
+    Normalize a user message into a canonical, comparison-friendly form.
+
+    The function standardizes text by trimming whitespace, applying Unicode
+    normalization, normalizing dash characters, collapsing repeated whitespace,
+    removing trailing punctuation that does not affect semantics, and converting
+    the result to lowercase.
+
+    Parameters
+    ----------
+    user_msg : str
+        Raw user input message.
+
+    Returns
+    -------
+    str
+        Normalized message suitable for matching, comparison, or hashing.
+    """
+    # 1) Strip leading/trailing whitespace
+    user_msg = user_msg.strip()
+
+    # 2) Unicode normalize to avoid weird characters (like fancy quotes, dashes)
+    user_msg = unicodedata.normalize("NFKC", user_msg)
+
+    # 3) Normalize dashes commonly used in chemistry (C–H, C—H, etc.)
+    user_msg = user_msg.replace("–", "-").replace("—", "-")
+
+    # 4) Collapse multiple spaces / newlines into a single space
+    user_msg = re.sub(r"\s+", " ", user_msg)
+
+    # 5) Remove trailing punctuation that doesn't change semantics
+    #    e.g., "?", "!", "." at the VERY end
+    user_msg = re.sub(r"[?!.\s]+$", "", user_msg)
+
+    user_msg = user_msg.lower()
+
+    return user_msg

{flowcept-0.9.16 → flowcept-0.9.18}/src/flowcept/agents/flowcept_agent.py

@@ -20,11 +20,14 @@ def main():
     def run():
         uvicorn.run(mcp_flowcept.streamable_http_app, host=AGENT_HOST, port=AGENT_PORT, lifespan="on")
 
-    Thread(target=run).start()
+    server_thread = Thread(target=run, daemon=False)
+    server_thread.start()
     sleep(2)
     # Wake up tool call
     print(run_tool(check_liveness, host=AGENT_HOST, port=AGENT_PORT)[0])
 
+    server_thread.join()
+
 
 if __name__ == "__main__":
     main()