flowcept 0.9.16__tar.gz → 0.9.17__tar.gz

{flowcept-0.9.16 → flowcept-0.9.17}/.github/workflows/create-release-n-publish.yml

@@ -18,10 +18,10 @@ jobs:
         with:
           fetch-depth: 1
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
 
       - name: Update version.py
         run: |

{flowcept-0.9.16 → flowcept-0.9.17}/.github/workflows/run-llm-tests.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: false

{flowcept-0.9.16 → flowcept-0.9.17}/.github/workflows/run-tests-all-dbs.yml

@@ -8,7 +8,7 @@ jobs:
     timeout-minutes: 40
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: true

{flowcept-0.9.16 → flowcept-0.9.17}/.github/workflows/run-tests-kafka.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: false

{flowcept-0.9.16 → flowcept-0.9.17}/.github/workflows/run-tests-py313.yml

@@ -1,4 +1,4 @@
-name: Tests on Py313
+name: Tests on Py_gt_312
 on: [pull_request]
 
 jobs:

{flowcept-0.9.16 → flowcept-0.9.17}/.github/workflows/run-tests-simple.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: false
       LMDB_ENABLED: true

{flowcept-0.9.16 → flowcept-0.9.17}/.github/workflows/run-tests.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: false

{flowcept-0.9.16 → flowcept-0.9.17}/.github/workflows/run_examples.sh

@@ -68,7 +68,9 @@ run_test() {
   echo "Ok, ran $test_path."
   if grep -iq "error" output.log; then
     echo "Test $test_path failed! See output.log for details."
+    echo "[BEGIN] Content of output.log"
     cat output.log
+    echo "[END] Content of output.log"
     exit 1
   fi
 

{flowcept-0.9.16 → flowcept-0.9.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.16
+Version: 0.9.17
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT
@@ -9,7 +9,7 @@ Keywords: agentic-ai,agentic-workflows,ai,big-data,dask,data-analytics,data-inte
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: msgpack
 Requires-Dist: numpy
 Requires-Dist: omegaconf
@@ -169,6 +169,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 [![Documentation](https://img.shields.io/badge/docs-readthedocs.io-green.svg)](https://flowcept.readthedocs.io/)
+[![Slack](https://img.shields.io/badge/Slack-%23flowcept%40Workflows%20Community-4A154B?logo=slack)](https://workflowscommunity.slack.com/archives/C06L5GYJKQS)
 [![Build](https://github.com/ORNL/flowcept/actions/workflows/create-release-n-publish.yml/badge.svg)](https://github.com/ORNL/flowcept/actions/workflows/create-release-n-publish.yml)
 [![PyPI](https://badge.fury.io/py/flowcept.svg)](https://pypi.org/project/flowcept)
 [![Tests](https://github.com/ORNL/flowcept/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ORNL/flowcept/actions/workflows/run-tests.yml)

{flowcept-0.9.16 → flowcept-0.9.17}/README.md

@@ -20,6 +20,7 @@ Flowcept captures and queries workflow provenance at runtime with minimal code c
 
 
 [![Documentation](https://img.shields.io/badge/docs-readthedocs.io-green.svg)](https://flowcept.readthedocs.io/)
+[![Slack](https://img.shields.io/badge/Slack-%23flowcept%40Workflows%20Community-4A154B?logo=slack)](https://workflowscommunity.slack.com/archives/C06L5GYJKQS)
 [![Build](https://github.com/ORNL/flowcept/actions/workflows/create-release-n-publish.yml/badge.svg)](https://github.com/ORNL/flowcept/actions/workflows/create-release-n-publish.yml)
 [![PyPI](https://badge.fury.io/py/flowcept.svg)](https://pypi.org/project/flowcept)
 [![Tests](https://github.com/ORNL/flowcept/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ORNL/flowcept/actions/workflows/run-tests.yml)

{flowcept-0.9.16 → flowcept-0.9.17}/docs/index.rst

@@ -18,6 +18,12 @@ Flowcept
        html[data-theme="auto"] .logo-light { display: none; }
        html[data-theme="auto"] .logo-dark  { display: inline; }
      }
+
+     .hero { text-align: center; margin: 1.25rem 0 0.25rem; }
+     .hero h1 { font-size: 2rem; line-height: 1.2; margin: 0; }
+     .tagline { text-align: center; font-size: 1.05rem; max-width: 60rem; margin: 0.25rem auto 1.25rem; }
+
+
    </style>
 
    <p align="center">
@@ -32,6 +38,18 @@ Flowcept
    :align: center
    :width: 120px
 
+.. raw:: html
+
+   <div class="hero">
+     <h4>Lightweight Distributed Workflow Provenance</h4>
+   </div>
+   <p class=\\"tagline\\">
+     Flowcept captures and queries workflow provenance at runtime with minimal code changes and low overhead.
+     It unifies data from diverse tools and workflows across the Edge–Cloud–HPC continuum and provides ML-aware capture,
+     MCP agents provenance, telemetry, extensible adapters, and flexible storage.
+   </p>
+
+
 .. toctree::
    :maxdepth: 2
    :caption: Contents:

{flowcept-0.9.16 → flowcept-0.9.17}/docs/prov_query.rst

@@ -142,8 +142,8 @@ Deleting a buffer file
 .. code-block:: python
 
    from flowcept import Flowcept
-   Flowcept.delete_buffer_file()                 # deletes default path from settings
-   Flowcept.delete_buffer_file(\"my_buffer.jsonl\")
+   Flowcept.delete_buffer_file()                     # deletes default path from settings
+   Flowcept.delete_buffer_file("my_buffer.jsonl")
 
 Notes
 ^^^^^

{flowcept-0.9.16 → flowcept-0.9.17}/docs/publications/README.md

@@ -7,48 +7,66 @@ Bibtex is available below.
 
 **Towards Lightweight Data Integration using Multi-workflow Provenance and Data Observability**  
 R. Souza, T. J. Skluzacek, S. R. Wilkinson, M. Ziatdinov, and R. Ferreira da Silva,  
-*IEEE International Conference on e-Science*, Limassol, Cyprus, 2023.  
-doi: [10.1109/e-Science58273.2023.10254822](https://doi.org/10.1109/e-Science58273.2023.10254822)  
-PDF: https://arxiv.org/pdf/2308.09004.pdf  
-
+*IEEE International Conference on e-Science*, Limassol, Cyprus, 2023.  <br/>
+**About**: Introduces Flowcept’s lightweight runtime provenance and data observability architecture and shows minimal-intrusion capture across heterogeneous workflows. <br/>
+[[doi]](https://doi.org/10.1109/e-Science58273.2023.10254822)
+[[pdf]](https://arxiv.org/pdf/2308.09004.pdf)  
 ---
 
 **PROV-AGENT: Unified Provenance for Tracking AI Agent Interactions in Agentic Workflows**  
 R. Souza, A. Gueroudji, S. DeWitt, D. Rosendo, T. Ghosal, R. Ross, P. Balaprakash, and R. Ferreira da Silva,  
-*IEEE International Conference on e-Science*, Chicago, USA., 2025.
-PDF: https://arxiv.org/pdf/2508.02866  
+*IEEE International Conference on e-Science*, Chicago, USA., 2025. <br/>
+**About**: Defines agentic provenance and a unified provenance model and tooling to capture, link, and query AI-agent interactions within agentic workflows. </br>
+[[doi]](https://doi.org/10.1109/eScience65000.2025.00093) 
+[[pdf]](https://arxiv.org/pdf/2508.02866)
+[[html]](https://arxiv.org/html/2508.02866v3) <br/>
+
 
 ---
 
 **Workflow Provenance in the Computing Continuum for Responsible, Trustworthy, and Energy-Efficient AI**  
 R. Souza, S. Caino-Lores, M. Coletti, T. J. Skluzacek, A. Costan, F. Suter, M. Mattoso, and R. Ferreira da Silva,  
-*IEEE International Conference on e-Science*, Osaka, Japan, 2024.  
-doi: [10.1109/e-Science62913.2024.10678731](https://doi.org/10.1109/e-Science62913.2024.10678731)  
-PDF: https://hal.science/hal-04902079v1/document  
+*IEEE International Conference on e-Science*, Osaka, Japan, 2024. <br/>
+**About**: Explains how end-to-end provenance across edge, cloud, and HPC supports responsible, trustworthy, and energy-aware AI workflows. <br/>
+[[doi]](https://doi.org/10.1109/e-Science62913.2024.10678731) 
+[[pdf]](https://hal.science/hal-04902079v1/document) <br/>
+
 
 ---
 
 **LLM Agents for Interactive Workflow Provenance: Reference Architecture and Evaluation Methodology**  
 R. Souza, T. Poteet, B. Etz, D. Rosendo, A. Gueroudji, W. Shin, P. Balaprakash, and R. Ferreira da Silva,  
 *Workflows in Support of Large-Scale Science (WORKS) co-located with the ACM/IEEE International Conference for High Performance Computing, Networking, Storage, and Analysis (SC)*, St. Louis, USA, 2025.  
-doi: [10.1145/3731599.3767582](https://doi.org/10.1145/3731599.3767582)  
+**About**: Presents a reference architecture and evaluation method for LLM agents that query and act on large-scale provenance databases. </br>
+[[doi]](https://doi.org/10.1145/3731599.3767582) 
+[[pdf]](https://arxiv.org/pdf/2509.13978)
+[[html]](https://arxiv.org/html/2509.13978)
 
 
+--- 
+
 ### Papers that used Flowcept
 
 
 **Toward a Persistent Event-Streaming System for High-Performance Computing Applications**
 M. Dorier, A. Gueroudji, V. Hayot-Sasson, H. Nguyen, S. Ockerman, R. Souza, T. Bicer, H. Pan, P. Carns, K. Chard, and others
-*Frontiers in High Performance Computing*, 2025.
-doi: [10.3389/fhpcp.2025.1638203](https://doi.org/10.3389/fhpcp.2025.1638203)
-PDF: https://web.cels.anl.gov/~woz/papers/Mofka_2025.pdf
+*Frontiers in High Performance Computing*, 2025. <br/>
+**About**: Demonstrates Flowcept generating high-volume provenance that is persistently streamed with Mofka for HPC applications. <br/>
+[[doi]](https://doi.org/10.3389/fhpcp.2025.1638203) 
+[[pdf]](https://web.cels.anl.gov/~woz/papers/Mofka_2025.pdf) 
+[[html]](https://www.frontiersin.org/journals/high-performance-computing/articles/10.3389/fhpcp.2025.1638203/full)
+<br/>
+
 
 ---
 
 **AI Agents for Enabling Autonomous Experiments at ORNL’s HPC and Manufacturing User Facilities**
 D. Rosendo, S. DeWitt, R. Souza, P. Austria, T. Ghosal, M. McDonnell, R. Miller, T. Skluzacek, J. Haley, B. Turcksin, and others
-*Extreme-Scale Experiment-in-the-Loop Computing (XLOOP) co-located with the ACM/IEEE International Conference for High Performance Computing, Networking, Storage, and Analysis (SC)*, 2025.
-PDF: https://rafaelsilva.com/files/publications/rosendo2025xloop.pdf
+*Extreme-Scale Experiment-in-the-Loop Computing (XLOOP) co-located with the ACM/IEEE International Conference for High Performance Computing, Networking, Storage, and Analysis (SC)*, 2025. <br/>
+**About**: Leverages Flowcept’s agentic provenance to coordinate multi-agent experiments and connect agents with HPC simulations through a shared provenance stream. <br/>
+[[doi]](https://doi.org/10.1145/3731599.3767592)
+[[pdf]](https://rafaelsilva.com/files/publications/rosendo2025xloop.pdf)
+[[html]](https://camps.aptaracorp.com/ACM_PMS/PMS/ACM/SCWORKSHOPS25/253/0b09762d-8e84-11f0-957d-16ffd757ba29/OUT/scworkshops25-253.html)
 
 --- 
 

{flowcept-0.9.16 → flowcept-0.9.17}/examples/llm_complex/llm_main_example.py

@@ -123,6 +123,13 @@ def search_workflow(ntokens, dataset_ref, train_data_path, val_data_path, test_d
 
 def start_dask(scheduler_file=None, start_dask_cluster=False, with_flowcept=True):
     from distributed import Client
+    try:
+        # Downgrading eventual dask comm errors in the logs
+        import logging
+        logging.getLogger("distributed.worker").setLevel(logging.WARNING)
+        logging.getLogger("distributed.comm").setLevel(logging.WARNING)
+    except:
+        pass
 
     if start_dask_cluster:
         import subprocess
@@ -196,9 +203,17 @@ def close_dask(client, cluster, scheduler_file=None, start_dask_cluster=False, _
                 print("Flowcept consumer closed.")
         else:
             print("Closing dask...")
-            client.close()
-            cluster.close()
-            print("Dask closed.")
+            try:
+                client.close()
+                cluster.close()
+                print("Dask closed.")
+            except Exception as e:
+                print(f"Some exception when closing dask: {e}")
+                try:
+                    import logging
+                    logging.getLogger("distributed.worker").setLevel(logging.WARNING)
+                except:
+                    pass
             if _flowcept:
                 print("Now closing flowcept consumer...")
                 _flowcept.stop()

{flowcept-0.9.16 → flowcept-0.9.17}/examples/llm_complex/llm_test_runner.py

@@ -67,7 +67,7 @@ def all():
         try:
             success = run_test(max_runs=3, config=config)
             if not success:
-                write_log(str(i) + "--> " + str(config))
+                write_log("GOT ERROR! " + str(i) + "--> " + str(config))
                 sys.exit(1)
         except Exception as e:
             write_exception(f"\n\n!!!!!!##### ERROR for combination {config}\n", e)

{flowcept-0.9.16 → flowcept-0.9.17}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "flowcept"
 dynamic = ["version"]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 
 authors = [{name = "Oak Ridge National Laboratory"}]
 description = "Capture and query workflow provenance data using data observability"

{flowcept-0.9.16 → flowcept-0.9.17}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.9.16 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.9.17 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.

{flowcept-0.9.16 → flowcept-0.9.17}/src/flowcept/flowceptor/adapters/mlflow/interception_event_handler.py

@@ -1,16 +1,19 @@
 """Event handler module."""
 
+from pathlib import Path
 from watchdog.events import FileSystemEventHandler
 
 
 class InterceptionEventHandler(FileSystemEventHandler):
     """Event handler class."""
 
-    def __init__(self, interceptor_instance, callback_function):
+    def __init__(self, interceptor_instance, file_path_to_watch, callback_function):
         super().__init__()
+        self.file_path_to_watch = file_path_to_watch
         self.callback_function = callback_function
         self.interceptor_instance = interceptor_instance
 
     def on_modified(self, event):
         """Get on modified."""
-        self.callback_function(self.interceptor_instance)
+        if Path(event.src_path).resolve() == Path(self.file_path_to_watch).resolve():
+            self.callback_function(self.interceptor_instance)

{flowcept-0.9.16 → flowcept-0.9.17}/src/flowcept/flowceptor/adapters/mlflow/mlflow_interceptor.py

@@ -84,7 +84,7 @@ class MLFlowInterceptor(BaseInterceptor):
     def observe(self):
         """Observe it."""
         self.logger.debug("Observing")
-        event_handler = InterceptionEventHandler(self, self.__class__.callback)
+        event_handler = InterceptionEventHandler(self, self.settings.file_path, self.__class__.callback)
         while not os.path.isfile(self.settings.file_path):
             self.logger.warning(
                 f"I can't watch the file {self.settings.file_path},"
@@ -95,6 +95,7 @@ class MLFlowInterceptor(BaseInterceptor):
             sleep(self.settings.watch_interval_sec)
 
         self._observer = PollingObserver()
-        self._observer.schedule(event_handler, self.settings.file_path, recursive=True)
+        watch_dir = os.path.dirname(self.settings.file_path) or "."
+        self._observer.schedule(event_handler, watch_dir, recursive=True)
         self._observer.start()
-        self.logger.info(f"Watching {self.settings.file_path}")
+        self.logger.info(f"Watching directory {watch_dir} with file {self.settings.file_path} ")

{flowcept-0.9.16 → flowcept-0.9.17}/src/flowcept/flowceptor/adapters/tensorboard/tensorboard_interceptor.py

@@ -1,6 +1,7 @@
 """Tensorboard interceptor module."""
 
 import os
+from pathlib import Path
 from time import sleep
 
 from tbparse import SummaryReader
@@ -29,6 +30,8 @@ class TensorboardInterceptor(BaseInterceptor):
     def __init__(self, plugin_key="tensorboard"):
         super().__init__(plugin_key)
         self._observer: PollingObserver = None
+        if not Path(self.settings.file_path).is_dir():
+            raise Exception("Tensorboard Observer must observe directories.")
         self.state_manager = InterceptorStateManager(self.settings)
         self.state_manager.reset()
         self.log_metrics = set(self.settings.log_metrics)
@@ -108,13 +111,15 @@ class TensorboardInterceptor(BaseInterceptor):
 
     def observe(self):
         """Observe it."""
-        event_handler = InterceptionEventHandler(self, self.__class__.callback)
+        self.logger.debug("Observing")
+        event_handler = InterceptionEventHandler(self, self.settings.file_path, self.__class__.callback)
         while not os.path.isdir(self.settings.file_path):
             self.logger.debug(f"I can't watch the file {self.settings.file_path}, as it does not exist.")
             self.logger.debug(f"\tI will sleep for {self.settings.watch_interval_sec} s to see if it appears.")
             sleep(self.settings.watch_interval_sec)
 
         self._observer = PollingObserver()
+
         self._observer.schedule(event_handler, self.settings.file_path, recursive=True)
         self._observer.start()
         self.logger.debug(f"Watching {self.settings.file_path}")

{flowcept-0.9.16 → flowcept-0.9.17}/src/flowcept/version.py

@@ -4,4 +4,4 @@
 # The expected format is: <Major>.<Minor>.<Patch>
 # This file is supposed to be automatically modified by the CI Bot.
 # See .github/workflows/version_bumper.py
-__version__ = "0.9.16"
+__version__ = "0.9.17"

{flowcept-0.9.16 → flowcept-0.9.17}/tests/adapters/test_file_observer.py

@@ -13,22 +13,26 @@ class TestFileObserver(unittest.TestCase):
         self.test_file_name = self.test_file.name
         self.test_file.close()
 
+        time.sleep(1.1)
+
         # Thread event to signal that the callback was called
         self.callback_called_event = threading.Event()
 
         # Define the callback function to be triggered on modification
-        def callback(file_path):
-            if file_path == self.test_file_name:
-                print(f"Callback triggered for {file_path}")
+        def callback(event):
+            if not event.is_directory and event.src_path == self.test_file_name:
+                print(f"Callback triggered for {event.src_path}")
                 self.callback_called_event.set()
 
+        watch_dir = os.path.dirname(self.test_file_name) or "."
+
         # Create an event handler and bind it to the callback
         self.event_handler = FileSystemEventHandler()
-        self.event_handler.on_modified = lambda event: callback(event.src_path)
+        self.event_handler.on_modified = lambda event: callback(event)
 
         # Set up watchdog observer
         self.observer = PollingObserver()
-        self.observer.schedule(self.event_handler, path=self.test_file_name, recursive=False)
+        self.observer.schedule(self.event_handler, path=watch_dir, recursive=False)
         self.observer.start()
 
     def tearDown(self):
@@ -44,10 +48,10 @@ class TestFileObserver(unittest.TestCase):
             f.flush()
             os.fsync(f.fileno())  # Ensure file system updates
 
-        # Add a small delay to ensure the observer catches the event
+        # Add a delay to ensure the observer catches the event
         time.sleep(2)
 
-        # Wait for the callback to be called (max wait 5 seconds)
+        # Wait for the callback to be called
         callback_triggered = self.callback_called_event.wait(timeout=10)
 
         # Assert that the callback was called