flowcept 0.9.8__tar.gz → 0.9.10__tar.gz

{flowcept-0.9.8 → flowcept-0.9.10}/Makefile

@@ -45,6 +45,7 @@ clean:
 	@find . -type d -name "*output_data*" -exec sh -c 'rm -rf "$$@" 2>/dev/null || true' sh {} +
 	@find . -type f -name "*nohup*" -exec sh -c 'rm -f "$$@" 2>/dev/null || true' sh {} +
 	@sh -c 'sphinx-build -M clean docs docs/_build > /dev/null 2>&1 || true'
+	@sh -c 'rm -f docs/generated/* 2>/dev/null || true'
 
 # Build the HTML documentation using Sphinx
 .PHONY: docs

{flowcept-0.9.8 → flowcept-0.9.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.9.8
+Version: 0.9.10
 Summary: Capture and query workflow provenance data using data observability
 Author: Oak Ridge National Laboratory
 License-Expression: MIT
@@ -56,6 +56,7 @@ Requires-Dist: tensorflow; extra == 'all'
 Requires-Dist: tomli; extra == 'all'
 Requires-Dist: watchdog; extra == 'all'
 Provides-Extra: analytics
+Requires-Dist: matplotlib; extra == 'analytics'
 Requires-Dist: plotly; extra == 'analytics'
 Requires-Dist: scipy; extra == 'analytics'
 Requires-Dist: seaborn; extra == 'analytics'

{flowcept-0.9.8 → flowcept-0.9.10}/docs/api-reference.rst

@@ -1,27 +1,17 @@
 API Reference
 =============
 
-.. autosummary::
-   :toctree: generated/
-   :recursive:
-
-   flowcept.Flowcept
-   flowcept.flowcept_api.db_api.DBAPI
-   flowcept.TaskObject
-   flowcept.WorkflowObject
-   flowcept.FlowceptTask
-   flowcept.FlowceptLoop
-   flowcept.FlowceptLightweightLoop
 
 Main Flowcept Object
 --------------------
 
+See also: `Flowcept object <https://flowcept.readthedocs.io/en/latest/prov_capture.html>`_ in provenance capture methods.
+
 .. autoclass:: flowcept.Flowcept
    :members:
    :special-members: __init__
    :exclude-members: __weakref__, __dict__, __module__
-
-
+   :noindex:
 
 Flowcept.db: Querying the Database
 ----------------------------------
@@ -49,18 +39,20 @@ Typical usage:
    :members:
    :undoc-members:
    :show-inheritance:
-
+   :noindex:
 
 Main Message Objects
 ---------------------
 
 .. autoclass:: flowcept.TaskObject
    :members:
+   :noindex:
 
 .. autoclass:: flowcept.WorkflowObject
    :members:
+   :noindex:
 
-FlowceptTask object
+FlowceptTask
 -------------------
 
 .. autoclass:: flowcept.FlowceptTask
@@ -68,8 +60,9 @@ FlowceptTask object
    :special-members: __init__
    :undoc-members:
    :show-inheritance:
+   :noindex:
 
-FlowceptLoop object
+FlowceptLoop
 -------------------
 
 .. autoclass:: flowcept.FlowceptLoop
@@ -77,13 +70,14 @@ FlowceptLoop object
    :special-members: __init__
    :undoc-members:
    :show-inheritance:
+   :noindex:
 
-
-FlowceptLightweightLoop object
+FlowceptLightweightLoop
 ------------------------------
 
 .. autoclass:: flowcept.FlowceptLightweightLoop
    :members:
    :special-members: __init__
    :undoc-members:
-   :show-inheritance:
+   :show-inheritance:
+   :noindex:

{flowcept-0.9.8 → flowcept-0.9.10}/docs/architecture.rst

@@ -8,9 +8,7 @@ Flowcept Architecture
 Overview
 --------
 
-Rather than designing a new provenance system from scratch, Flowcept builds on a **reference architecture**
-(Figure below) that follows established distributed design principles and is extensible to support
-LLM-based provenance agents.  
+Flowcept follows established distributed design principles and is extensible to support multiple MQs as backends, multiple database models (e.g., Document, Relational, Graph), and LLM-based agents.
 This provides a flexible, loosely coupled framework that scales from small centralized workflows
 to large HPC workflows across the **Edge–Cloud–HPC (ECH)** continuum.
 
@@ -19,8 +17,6 @@ to large HPC workflows across the **Edge–Cloud–HPC (ECH)** continuum.
    :align: center
    :width: 100%
 
-   Reference architecture for distributed workflow provenance with a provenance AI agent.
-
 Capture Mechanisms
 ------------------
 
@@ -59,7 +55,7 @@ They can store provenance into different backends, depending on performance and
 
 - **MongoDB** → efficient bulk writes and flexible aggregation queries.  
 - **LMDB** → lightweight key–value store, optimized for high-frequency inserts.  
-- **Neo4j** → graph database, enabling graph traversal and relationship queries.
+- **Custom** → Other storage mechanisms can be implemented by following Flowcept's Base Consumer class. See `here <https://flowcept.readthedocs.io/en/latest/prov_storage.html#provenance-consumer>`_.
 
 Access and Querying
 -------------------
@@ -70,6 +66,8 @@ Users can access provenance data through multiple interfaces:
 - **Grafana dashboards** for monitoring and visualization  
 - **Natural language queries** through Flowcept’s LLM-powered agent (introduced in this work)
 
+See more about `querying <https://flowcept.readthedocs.io/en/latest/prov_query.html>`_.
+
 Deployment Flexibility
 ----------------------
 

{flowcept-0.9.8 → flowcept-0.9.10}/docs/conf.py

@@ -13,8 +13,12 @@ author = "Oak Ridge National Lab"
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
-extensions = ["sphinx.ext.autodoc"]
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+]
 
+autosummary_generate = True
 templates_path = ["_templates"]
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 

flowcept-0.9.10/docs/img/PROV-AGENT.svg

@@ -0,0 +1,226 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="620pt" height="666pt" viewBox="0.00 0.00 619.79 665.60">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 661.6)">
+<title>AIAgentProvenance</title>
+<polygon fill="white" stroke="none" points="-4,4 -4,-661.6 615.79,-661.6 615.79,4 -4,4"/>
+<!-- DataObject -->
+<g id="node1" class="node">
+<title>DataObject</title>
+<ellipse fill="lightyellow" stroke="black" cx="270.92" cy="-373.2" rx="59.34" ry="18"/>
+<text text-anchor="middle" x="270.92" y="-369" font-family="Helvetica,sans-Serif" font-size="14.00">DataObject</text>
+</g>
+<!-- ToolExecution -->
+<g id="node11" class="node">
+<title>ToolExecution</title>
+<polygon fill="lightblue" stroke="black" points="312.83,-302.4 233.01,-302.4 233.01,-266.4 312.83,-266.4 312.83,-302.4"/>
+<text text-anchor="middle" x="272.92" y="-280.2" font-family="Helvetica,sans-Serif" font-size="14.00">AgentTool</text>
+</g>
+<!-- DataObject&#45;&gt;ToolExecution -->
+<g id="edge8" class="edge">
+<title>DataObject-&gt;ToolExecution</title>
+<path fill="none" stroke="black" d="M245,-356.58C232.59,-346.96 221.72,-333.91 228.29,-320.4 230.02,-316.84 232.31,-313.53 234.94,-310.49"/>
+<polygon fill="black" stroke="black" points="237.24,-313.14 241.98,-303.67 232.37,-308.11 237.24,-313.14"/>
+<text text-anchor="middle" x="276.11" y="-324.6" font-family="Times,serif" font-size="14.00">wasGeneratedBy</text>
+</g>
+<!-- AIAgent -->
+<g id="node13" class="node">
+<title>AIAgent</title>
+<polygon fill="orange" stroke="black" points="447.03,-23.56 390.92,-36 334.81,-23.56 356.24,-3.44 425.6,-3.44 447.03,-23.56"/>
+<text text-anchor="middle" x="390.92" y="-13.8" font-family="Helvetica,sans-Serif" font-size="14.00">AIAgent</text>
+</g>
+<!-- DataObject&#45;&gt;AIAgent -->
+<g id="edge9" class="edge">
+<title>DataObject-&gt;AIAgent</title>
+<path fill="none" stroke="black" d="M323.39,-364.52C395.64,-352.44 516.92,-326.1 516.92,-285.4 516.92,-285.4 516.92,-285.4 516.92,-105.8 516.92,-66.68 475.22,-43.85 439.8,-31.5"/>
+<polygon fill="black" stroke="black" points="441.1,-28.24 430.5,-28.47 438.93,-34.9 441.1,-28.24"/>
+<text text-anchor="middle" x="564.35" y="-191.4" font-family="Times,serif" font-size="14.00">wasAttributedTo</text>
+</g>
+<!-- Model -->
+<g id="node2" class="node">
+<title>Model</title>
+<ellipse fill="lightyellow" stroke="black" cx="316.92" cy="-106.8" rx="46.46" ry="18"/>
+<text text-anchor="middle" x="316.92" y="-102.6" font-family="Helvetica,sans-Serif" font-size="14.00">AIModel</text>
+</g>
+<!-- DomainData -->
+<g id="node3" class="node">
+<title>DomainData</title>
+<ellipse fill="lightyellow" stroke="black" cx="188.92" cy="-462" rx="64.71" ry="18"/>
+<text text-anchor="middle" x="188.92" y="-457.8" font-family="Helvetica,sans-Serif" font-size="14.00">DomainData</text>
+</g>
+<!-- DomainData&#45;&gt;DataObject -->
+<g id="edge17" class="edge">
+<title>DomainData-&gt;DataObject</title>
+<path fill="none" stroke="gray" stroke-dasharray="5,2" d="M204.73,-444.26C216.87,-431.42 233.82,-413.48 247.52,-398.97"/>
+<polygon fill="gray" stroke="gray" points="249.8,-401.66 254.12,-391.98 244.71,-396.85 249.8,-401.66"/>
+</g>
+<!-- Task -->
+<g id="node10" class="node">
+<title>Task</title>
+<polygon fill="lightblue" stroke="black" points="328.92,-568.8 274.92,-568.8 274.92,-532.8 328.92,-532.8 328.92,-568.8"/>
+<text text-anchor="middle" x="301.92" y="-546.6" font-family="Helvetica,sans-Serif" font-size="14.00">Task</text>
+</g>
+<!-- DomainData&#45;&gt;Task -->
+<g id="edge11" class="edge">
+<title>DomainData-&gt;Task</title>
+<path fill="none" stroke="black" d="M180.09,-479.95C175.74,-490.89 172.83,-504.89 180.29,-514.8 199.26,-539.99 235.18,-547.94 263.08,-550.06"/>
+<polygon fill="black" stroke="black" points="262.8,-553.55 272.97,-550.57 263.16,-546.56 262.8,-553.55"/>
+<text text-anchor="middle" x="228.11" y="-502.2" font-family="Times,serif" font-size="14.00">wasGeneratedBy</text>
+</g>
+<!-- Telemetry -->
+<g id="node4" class="node">
+<title>Telemetry</title>
+<ellipse fill="lightyellow" stroke="black" cx="522.92" cy="-462" rx="74.36" ry="18"/>
+<text text-anchor="middle" x="522.92" y="-457.8" font-family="Helvetica,sans-Serif" font-size="14.00">TelemetryData</text>
+</g>
+<!-- Telemetry&#45;&gt;DataObject -->
+<g id="edge18" class="edge">
+<title>Telemetry-&gt;DataObject</title>
+<path fill="none" stroke="gray" stroke-dasharray="5,2" d="M481.31,-446.67C436.92,-431.38 366.66,-407.18 319.7,-391"/>
+<polygon fill="gray" stroke="gray" points="320.91,-387.72 310.31,-387.77 318.63,-394.33 320.91,-387.72"/>
+</g>
+<!-- Telemetry&#45;&gt;Task -->
+<g id="edge12" class="edge">
+<title>Telemetry-&gt;Task</title>
+<path fill="none" stroke="black" d="M499.8,-479.44C483.03,-490.74 459.41,-505.34 436.92,-514.8 405.57,-527.98 367.99,-537.23 340.45,-542.9"/>
+<polygon fill="black" stroke="black" points="339.92,-539.43 330.79,-544.81 341.28,-546.3 339.92,-539.43"/>
+<text text-anchor="middle" x="516.72" y="-502.2" font-family="Times,serif" font-size="14.00">wasGeneratedBy</text>
+</g>
+<!-- SchedulingData -->
+<g id="node5" class="node">
+<title>SchedulingData</title>
+<ellipse fill="lightyellow" stroke="black" cx="350.92" cy="-462" rx="79.22" ry="18"/>
+<text text-anchor="middle" x="350.92" y="-457.8" font-family="Helvetica,sans-Serif" font-size="14.00">SchedulingData</text>
+</g>
+<!-- SchedulingData&#45;&gt;DataObject -->
+<g id="edge19" class="edge">
+<title>SchedulingData-&gt;DataObject</title>
+<path fill="none" stroke="gray" stroke-dasharray="5,2" d="M335.49,-444.26C323.65,-431.42 307.12,-413.48 293.75,-398.97"/>
+<polygon fill="gray" stroke="gray" points="296.67,-396.98 287.32,-392 291.53,-401.72 296.67,-396.98"/>
+</g>
+<!-- SchedulingData&#45;&gt;Task -->
+<g id="edge10" class="edge">
+<title>SchedulingData-&gt;Task</title>
+<path fill="none" stroke="black" d="M344.84,-480.33C340.91,-490.65 335.33,-503.84 328.92,-514.8 327.25,-517.66 325.37,-520.54 323.41,-523.37"/>
+<polygon fill="black" stroke="black" points="320.61,-521.28 317.48,-531.4 326.24,-525.44 320.61,-521.28"/>
+<text text-anchor="middle" x="385.18" y="-502.2" font-family="Times,serif" font-size="14.00">wasGeneratedBy</text>
+</g>
+<!-- Prompt -->
+<g id="node6" class="node">
+<title>Prompt</title>
+<ellipse fill="lightyellow" stroke="black" cx="209.92" cy="-106.8" rx="42.16" ry="18"/>
+<text text-anchor="middle" x="209.92" y="-102.6" font-family="Helvetica,sans-Serif" font-size="14.00">Prompt</text>
+</g>
+<!-- Prompt&#45;&gt;DataObject -->
+<g id="edge20" class="edge">
+<title>Prompt-&gt;DataObject</title>
+<path fill="none" stroke="gray" stroke-dasharray="5,2" d="M177.76,-118.85C159.79,-125.75 140.01,-134.8 134.92,-142.8 130.91,-149.1 133.23,-152.33 134.92,-159.6 154.77,-245.23 165.09,-270.15 221.92,-337.2 225.85,-341.84 230.52,-346.17 235.39,-350.1"/>
+<polygon fill="gray" stroke="gray" points="232.94,-352.64 243.05,-355.82 237.13,-347.03 232.94,-352.64"/>
+</g>
+<!-- ResponseData -->
+<g id="node7" class="node">
+<title>ResponseData</title>
+<ellipse fill="lightyellow" stroke="black" cx="74.92" cy="-106.8" rx="74.92" ry="18"/>
+<text text-anchor="middle" x="74.92" y="-102.6" font-family="Helvetica,sans-Serif" font-size="14.00">ResponseData</text>
+</g>
+<!-- ResponseData&#45;&gt;DataObject -->
+<g id="edge21" class="edge">
+<title>ResponseData-&gt;DataObject</title>
+<path fill="none" stroke="gray" stroke-dasharray="5,2" d="M80.55,-125.14C92.61,-160.7 123.44,-243.45 166.92,-302.4 182.67,-323.76 207.04,-340.61 228.23,-352.41"/>
+<polygon fill="gray" stroke="gray" points="226.51,-355.46 236.98,-357.07 229.81,-349.28 226.51,-355.46"/>
+</g>
+<!-- LLMInvocation -->
+<g id="node12" class="node">
+<title>LLMInvocation</title>
+<polygon fill="lightblue" stroke="black" points="338.51,-213.6 207.33,-213.6 207.33,-177.6 338.51,-177.6 338.51,-213.6"/>
+<text text-anchor="middle" x="272.92" y="-191.4" font-family="Helvetica,sans-Serif" font-size="14.00">AIModelInvocation</text>
+</g>
+<!-- ResponseData&#45;&gt;LLMInvocation -->
+<g id="edge13" class="edge">
+<title>ResponseData-&gt;LLMInvocation</title>
+<path fill="none" stroke="black" d="M95.56,-124.45C110.34,-135.69 131.13,-150.15 151.29,-159.6 165.3,-166.16 180.81,-171.76 195.92,-176.43"/>
+<polygon fill="black" stroke="black" points="194.8,-179.75 205.38,-179.24 196.79,-173.04 194.8,-179.75"/>
+<text text-anchor="middle" x="199.11" y="-147" font-family="Times,serif" font-size="14.00">wasGeneratedBy</text>
+</g>
+<!-- ResponseData&#45;&gt;AIAgent -->
+<g id="edge14" class="edge">
+<title>ResponseData-&gt;AIAgent</title>
+<path fill="none" stroke="black" d="M122.91,-92.62C183.09,-76.09 285.53,-47.95 344.81,-31.67"/>
+<polygon fill="black" stroke="black" points="345.54,-35.1 354.25,-29.07 343.68,-28.35 345.54,-35.1"/>
+<text text-anchor="middle" x="308.66" y="-58.2" font-family="Times,serif" font-size="14.00">wasAttributedTo</text>
+</g>
+<!-- Campaign -->
+<g id="node8" class="node">
+<title>Campaign</title>
+<polygon fill="lightblue" stroke="black" points="164.83,-657.6 85.01,-657.6 85.01,-621.6 164.83,-621.6 164.83,-657.6"/>
+<text text-anchor="middle" x="124.92" y="-635.4" font-family="Helvetica,sans-Serif" font-size="14.00">Campaign</text>
+</g>
+<!-- Workflow -->
+<g id="node9" class="node">
+<title>Workflow</title>
+<polygon fill="lightblue" stroke="black" points="161.7,-568.8 88.14,-568.8 88.14,-532.8 161.7,-532.8 161.7,-568.8"/>
+<text text-anchor="middle" x="124.92" y="-546.6" font-family="Helvetica,sans-Serif" font-size="14.00">Workflow</text>
+</g>
+<!-- Campaign&#45;&gt;Workflow -->
+<g id="edge1" class="edge">
+<title>Campaign-&gt;Workflow</title>
+<path fill="none" stroke="black" d="M124.92,-621.45C124.92,-609.76 124.92,-593.99 124.92,-580.42"/>
+<polygon fill="black" stroke="black" points="128.42,-580.72 124.92,-570.72 121.42,-580.72 128.42,-580.72"/>
+<text text-anchor="middle" x="158.74" y="-591" font-family="Times,serif" font-size="14.00">hadMember</text>
+</g>
+<!-- Workflow&#45;&gt;Task -->
+<g id="edge2" class="edge">
+<title>Workflow-&gt;Task</title>
+<path fill="none" stroke="black" d="M162.02,-550.8C191.74,-550.8 233.27,-550.8 263.36,-550.8"/>
+<polygon fill="black" stroke="black" points="263.17,-554.3 273.17,-550.8 263.17,-547.3 263.17,-554.3"/>
+<text text-anchor="middle" x="218.31" y="-558" font-family="Times,serif" font-size="14.00">hadMember</text>
+</g>
+<!-- Workflow&#45;&gt;ToolExecution -->
+<g id="edge3" class="edge">
+<title>Workflow-&gt;ToolExecution</title>
+<path fill="none" stroke="black" d="M118.7,-532.33C112.06,-510.84 103.9,-473.56 114.92,-444 137.13,-384.41 195.5,-336.2 234.73,-309.17"/>
+<polygon fill="black" stroke="black" points="236.66,-312.09 243,-303.6 232.75,-306.28 236.66,-312.09"/>
+<text text-anchor="middle" x="165.2" y="-413.4" font-family="Times,serif" font-size="14.00">hadMember</text>
+</g>
+<!-- Task&#45;&gt;DomainData -->
+<g id="edge4" class="edge">
+<title>Task-&gt;DomainData</title>
+<path fill="none" stroke="black" d="M298.25,-532.41C295.09,-521.28 289.48,-507.23 279.92,-498 271.16,-489.54 260.13,-483.06 248.8,-478.13"/>
+<polygon fill="black" stroke="black" points="250.37,-474.99 239.78,-474.58 247.81,-481.5 250.37,-474.99"/>
+<text text-anchor="middle" x="304.12" y="-502.2" font-family="Times,serif" font-size="14.00">used</text>
+</g>
+<!-- ToolExecution&#45;&gt;DataObject -->
+<g id="edge7" class="edge">
+<title>ToolExecution-&gt;DataObject</title>
+<path fill="none" stroke="black" d="M310.58,-302.78C324.57,-311.84 335.41,-323.81 327.92,-337.2 324.72,-342.91 320.17,-347.81 315.01,-351.97"/>
+<polygon fill="black" stroke="black" points="313.37,-348.85 307.14,-357.42 317.35,-354.6 313.37,-348.85"/>
+<text text-anchor="middle" x="343.19" y="-324.6" font-family="Times,serif" font-size="14.00">used</text>
+</g>
+<!-- ToolExecution&#45;&gt;LLMInvocation -->
+<g id="edge16" class="edge">
+<title>ToolExecution-&gt;LLMInvocation</title>
+<path fill="none" stroke="black" d="M272.92,-266.25C272.92,-254.56 272.92,-238.79 272.92,-225.22"/>
+<polygon fill="black" stroke="black" points="276.42,-225.52 272.92,-215.52 269.42,-225.52 276.42,-225.52"/>
+<text text-anchor="middle" x="318.02" y="-235.8" font-family="Times,serif" font-size="14.00">wasInformedBy</text>
+</g>
+<!-- ToolExecution&#45;&gt;AIAgent -->
+<g id="edge15" class="edge">
+<title>ToolExecution-&gt;AIAgent</title>
+<path fill="none" stroke="black" d="M313,-278.36C332.31,-273.78 354.01,-265.05 366.92,-248.4 390.69,-217.72 392.22,-101.98 391.57,-47.58"/>
+<polygon fill="black" stroke="black" points="395.07,-47.81 391.41,-37.87 388.07,-47.92 395.07,-47.81"/>
+<text text-anchor="middle" x="445.03" y="-147" font-family="Times,serif" font-size="14.00">wasAssociatedWith</text>
+</g>
+<!-- LLMInvocation&#45;&gt;Model -->
+<g id="edge6" class="edge">
+<title>LLMInvocation-&gt;Model</title>
+<path fill="none" stroke="black" d="M288.18,-177.24C292.39,-171.84 296.69,-165.7 299.92,-159.6 303.79,-152.29 307,-143.94 309.54,-136.11"/>
+<polygon fill="black" stroke="black" points="312.86,-137.22 312.37,-126.64 306.16,-135.22 312.86,-137.22"/>
+<text text-anchor="middle" x="319.93" y="-147" font-family="Times,serif" font-size="14.00">used</text>
+</g>
+<!-- LLMInvocation&#45;&gt;Prompt -->
+<g id="edge5" class="edge">
+<title>LLMInvocation-&gt;Prompt</title>
+<path fill="none" stroke="black" d="M267.8,-177.23C264.16,-166.65 258.55,-153.2 250.92,-142.8 247.63,-138.32 243.66,-134.03 239.51,-130.1"/>
+<polygon fill="black" stroke="black" points="241.84,-127.49 232.01,-123.54 237.24,-132.76 241.84,-127.49"/>
+<text text-anchor="middle" x="273.1" y="-147" font-family="Times,serif" font-size="14.00">used</text>
+</g>
+</g>
+</svg>

flowcept-0.9.10/docs/large_data.rst

@@ -0,0 +1,20 @@
+# Handling Large Data
+
+As a provenance-based tool, Flowcept leverages provenance data (typically seen as metadata) to keep track of data, including large datasets or data files, workflows, and ML models.
+
+There are various ways to handle large data in Flowcept
+
+## Sending large blob data as messages
+
+### Sending as files
+
+Importance of MimeType and the custom_metadata for files 
+
+## Managing blob data
+
+## Managing PyTorch models
+
+
+
+
+