flowcept 0.8.10__tar.gz → 0.8.11__tar.gz

{flowcept-0.8.10 → flowcept-0.8.11}/.github/workflows/run-tests-py313.yml

@@ -47,7 +47,7 @@ jobs:
 
       - name: Test with pytest and redis, ignoring the ones that (as of Dec 21 2024) don't work on py3.13)
         run: |
-          pytest  --ignore=tests/adapters/test_tensorboard.py --ignore tests/instrumentation_tests/ml_tests/
+          pytest  --ignore=tests/adapters/test_tensorboard.py --ignore=tests/adapters/test_broker.py --ignore tests/instrumentation_tests/ml_tests/
 
       - name: Shut down docker compose
         run: make services-stop-mongo

{flowcept-0.8.10 → flowcept-0.8.11}/.github/workflows/run_examples.sh

@@ -81,7 +81,8 @@ echo "Using examples directory: $EXAMPLES_DIR"
 echo "With Mongo? ${WITH_MONGO}"
 
 # Define the test cases
-default_tests=("instrumented_simple_example.py" "instrumented_loop_example.py" "distributed_consumer_example.py" "dask_example.py" "mlflow_example.py" "tensorboard_example.py" "single_layer_perceptron_example.py" "llm_complex/llm_main_example.py" "unmanaged/main.py")
+default_tests=("instrumented_simple_example.py" "instrumented_loop_example.py" "distributed_consumer_example.py" "dask_example.py" "mlflow_example.py" "single_layer_perceptron_example.py" "llm_complex/llm_main_example.py" "unmanaged/main.py")
+# Removing tensorboard_example.py from the list above while the dataset link is not fixed.
 
 # Use the third argument if provided, otherwise use default tests
 if [[ -n "$3" ]]; then

{flowcept-0.8.10 → flowcept-0.8.11}/.gitignore

@@ -31,3 +31,4 @@ launch.json
 core.*
 *.csv
 
+uv.lock

{flowcept-0.8.10 → flowcept-0.8.11}/Makefile

@@ -83,10 +83,17 @@ tests-in-container-kafka:
 liveness:
 	python -c 'from flowcept import Flowcept; print(Flowcept.services_alive())'
 
+dev_agent:
+	mcp dev src/flowcept/flowceptor/adapters/agents/flowcept_agent.py
+
+install_dev_agent: # Run this to fix python env problems in the MCP studio env
+	mcp install src/flowcept/flowceptor/adapters/agents/flowcept_agent.py
+
+
 # Run unit tests using pytest
 .PHONY: tests
 tests:
-	pytest
+	pytest --ignore=tests/adapters/test_tensorboard.py
 
 .PHONY: tests-notebooks
 tests-notebooks:

{flowcept-0.8.10 → flowcept-0.8.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.8.10
+Version: 0.8.11
 Summary: Capture and query workflow provenance data using data observability
 Project-URL: GitHub, https://github.com/ORNL/flowcept
 Author: Oak Ridge National Laboratory
@@ -29,6 +29,7 @@ Requires-Dist: furo; extra == 'all'
 Requires-Dist: jupyterlab; extra == 'all'
 Requires-Dist: mlflow-skinny; extra == 'all'
 Requires-Dist: nbmake; extra == 'all'
+Requires-Dist: paho-mqtt; extra == 'all'
 Requires-Dist: pika; extra == 'all'
 Requires-Dist: plotly; extra == 'all'
 Requires-Dist: pymongo; extra == 'all'
@@ -65,6 +66,12 @@ Requires-Dist: furo; extra == 'docs'
 Requires-Dist: sphinx; extra == 'docs'
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka<=2.8.0; extra == 'kafka'
+Provides-Extra: llm-agent
+Requires-Dist: faiss-cpu; extra == 'llm-agent'
+Requires-Dist: langchain-community; extra == 'llm-agent'
+Requires-Dist: mcp[cli]; extra == 'llm-agent'
+Requires-Dist: sentence-transformers; extra == 'llm-agent'
+Requires-Dist: tiktoken; extra == 'llm-agent'
 Provides-Extra: ml-dev
 Requires-Dist: datasets==2.17.0; extra == 'ml-dev'
 Requires-Dist: nltk; extra == 'ml-dev'
@@ -80,6 +87,8 @@ Requires-Dist: sqlalchemy; extra == 'mlflow'
 Requires-Dist: watchdog; extra == 'mlflow'
 Provides-Extra: mongo
 Requires-Dist: pymongo; extra == 'mongo'
+Provides-Extra: mqtt
+Requires-Dist: paho-mqtt; extra == 'mqtt'
 Provides-Extra: nvidia
 Requires-Dist: nvidia-ml-py; extra == 'nvidia'
 Provides-Extra: tensorboard

flowcept-0.8.11/examples/agents/aec_agent_mock.py

@@ -0,0 +1,144 @@
+import json
+import os
+from typing import Dict
+import textwrap
+
+import uvicorn
+from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
+from mcp.server.fastmcp import FastMCP
+from mcp.server.fastmcp.prompts import base
+
+from flowcept.configs import AGENT
+from flowcept.flowcept_api.flowcept_controller import Flowcept
+from flowcept.flowceptor.adapters.agents.agents_utils import convert_mcp_to_langchain
+from flowcept.flowceptor.adapters.agents.flowcept_llm_prov_capture import invoke_llm, add_preamble_to_response
+from flowcept.flowceptor.adapters.agents.prompts import get_question_prompt,  BASE_SINGLETASK_PROMPT
+from flowcept.commons.utils import get_utc_now
+
+os.environ["SAMBASTUDIO_URL"] = AGENT.get("llm_server_url")
+os.environ["SAMBASTUDIO_API_KEY"] = AGENT.get("api_key")
+
+agent_controller = BaseAgentContextManager()
+mcp = FastMCP("AnC_Agent_mock", require_session=True, lifespan=agent_controller.lifespan)
+
+
+#################################################
+# PROMPTS
+#################################################
+
+@mcp.prompt()
+def single_task_used_generated_prompt(task_data: Dict, question: str) -> list[base.Message]:
+    """
+    Generates a prompt to ask about one particular task.
+    """
+    msgs = BASE_SINGLETASK_PROMPT.copy()
+    msgs.append(get_question_prompt(question))
+    msgs.append(base.UserMessage(f"This is the task object I need you to focus on: \n {task_data}\n"))
+    return msgs
+
+
+@mcp.prompt()
+def adamantine_prompt(layer: int, simulation_output: Dict, question: str) -> list[base.Message]:
+    control_options = simulation_output.get("control_options")
+    l2_error = simulation_output.get("l2_error")
+
+    control_options_str = ""
+    for o in range(len(control_options)):
+        control_options_str += f"Option {o + 1}: {control_options[o]}\n"
+
+    l2_error_str = ""
+    for o in range(len(l2_error)):
+        l2_error_str += f"Option {o + 1}: {l2_error[o]}\n"
+
+    prompt = textwrap.dedent(f"""\
+    SUMMARY OF CURRENT STATE: Currently, the printer is printing layer {layer}. You need to make a control decision for layer {layer + 2}. It is currently {get_utc_now()}.
+
+    CONTROL OPTIONS: 
+    {control_options_str}
+
+    AUTOMATED ANALYSIS FROM SIMULATIONS:
+    Full volume L2 error (lower is better)
+
+    {l2_error_str}
+    """).strip()
+
+    return [
+        base.UserMessage(prompt),
+        base.UserMessage(f"Based on this provided information, here is the question: {question}")
+    ]
+
+
+#################################################
+# TOOLS
+#################################################
+
+@mcp.tool()
+def get_latest(n: int = None) -> str:
+    """
+    Return the latest task(s) as a JSON string.
+    """
+    ctx = mcp.get_context()
+    tasks = ctx.request_context.lifespan_context.tasks
+    if not tasks:
+        return "No tasks available."
+    if n is None:
+        return json.dumps(tasks[-1])
+    return json.dumps(tasks[-n])
+
+
+@mcp.tool()
+def check_liveness() -> str:
+    """
+    Check if the agent is running.
+    """
+
+    return f"I'm {mcp.name} and I'm ready!"
+
+
+@mcp.tool()
+def check_llm() -> str:
+    """
+    Check if the agent can talk to the LLM service.
+    """
+
+    messages = [base.UserMessage(f"Hi, are you working properly?")]
+
+    langchain_messages = convert_mcp_to_langchain(messages)
+    response = invoke_llm(langchain_messages)
+    result = add_preamble_to_response(response, mcp)
+
+    return result
+
+
+@mcp.tool()
+def adamantine_ask_about_latest_iteration(question) -> str:
+    ctx = mcp.get_context()
+    tasks = ctx.request_context.lifespan_context.tasks
+    if not tasks:
+        return "No tasks available."
+    task_data = tasks[-1]
+
+    layer = task_data.get('used').get('layer_number', 0)
+    simulation_output = task_data.get('generated')
+
+    messages = adamantine_prompt(layer, simulation_output, question)
+
+    langchain_messages = convert_mcp_to_langchain(messages)
+
+    response = invoke_llm(langchain_messages)
+    result = add_preamble_to_response(response, mcp, task_data)
+    return result
+
+
+def main():
+    """
+    Start the MCP server.
+    """
+    f = Flowcept(start_persistence=False, save_workflow=False, check_safe_stops=False).start()
+    f.logger.info(f"This section's workflow_id={Flowcept.current_workflow_id}")
+    setattr(mcp, "workflow_id", f.current_workflow_id)
+    uvicorn.run(mcp.streamable_http_app, host="0.0.0.0", port=8000, lifespan="on")
+
+
+if __name__ == "__main__":
+    main()

flowcept-0.8.11/examples/agents/opt_driver_mock.py

@@ -0,0 +1,49 @@
+from time import sleep
+import random
+from flowcept.flowcept_api.flowcept_controller import Flowcept
+from flowcept.flowceptor.consumers.agent.client_agent import run_tool
+from flowcept.instrumentation.flowcept_task import flowcept_task
+
+
+@flowcept_task
+def simulate_layer(layer_number: int):
+    power_arr = [0, 15, 25, 50, 75, 100, 125, 150, 175, 200, 250, 300, 350]  # floating number from 0 to 350
+    dwell_arr = list(range(10, 121, 5))
+
+    control_options = [
+        f"{power_arr[random.randint(0, len(power_arr)-1)]}W power reheat pass, {power_arr[random.randint(0, len(power_arr)-1)]}s dwell",
+        f"{dwell_arr[random.randint(0, len(dwell_arr)-1)]}s dwell, {dwell_arr[random.randint(0, len(dwell_arr)-1)]}s dwell",
+        f"{dwell_arr[random.randint(0, len(dwell_arr)-1)]}s dwell, {power_arr[random.randint(0, len(power_arr)-1)]}W power reheat pass",
+        f"{power_arr[random.randint(0, len(power_arr)-1)]}W power reheat pass, {power_arr[random.randint(0, len(power_arr)-1)]}W power reheat pass"
+    ]
+    l2_error = [
+        random.randint(100, 350),
+        random.randint(100, 500),
+        random.randint(100, 500),
+        random.randint(100, 600)
+    ]
+    sleep(5/(layer_number+1))
+    return {"control_options": control_options, "l2_error": l2_error}
+
+
+try:
+    print(run_tool("check_liveness"))
+except Exception as e:
+    print(e)
+    pass
+
+
+def adaptive_control_workflow(config):
+    for i in range(config["max_layers"]):
+        print()
+        print(f"Starting simulation for Layer {i}; ", end='')
+        simulation_output = simulate_layer(layer_number=i)
+        print(simulation_output)
+
+
+if __name__ == "__main__":
+    with Flowcept(start_persistence=False, save_workflow=False, check_safe_stops=False):
+        config = {"max_layers": 5}
+        adaptive_control_workflow(config)
+
+

flowcept-0.8.11/examples/consumers/simple_consumer.py

@@ -0,0 +1,23 @@
+from typing import Dict
+
+from flowcept.flowceptor.consumers.base_consumer import BaseConsumer
+
+
+class MyConsumer(BaseConsumer):
+
+    def __init__(self):
+        super().__init__()
+
+    def message_handler(self, msg_obj: Dict) -> bool:
+        if msg_obj.get('type', '') == 'task':
+            print(msg_obj)
+        else:
+            print(f"We got a msg with different type: {msg_obj.get("type", None)}")
+        return True
+
+
+if __name__ == "__main__":
+
+    print("Starting consumer indefinitely. Press ctrl+c to stop")
+    consumer = MyConsumer()
+    consumer.start(daemon=False)

{flowcept-0.8.10 → flowcept-0.8.11}/examples/mlflow_example.py

@@ -31,7 +31,7 @@ if __name__ == "__main__":
             mlflow.log_params({"param1": 1})
             mlflow.log_params({"param2": 2})
             mlflow.log_metric("metric1", 10)
-            run_id = run.info.run_uuid
+            run_id = run.info.run_id
     task = Flowcept.db.query(filter={"task_id": run_id})[0]
     assert task["status"] == "FINISHED"
     assert "param1" in task["used"]

{flowcept-0.8.10 → flowcept-0.8.11}/notebooks/mlflow.ipynb

@@ -97,7 +97,7 @@
     "        # Actual training code would go here\n",
     "        print(\"Generated training metadata.\")\n",
     "        mlflow.log_metric(\"loss\", 0.04)\n",
-    "        return run.info.run_uuid"
+    "        return run.info.run_id"
    ]
   },
   {
@@ -413,7 +413,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.12.9"
   }
  },
  "nbformat": 4,

{flowcept-0.8.10 → flowcept-0.8.11}/pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
     "redis",
     "requests",
     "lmdb",
-    "pyarrow"
+    "pyarrow",
 ]
 authors = [{name = "Oak Ridge National Laboratory"}]
 description = "Capture and query workflow provenance data using data observability"
@@ -62,7 +62,10 @@ docs = ["sphinx", "furo"]
 kafka = ["confluent-kafka<=2.8.0"]  # As of today, 2/28/2025, version 2.8.1 is stale. When this gets fixed, let's remove the version constraint. https://pypi.org/project/confluent-kafka/#history
 mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog"]
 nvidia = ["nvidia-ml-py"]
+mqtt = ["paho-mqtt"]
 tensorboard = ["tensorboard", "tensorflow", "tbparse"]
+llm_agent = ["mcp[cli]", "langchain_community", "sentence-transformers", "tiktoken", "faiss-cpu"]
+
 dev = [
     "flowcept[docs]",
     "jupyterlab",
@@ -88,6 +91,7 @@ all = [
     "flowcept[dask]",
     "flowcept[kafka]",
     "flowcept[mlflow]",
+    "flowcept[mqtt]",
     "flowcept[tensorboard]",
     "flowcept[dev]",
 ]
@@ -105,6 +109,12 @@ ignore = ["D200", "D212", "D105", "D401", "D205", "D100"]
 [tool.ruff.lint.pydocstyle]
 convention = "numpy"
 
+[tool.uv.workspace]
+members = [
+    "tmp_tests/mcp-server-demo",
+    "tmp_tests/mcp-server-demo/mcp-server-demo",
+]
+
 [tool.hatch.build.targets.wheel]
 packages = ["src/flowcept"]
 

{flowcept-0.8.10 → flowcept-0.8.11}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.8.10 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.8.11 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
@@ -39,18 +39,22 @@ experiment:
 mq:
   type: redis  # or kafka or mofka; Please adjust the port (kafka's default is 9092; redis is 6379). If mofka, adjust the group_file.
   host: localhost
-  # instances: ["localhost:6379"] # We can have multiple redis instances being accessed by the consumers but each interceptor will currently access one single redis.
+  # uri: ?
+  # instances: ["localhost:6379"] # We can have multiple MQ instances being accessed by the consumers but each interceptor will currently access one single MQ..
   port: 6379
   # group_file: mofka.json
   channel: interception
   buffer_size: 50
   insertion_buffer_time_secs: 5
   timing: false
+  # uri: use Redis connection uri here
   chunk_size: -1  # use 0 or -1 to disable this. Or simply omit this from the config file.
+  same_as_kvdb: false # Set this to true if you are using the same Redis instance both as an MQ and as the KV_DB. In that case, no need to repeat connection parameters in MQ. Use only what you define in KV_DB.
 
-kv_db:  # You can optionally use KV == MQ if MQ is Redis. Otherwise, these will be the Redis instance credentials.
+kv_db:
   host: localhost
   port: 6379
+  enabled: true
   # uri: use Redis connection uri here
 
 web_server:
@@ -75,6 +79,15 @@ db_buffer:
   stop_max_trials: 240    # Maximum number of trials before giving up when waiting for a fully safe stop (i.e., all records have been inserted as expected).
   stop_trials_sleep: 0.01   # Sleep duration (in seconds) between trials when waiting for a fully safe stop.
 
+agent:
+  enabled: false
+  mcp_host: localhost
+  mcp_port: 8000
+  llm_server_url: '?'
+  api_key: '?'
+  model: '?'
+  model_kwargs: {}
+
 databases:
 
   lmdb:
@@ -90,16 +103,26 @@ databases:
 
 adapters:
   # For each key below, you can have multiple instances. Like mlflow1, mlflow2; zambeze1, zambeze2. Use an empty dict, {}, if you won't use any adapter.
-  zambeze:
-    kind: zambeze
-    host: localhost
-    port: 5672
-    queue_names:
-      - hello
-      - hello2
-#    key_values_to_filter:
-#      - key: activity_status
-#        value: CREATED
+
+  broker_mqtt:
+    kind: broker
+    host: h
+    port: 30011
+    protocol: mqtt3.1.1
+    queues: ["#"]
+    username: postman
+    password: p
+    qos: 2
+    task_subtype: intersect_msg
+    tracked_keys:
+      used: payload
+      generated: ~
+      custom_metadata: [headers, msgId]
+      activity_id: operationId
+      submitted_at: ~
+      started_at: ~
+      ended_at: ~
+      registered_at: ~
 
   mlflow:
     kind: mlflow
@@ -122,3 +145,4 @@ adapters:
     worker_should_get_output: true
     scheduler_create_timestamps: true
     worker_create_timestamps: false
+

flowcept-0.8.11/resources/simple_redis_consumer.py

@@ -0,0 +1,33 @@
+import redis
+import msgpack
+
+from flowcept.commons.daos.mq_dao.mq_dao_redis import MQDaoRedis
+from flowcept.configs import MQ_HOST, MQ_PORT, MQ_CHANNEL, KVDB_URI
+# Connect to Redis
+redis_client = (
+    redis.from_url(KVDB_URI) if KVDB_URI else redis.Redis(host=MQ_HOST, port=MQ_PORT, db=0)
+)
+# Subscribe to a channel
+pubsub = redis_client.pubsub()
+pubsub.subscribe(MQ_CHANNEL)
+
+print("Listening for messages...")
+
+
+for message in pubsub.listen():
+    print()
+    print("Received a message!", end=' ')
+    if message and message["type"] in MQDaoRedis.MESSAGE_TYPES_IGNORE:
+        continue
+
+    if not isinstance(message["data"], (bytes, bytearray)):
+        print(
+            f"Skipping message with unexpected data type: {type(message["data"])} - {message["data"]}")
+        continue
+
+    try:
+        msg_obj = msgpack.loads(message["data"], strict_map_key=False)
+        msg_type = msg_obj.get("type", None)
+        print(msg_type)
+    except Exception as e:
+        print(e)