flowcept 0.8.9__tar.gz → 0.8.11__tar.gz

{flowcept-0.8.9 → flowcept-0.8.11}/.github/workflows/checks.yml

@@ -1,4 +1,4 @@
-name: Linter, formatter, and docs checks
+name: Code and doc checks
 
 on: pull_request
 

{flowcept-0.8.9 → flowcept-0.8.11}/.github/workflows/run-tests-py313.yml

@@ -47,7 +47,7 @@ jobs:
 
       - name: Test with pytest and redis, ignoring the ones that (as of Dec 21 2024) don't work on py3.13)
         run: |
-          pytest  --ignore=tests/adapters/test_tensorboard.py --ignore tests/instrumentation_tests/ml_tests/
+          pytest  --ignore=tests/adapters/test_tensorboard.py --ignore=tests/adapters/test_broker.py --ignore tests/instrumentation_tests/ml_tests/
 
       - name: Shut down docker compose
         run: make services-stop-mongo

{flowcept-0.8.9 → flowcept-0.8.11}/.github/workflows/run_examples.sh

@@ -81,7 +81,8 @@ echo "Using examples directory: $EXAMPLES_DIR"
 echo "With Mongo? ${WITH_MONGO}"
 
 # Define the test cases
-default_tests=("instrumented_simple_example.py" "instrumented_loop_example.py" "distributed_consumer_example.py" "dask_example.py" "mlflow_example.py" "tensorboard_example.py" "single_layer_perceptron_example.py" "llm_complex/llm_main_example.py" "unmanaged/main.py")
+default_tests=("instrumented_simple_example.py" "instrumented_loop_example.py" "distributed_consumer_example.py" "dask_example.py" "mlflow_example.py" "single_layer_perceptron_example.py" "llm_complex/llm_main_example.py" "unmanaged/main.py")
+# Removing tensorboard_example.py from the list above while the dataset link is not fixed.
 
 # Use the third argument if provided, otherwise use default tests
 if [[ -n "$3" ]]; then

{flowcept-0.8.9 → flowcept-0.8.11}/.gitignore

@@ -31,3 +31,4 @@ launch.json
 core.*
 *.csv
 
+uv.lock

{flowcept-0.8.9 → flowcept-0.8.11}/Makefile

@@ -83,10 +83,17 @@ tests-in-container-kafka:
 liveness:
 	python -c 'from flowcept import Flowcept; print(Flowcept.services_alive())'
 
+dev_agent:
+	mcp dev src/flowcept/flowceptor/adapters/agents/flowcept_agent.py
+
+install_dev_agent: # Run this to fix python env problems in the MCP studio env
+	mcp install src/flowcept/flowceptor/adapters/agents/flowcept_agent.py
+
+
 # Run unit tests using pytest
 .PHONY: tests
 tests:
-	pytest
+	pytest --ignore=tests/adapters/test_tensorboard.py
 
 .PHONY: tests-notebooks
 tests-notebooks:

{flowcept-0.8.9 → flowcept-0.8.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowcept
-Version: 0.8.9
+Version: 0.8.11
 Summary: Capture and query workflow provenance data using data observability
 Project-URL: GitHub, https://github.com/ORNL/flowcept
 Author: Oak Ridge National Laboratory
@@ -29,6 +29,7 @@ Requires-Dist: furo; extra == 'all'
 Requires-Dist: jupyterlab; extra == 'all'
 Requires-Dist: mlflow-skinny; extra == 'all'
 Requires-Dist: nbmake; extra == 'all'
+Requires-Dist: paho-mqtt; extra == 'all'
 Requires-Dist: pika; extra == 'all'
 Requires-Dist: plotly; extra == 'all'
 Requires-Dist: pymongo; extra == 'all'
@@ -65,6 +66,12 @@ Requires-Dist: furo; extra == 'docs'
 Requires-Dist: sphinx; extra == 'docs'
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka<=2.8.0; extra == 'kafka'
+Provides-Extra: llm-agent
+Requires-Dist: faiss-cpu; extra == 'llm-agent'
+Requires-Dist: langchain-community; extra == 'llm-agent'
+Requires-Dist: mcp[cli]; extra == 'llm-agent'
+Requires-Dist: sentence-transformers; extra == 'llm-agent'
+Requires-Dist: tiktoken; extra == 'llm-agent'
 Provides-Extra: ml-dev
 Requires-Dist: datasets==2.17.0; extra == 'ml-dev'
 Requires-Dist: nltk; extra == 'ml-dev'
@@ -80,6 +87,8 @@ Requires-Dist: sqlalchemy; extra == 'mlflow'
 Requires-Dist: watchdog; extra == 'mlflow'
 Provides-Extra: mongo
 Requires-Dist: pymongo; extra == 'mongo'
+Provides-Extra: mqtt
+Requires-Dist: paho-mqtt; extra == 'mqtt'
 Provides-Extra: nvidia
 Requires-Dist: nvidia-ml-py; extra == 'nvidia'
 Provides-Extra: tensorboard
@@ -112,7 +121,9 @@ Description-Content-Type: text/markdown
 
 ## Overview
 
-Flowcept is a runtime data integration system that captures and queries workflow provenance with minimal or no code changes. It unifies data across diverse workflows and tools, enabling integrated analysis and insights, especially in federated environments. Designed for scenarios involving critical data from multiple workflows, Flowcept seamlessly integrates data at runtime, providing a unified view for end-to-end monitoring and analysis, and enhanced support for Machine Learning (ML) workflows.
+Flowcept is a runtime data integration system that captures and queries workflow provenance with minimal or no code changes. It unifies data from diverse workflows and tools, enabling integrated analysis and insights, especially in federated environments. 
+
+Designed for scenarios involving critical data from multiple workflows, Flowcept supports end-to-end monitoring, analysis, querying, and enhanced support for Machine Learning (ML) workflows.
 
 ## Features
 
@@ -135,8 +146,9 @@ Notes:
   - TensorBoard
 - Python scripts can be easily instrumented via `@decorators` using `@flowcept_task` (for generic Python method) or `@torch_task` (for methods that encapsulate PyTorch model manipulation, such as training or evaluation). 
 - Currently supported MQ systems:
-  - Kafka
-  - Redis
+  - [Kafka](https://kafka.apache.org)
+  - [Redis](https://redis.io)
+  - [Mofka](https://mofka.readthedocs.io)
 - Currently supported database systems:
   - MongoDB
   - Lightning Memory-Mapped Database (lightweight file-only database system)
@@ -181,7 +193,7 @@ If you want to install all optional dependencies, use:
 pip install flowcept[all]
 ```
 
-This is a convenient way to ensure all adapters are available, but it may install dependencies you don't need.
+This is useful mostly for Flowcept developers. Please avoid installing like this if you can, as it may install several dependencies you will never use.
 
 ### 4. Installing from Source
 To install Flowcept from the source repository:
@@ -363,7 +375,7 @@ Some unit tests utilize `torch==2.2.2`, `torchtext=0.17.2`, and `torchvision==0.
 
 ## Documentation
 
-Full documentation is available on [Read the Docs](https://myproject.readthedocs.io/en/latest/).
+Full documentation is available on [Read the Docs](https://flowcept.readthedocs.io/).
 
 ## Cite us
 

{flowcept-0.8.9 → flowcept-0.8.11}/README.md

@@ -22,7 +22,9 @@
 
 ## Overview
 
-Flowcept is a runtime data integration system that captures and queries workflow provenance with minimal or no code changes. It unifies data across diverse workflows and tools, enabling integrated analysis and insights, especially in federated environments. Designed for scenarios involving critical data from multiple workflows, Flowcept seamlessly integrates data at runtime, providing a unified view for end-to-end monitoring and analysis, and enhanced support for Machine Learning (ML) workflows.
+Flowcept is a runtime data integration system that captures and queries workflow provenance with minimal or no code changes. It unifies data from diverse workflows and tools, enabling integrated analysis and insights, especially in federated environments. 
+
+Designed for scenarios involving critical data from multiple workflows, Flowcept supports end-to-end monitoring, analysis, querying, and enhanced support for Machine Learning (ML) workflows.
 
 ## Features
 
@@ -45,8 +47,9 @@ Notes:
   - TensorBoard
 - Python scripts can be easily instrumented via `@decorators` using `@flowcept_task` (for generic Python method) or `@torch_task` (for methods that encapsulate PyTorch model manipulation, such as training or evaluation). 
 - Currently supported MQ systems:
-  - Kafka
-  - Redis
+  - [Kafka](https://kafka.apache.org)
+  - [Redis](https://redis.io)
+  - [Mofka](https://mofka.readthedocs.io)
 - Currently supported database systems:
   - MongoDB
   - Lightning Memory-Mapped Database (lightweight file-only database system)
@@ -91,7 +94,7 @@ If you want to install all optional dependencies, use:
 pip install flowcept[all]
 ```
 
-This is a convenient way to ensure all adapters are available, but it may install dependencies you don't need.
+This is useful mostly for Flowcept developers. Please avoid installing like this if you can, as it may install several dependencies you will never use.
 
 ### 4. Installing from Source
 To install Flowcept from the source repository:
@@ -273,7 +276,7 @@ Some unit tests utilize `torch==2.2.2`, `torchtext=0.17.2`, and `torchvision==0.
 
 ## Documentation
 
-Full documentation is available on [Read the Docs](https://myproject.readthedocs.io/en/latest/).
+Full documentation is available on [Read the Docs](https://flowcept.readthedocs.io/).
 
 ## Cite us
 

flowcept-0.8.11/docs/index.rst

@@ -0,0 +1,18 @@
+Flowcept
+========
+
+GitHub Repository: https://github.com/ornl/flowcept
+
+Flowcept is a runtime data integration system that captures and queries workflow provenance with minimal or no code changes. It unifies data from diverse workflows and tools, enabling integrated analysis and insights, especially in federated environments.
+
+Designed for scenarios involving critical data from multiple workflows, Flowcept supports end-to-end monitoring, analysis, querying, and enhanced support for Machine Learning (ML) workflows.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   getstarted
+   schemas
+   contributing
+   api-reference
+

flowcept-0.8.11/examples/agents/aec_agent_mock.py

@@ -0,0 +1,144 @@
+import json
+import os
+from typing import Dict
+import textwrap
+
+import uvicorn
+from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
+from mcp.server.fastmcp import FastMCP
+from mcp.server.fastmcp.prompts import base
+
+from flowcept.configs import AGENT
+from flowcept.flowcept_api.flowcept_controller import Flowcept
+from flowcept.flowceptor.adapters.agents.agents_utils import convert_mcp_to_langchain
+from flowcept.flowceptor.adapters.agents.flowcept_llm_prov_capture import invoke_llm, add_preamble_to_response
+from flowcept.flowceptor.adapters.agents.prompts import get_question_prompt,  BASE_SINGLETASK_PROMPT
+from flowcept.commons.utils import get_utc_now
+
+os.environ["SAMBASTUDIO_URL"] = AGENT.get("llm_server_url")
+os.environ["SAMBASTUDIO_API_KEY"] = AGENT.get("api_key")
+
+agent_controller = BaseAgentContextManager()
+mcp = FastMCP("AnC_Agent_mock", require_session=True, lifespan=agent_controller.lifespan)
+
+
+#################################################
+# PROMPTS
+#################################################
+
+@mcp.prompt()
+def single_task_used_generated_prompt(task_data: Dict, question: str) -> list[base.Message]:
+    """
+    Generates a prompt to ask about one particular task.
+    """
+    msgs = BASE_SINGLETASK_PROMPT.copy()
+    msgs.append(get_question_prompt(question))
+    msgs.append(base.UserMessage(f"This is the task object I need you to focus on: \n {task_data}\n"))
+    return msgs
+
+
+@mcp.prompt()
+def adamantine_prompt(layer: int, simulation_output: Dict, question: str) -> list[base.Message]:
+    control_options = simulation_output.get("control_options")
+    l2_error = simulation_output.get("l2_error")
+
+    control_options_str = ""
+    for o in range(len(control_options)):
+        control_options_str += f"Option {o + 1}: {control_options[o]}\n"
+
+    l2_error_str = ""
+    for o in range(len(l2_error)):
+        l2_error_str += f"Option {o + 1}: {l2_error[o]}\n"
+
+    prompt = textwrap.dedent(f"""\
+    SUMMARY OF CURRENT STATE: Currently, the printer is printing layer {layer}. You need to make a control decision for layer {layer + 2}. It is currently {get_utc_now()}.
+
+    CONTROL OPTIONS: 
+    {control_options_str}
+
+    AUTOMATED ANALYSIS FROM SIMULATIONS:
+    Full volume L2 error (lower is better)
+
+    {l2_error_str}
+    """).strip()
+
+    return [
+        base.UserMessage(prompt),
+        base.UserMessage(f"Based on this provided information, here is the question: {question}")
+    ]
+
+
+#################################################
+# TOOLS
+#################################################
+
+@mcp.tool()
+def get_latest(n: int = None) -> str:
+    """
+    Return the latest task(s) as a JSON string.
+    """
+    ctx = mcp.get_context()
+    tasks = ctx.request_context.lifespan_context.tasks
+    if not tasks:
+        return "No tasks available."
+    if n is None:
+        return json.dumps(tasks[-1])
+    return json.dumps(tasks[-n])
+
+
+@mcp.tool()
+def check_liveness() -> str:
+    """
+    Check if the agent is running.
+    """
+
+    return f"I'm {mcp.name} and I'm ready!"
+
+
+@mcp.tool()
+def check_llm() -> str:
+    """
+    Check if the agent can talk to the LLM service.
+    """
+
+    messages = [base.UserMessage(f"Hi, are you working properly?")]
+
+    langchain_messages = convert_mcp_to_langchain(messages)
+    response = invoke_llm(langchain_messages)
+    result = add_preamble_to_response(response, mcp)
+
+    return result
+
+
+@mcp.tool()
+def adamantine_ask_about_latest_iteration(question) -> str:
+    ctx = mcp.get_context()
+    tasks = ctx.request_context.lifespan_context.tasks
+    if not tasks:
+        return "No tasks available."
+    task_data = tasks[-1]
+
+    layer = task_data.get('used').get('layer_number', 0)
+    simulation_output = task_data.get('generated')
+
+    messages = adamantine_prompt(layer, simulation_output, question)
+
+    langchain_messages = convert_mcp_to_langchain(messages)
+
+    response = invoke_llm(langchain_messages)
+    result = add_preamble_to_response(response, mcp, task_data)
+    return result
+
+
+def main():
+    """
+    Start the MCP server.
+    """
+    f = Flowcept(start_persistence=False, save_workflow=False, check_safe_stops=False).start()
+    f.logger.info(f"This section's workflow_id={Flowcept.current_workflow_id}")
+    setattr(mcp, "workflow_id", f.current_workflow_id)
+    uvicorn.run(mcp.streamable_http_app, host="0.0.0.0", port=8000, lifespan="on")
+
+
+if __name__ == "__main__":
+    main()

flowcept-0.8.11/examples/agents/opt_driver_mock.py

@@ -0,0 +1,49 @@
+from time import sleep
+import random
+from flowcept.flowcept_api.flowcept_controller import Flowcept
+from flowcept.flowceptor.consumers.agent.client_agent import run_tool
+from flowcept.instrumentation.flowcept_task import flowcept_task
+
+
+@flowcept_task
+def simulate_layer(layer_number: int):
+    power_arr = [0, 15, 25, 50, 75, 100, 125, 150, 175, 200, 250, 300, 350]  # floating number from 0 to 350
+    dwell_arr = list(range(10, 121, 5))
+
+    control_options = [
+        f"{power_arr[random.randint(0, len(power_arr)-1)]}W power reheat pass, {power_arr[random.randint(0, len(power_arr)-1)]}s dwell",
+        f"{dwell_arr[random.randint(0, len(dwell_arr)-1)]}s dwell, {dwell_arr[random.randint(0, len(dwell_arr)-1)]}s dwell",
+        f"{dwell_arr[random.randint(0, len(dwell_arr)-1)]}s dwell, {power_arr[random.randint(0, len(power_arr)-1)]}W power reheat pass",
+        f"{power_arr[random.randint(0, len(power_arr)-1)]}W power reheat pass, {power_arr[random.randint(0, len(power_arr)-1)]}W power reheat pass"
+    ]
+    l2_error = [
+        random.randint(100, 350),
+        random.randint(100, 500),
+        random.randint(100, 500),
+        random.randint(100, 600)
+    ]
+    sleep(5/(layer_number+1))
+    return {"control_options": control_options, "l2_error": l2_error}
+
+
+try:
+    print(run_tool("check_liveness"))
+except Exception as e:
+    print(e)
+    pass
+
+
+def adaptive_control_workflow(config):
+    for i in range(config["max_layers"]):
+        print()
+        print(f"Starting simulation for Layer {i}; ", end='')
+        simulation_output = simulate_layer(layer_number=i)
+        print(simulation_output)
+
+
+if __name__ == "__main__":
+    with Flowcept(start_persistence=False, save_workflow=False, check_safe_stops=False):
+        config = {"max_layers": 5}
+        adaptive_control_workflow(config)
+
+

flowcept-0.8.11/examples/consumers/simple_consumer.py

@@ -0,0 +1,23 @@
+from typing import Dict
+
+from flowcept.flowceptor.consumers.base_consumer import BaseConsumer
+
+
+class MyConsumer(BaseConsumer):
+
+    def __init__(self):
+        super().__init__()
+
+    def message_handler(self, msg_obj: Dict) -> bool:
+        if msg_obj.get('type', '') == 'task':
+            print(msg_obj)
+        else:
+            print(f"We got a msg with different type: {msg_obj.get("type", None)}")
+        return True
+
+
+if __name__ == "__main__":
+
+    print("Starting consumer indefinitely. Press ctrl+c to stop")
+    consumer = MyConsumer()
+    consumer.start(daemon=False)

{flowcept-0.8.9 → flowcept-0.8.11}/examples/mlflow_example.py

@@ -31,7 +31,7 @@ if __name__ == "__main__":
             mlflow.log_params({"param1": 1})
             mlflow.log_params({"param2": 2})
             mlflow.log_metric("metric1", 10)
-            run_id = run.info.run_uuid
+            run_id = run.info.run_id
     task = Flowcept.db.query(filter={"task_id": run_id})[0]
     assert task["status"] == "FINISHED"
     assert "param1" in task["used"]

{flowcept-0.8.9 → flowcept-0.8.11}/notebooks/mlflow.ipynb

@@ -97,7 +97,7 @@
     "        # Actual training code would go here\n",
     "        print(\"Generated training metadata.\")\n",
     "        mlflow.log_metric(\"loss\", 0.04)\n",
-    "        return run.info.run_uuid"
+    "        return run.info.run_id"
    ]
   },
   {
@@ -413,7 +413,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.12.9"
   }
  },
  "nbformat": 4,

{flowcept-0.8.9 → flowcept-0.8.11}/pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
     "redis",
     "requests",
     "lmdb",
-    "pyarrow"
+    "pyarrow",
 ]
 authors = [{name = "Oak Ridge National Laboratory"}]
 description = "Capture and query workflow provenance data using data observability"
@@ -62,7 +62,10 @@ docs = ["sphinx", "furo"]
 kafka = ["confluent-kafka<=2.8.0"]  # As of today, 2/28/2025, version 2.8.1 is stale. When this gets fixed, let's remove the version constraint. https://pypi.org/project/confluent-kafka/#history
 mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog"]
 nvidia = ["nvidia-ml-py"]
+mqtt = ["paho-mqtt"]
 tensorboard = ["tensorboard", "tensorflow", "tbparse"]
+llm_agent = ["mcp[cli]", "langchain_community", "sentence-transformers", "tiktoken", "faiss-cpu"]
+
 dev = [
     "flowcept[docs]",
     "jupyterlab",
@@ -88,6 +91,7 @@ all = [
     "flowcept[dask]",
     "flowcept[kafka]",
     "flowcept[mlflow]",
+    "flowcept[mqtt]",
     "flowcept[tensorboard]",
     "flowcept[dev]",
 ]
@@ -105,8 +109,17 @@ ignore = ["D200", "D212", "D105", "D401", "D205", "D100"]
 [tool.ruff.lint.pydocstyle]
 convention = "numpy"
 
+[tool.uv.workspace]
+members = [
+    "tmp_tests/mcp-server-demo",
+    "tmp_tests/mcp-server-demo/mcp-server-demo",
+]
+
 [tool.hatch.build.targets.wheel]
 packages = ["src/flowcept"]
 
 [tool.hatch.build.targets.wheel.force-include]
 "resources/sample_settings.yaml" = "resources/sample_settings.yaml"
+
+[project.scripts]
+flowcept = "flowcept.cli:main"

{flowcept-0.8.9 → flowcept-0.8.11}/resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.8.9 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.8.11 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
@@ -25,7 +25,6 @@ telemetry_capture: # This toggles each individual type of telemetry capture. GPU
 
 instrumentation:
   enabled: true # This toggles data capture for instrumentation.
-  singleton: true # Use a single instrumentation instance per process. Defaults to true
   torch:
     what: parent_and_children # Scope of instrumentation: "parent_only" -- will capture only at the main model level, "parent_and_children" -- will capture the inner layers, or ~ (disable).
     children_mode: telemetry_and_tensor_inspection   # What to capture if parent_and_children is chosen in the scope. Possible values: "tensor_inspection" (i.e., tensor metadata), "telemetry", "telemetry_and_tensor_inspection"
@@ -40,18 +39,22 @@ experiment:
 mq:
   type: redis  # or kafka or mofka; Please adjust the port (kafka's default is 9092; redis is 6379). If mofka, adjust the group_file.
   host: localhost
-  # instances: ["localhost:6379"] # We can have multiple redis instances being accessed by the consumers but each interceptor will currently access one single redis.
+  # uri: ?
+  # instances: ["localhost:6379"] # We can have multiple MQ instances being accessed by the consumers but each interceptor will currently access one single MQ..
   port: 6379
   # group_file: mofka.json
   channel: interception
   buffer_size: 50
   insertion_buffer_time_secs: 5
   timing: false
+  # uri: use Redis connection uri here
   chunk_size: -1  # use 0 or -1 to disable this. Or simply omit this from the config file.
+  same_as_kvdb: false # Set this to true if you are using the same Redis instance both as an MQ and as the KV_DB. In that case, no need to repeat connection parameters in MQ. Use only what you define in KV_DB.
 
 kv_db:
   host: localhost
   port: 6379
+  enabled: true
   # uri: use Redis connection uri here
 
 web_server:
@@ -59,9 +62,9 @@ web_server:
   port: 5000
 
 sys_metadata:
-  environment_id: "laptop"
+  environment_id: "laptop"   # We use this to keep track of the environment used to run an experiment. Typical values include the cluster name, but it can be anything that you think will help identify your experimentation environment.
 
-extra_metadata:
+extra_metadata: # We use this to store any extra metadata you want to keep track of during an experiment.
   place_holder: ""
 
 analytics:
@@ -70,13 +73,20 @@ analytics:
     generated.accuracy: maximum_first
 
 db_buffer:
-  adaptive_buffer_size: true
-  insertion_buffer_time_secs: 5
-  max_buffer_size: 50
-  min_buffer_size: 10
-  remove_empty_fields: false
-  stop_max_trials: 240
-  stop_trials_sleep: 0.01
+  insertion_buffer_time_secs: 5   # Time interval (in seconds) to buffer incoming records before flushing to the database
+  buffer_size: 50    # Maximum number of records to hold in the buffer before forcing a flush
+  remove_empty_fields: false    # If true, fields with null/empty values will be removed before insertion
+  stop_max_trials: 240    # Maximum number of trials before giving up when waiting for a fully safe stop (i.e., all records have been inserted as expected).
+  stop_trials_sleep: 0.01   # Sleep duration (in seconds) between trials when waiting for a fully safe stop.
+
+agent:
+  enabled: false
+  mcp_host: localhost
+  mcp_port: 8000
+  llm_server_url: '?'
+  api_key: '?'
+  model: '?'
+  model_kwargs: {}
 
 databases:
 
@@ -89,20 +99,30 @@ databases:
     host: localhost
     port: 27017
     db: flowcept
-    create_collection_index: true
+    create_collection_index: true  # Whether flowcept should create collection indices if they haven't been created yet. This is done only at the Flowcept start up.
 
 adapters:
   # For each key below, you can have multiple instances. Like mlflow1, mlflow2; zambeze1, zambeze2. Use an empty dict, {}, if you won't use any adapter.
-  zambeze:
-    kind: zambeze
-    host: localhost
-    port: 5672
-    queue_names:
-      - hello
-      - hello2
-#    key_values_to_filter:
-#      - key: activity_status
-#        value: CREATED
+
+  broker_mqtt:
+    kind: broker
+    host: h
+    port: 30011
+    protocol: mqtt3.1.1
+    queues: ["#"]
+    username: postman
+    password: p
+    qos: 2
+    task_subtype: intersect_msg
+    tracked_keys:
+      used: payload
+      generated: ~
+      custom_metadata: [headers, msgId]
+      activity_id: operationId
+      submitted_at: ~
+      started_at: ~
+      ended_at: ~
+      registered_at: ~
 
   mlflow:
     kind: mlflow
@@ -125,3 +145,4 @@ adapters:
     worker_should_get_output: true
     scheduler_create_timestamps: true
     worker_create_timestamps: false
+

flowcept-0.8.11/resources/simple_redis_consumer.py

@@ -0,0 +1,33 @@
+import redis
+import msgpack
+
+from flowcept.commons.daos.mq_dao.mq_dao_redis import MQDaoRedis
+from flowcept.configs import MQ_HOST, MQ_PORT, MQ_CHANNEL, KVDB_URI
+# Connect to Redis
+redis_client = (
+    redis.from_url(KVDB_URI) if KVDB_URI else redis.Redis(host=MQ_HOST, port=MQ_PORT, db=0)
+)
+# Subscribe to a channel
+pubsub = redis_client.pubsub()
+pubsub.subscribe(MQ_CHANNEL)
+
+print("Listening for messages...")
+
+
+for message in pubsub.listen():
+    print()
+    print("Received a message!", end=' ')
+    if message and message["type"] in MQDaoRedis.MESSAGE_TYPES_IGNORE:
+        continue
+
+    if not isinstance(message["data"], (bytes, bytearray)):
+        print(
+            f"Skipping message with unexpected data type: {type(message["data"])} - {message["data"]}")
+        continue
+
+    try:
+        msg_obj = msgpack.loads(message["data"], strict_map_key=False)
+        msg_type = msg_obj.get("type", None)
+        print(msg_type)
+    except Exception as e:
+        print(e)