flowcept 0.8.9__py3-none-any.whl → 0.8.11__py3-none-any.whl

resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.8.9 # Version of the Flowcept package. This setting file is compatible with this version.
+flowcept_version: 0.8.11 # Version of the Flowcept package. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.
@@ -25,7 +25,6 @@ telemetry_capture: # This toggles each individual type of telemetry capture. GPU
 
 instrumentation:
   enabled: true # This toggles data capture for instrumentation.
-  singleton: true # Use a single instrumentation instance per process. Defaults to true
   torch:
     what: parent_and_children # Scope of instrumentation: "parent_only" -- will capture only at the main model level, "parent_and_children" -- will capture the inner layers, or ~ (disable).
     children_mode: telemetry_and_tensor_inspection   # What to capture if parent_and_children is chosen in the scope. Possible values: "tensor_inspection" (i.e., tensor metadata), "telemetry", "telemetry_and_tensor_inspection"
@@ -40,18 +39,22 @@ experiment:
 mq:
   type: redis  # or kafka or mofka; Please adjust the port (kafka's default is 9092; redis is 6379). If mofka, adjust the group_file.
   host: localhost
-  # instances: ["localhost:6379"] # We can have multiple redis instances being accessed by the consumers but each interceptor will currently access one single redis.
+  # uri: ?
+  # instances: ["localhost:6379"] # We can have multiple MQ instances being accessed by the consumers but each interceptor will currently access one single MQ..
   port: 6379
   # group_file: mofka.json
   channel: interception
   buffer_size: 50
   insertion_buffer_time_secs: 5
   timing: false
+  # uri: use Redis connection uri here
   chunk_size: -1  # use 0 or -1 to disable this. Or simply omit this from the config file.
+  same_as_kvdb: false # Set this to true if you are using the same Redis instance both as an MQ and as the KV_DB. In that case, no need to repeat connection parameters in MQ. Use only what you define in KV_DB.
 
 kv_db:
   host: localhost
   port: 6379
+  enabled: true
   # uri: use Redis connection uri here
 
 web_server:
@@ -59,9 +62,9 @@ web_server:
   port: 5000
 
 sys_metadata:
-  environment_id: "laptop"
+  environment_id: "laptop"   # We use this to keep track of the environment used to run an experiment. Typical values include the cluster name, but it can be anything that you think will help identify your experimentation environment.
 
-extra_metadata:
+extra_metadata: # We use this to store any extra metadata you want to keep track of during an experiment.
   place_holder: ""
 
 analytics:
@@ -70,13 +73,20 @@ analytics:
     generated.accuracy: maximum_first
 
 db_buffer:
-  adaptive_buffer_size: true
-  insertion_buffer_time_secs: 5
-  max_buffer_size: 50
-  min_buffer_size: 10
-  remove_empty_fields: false
-  stop_max_trials: 240
-  stop_trials_sleep: 0.01
+  insertion_buffer_time_secs: 5   # Time interval (in seconds) to buffer incoming records before flushing to the database
+  buffer_size: 50    # Maximum number of records to hold in the buffer before forcing a flush
+  remove_empty_fields: false    # If true, fields with null/empty values will be removed before insertion
+  stop_max_trials: 240    # Maximum number of trials before giving up when waiting for a fully safe stop (i.e., all records have been inserted as expected).
+  stop_trials_sleep: 0.01   # Sleep duration (in seconds) between trials when waiting for a fully safe stop.
+
+agent:
+  enabled: false
+  mcp_host: localhost
+  mcp_port: 8000
+  llm_server_url: '?'
+  api_key: '?'
+  model: '?'
+  model_kwargs: {}
 
 databases:
 
@@ -89,20 +99,30 @@ databases:
     host: localhost
     port: 27017
     db: flowcept
-    create_collection_index: true
+    create_collection_index: true  # Whether flowcept should create collection indices if they haven't been created yet. This is done only at the Flowcept start up.
 
 adapters:
   # For each key below, you can have multiple instances. Like mlflow1, mlflow2; zambeze1, zambeze2. Use an empty dict, {}, if you won't use any adapter.
-  zambeze:
-    kind: zambeze
-    host: localhost
-    port: 5672
-    queue_names:
-      - hello
-      - hello2
-#    key_values_to_filter:
-#      - key: activity_status
-#        value: CREATED
+
+  broker_mqtt:
+    kind: broker
+    host: h
+    port: 30011
+    protocol: mqtt3.1.1
+    queues: ["#"]
+    username: postman
+    password: p
+    qos: 2
+    task_subtype: intersect_msg
+    tracked_keys:
+      used: payload
+      generated: ~
+      custom_metadata: [headers, msgId]
+      activity_id: operationId
+      submitted_at: ~
+      started_at: ~
+      ended_at: ~
+      registered_at: ~
 
   mlflow:
     kind: mlflow
@@ -125,3 +145,4 @@ adapters:
     worker_should_get_output: true
     scheduler_create_timestamps: true
     worker_create_timestamps: false
+

flowcept/flowceptor/adapters/zambeze/__init__.py

@@ -1 +0,0 @@
-"""Zambeze subpackage."""

flowcept/flowceptor/adapters/zambeze/zambeze_dataclasses.py

@@ -1,41 +0,0 @@
-"""Zambeze dataclass module."""
-
-from dataclasses import dataclass
-from typing import List, Dict
-
-from flowcept.commons.flowcept_dataclasses.base_settings_dataclasses import (
-    BaseSettings,
-    KeyValue,
-)
-
-
-@dataclass
-class ZambezeMessage:
-    """Zambeze message."""
-
-    name: str
-    activity_id: str
-    campaign_id: str
-    origin_agent_id: str
-    files: List[str]
-    command: str
-    activity_status: str
-    arguments: List[str]
-    kwargs: Dict
-    depends_on: List[str]
-
-
-@dataclass
-class ZambezeSettings(BaseSettings):
-    """Zambeze settings."""
-
-    host: str
-    port: int
-    queue_names: List[str]
-    key_values_to_filter: List[KeyValue] = None
-    kind = "zambeze"
-
-    def __post_init__(self):
-        """Set attributes after init."""
-        self.observer_type = "message_broker"
-        self.observer_subtype = "rabbit_mq"

flowcept/flowceptor/adapters/zambeze/zambeze_interceptor.py

@@ -1,102 +0,0 @@
-"""Zambeze interceptor module."""
-
-from threading import Thread
-from time import sleep
-import pika
-import json
-from typing import Dict
-
-from flowcept.commons.utils import get_utc_now, get_status_from_str
-from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
-from flowcept.flowceptor.adapters.base_interceptor import (
-    BaseInterceptor,
-)
-
-
-class ZambezeInterceptor(BaseInterceptor):
-    """Zambeze interceptor."""
-
-    def __init__(self, plugin_key="zambeze"):
-        super().__init__(plugin_key)
-        self._consumer_tag = None
-        self._channel = None
-        self._observer_thread: Thread = None
-
-    def prepare_task_msg(self, zambeze_msg: Dict) -> TaskObject:
-        """Prepare a task."""
-        task_msg = TaskObject()
-        task_msg.utc_timestamp = get_utc_now()
-        task_msg.campaign_id = zambeze_msg.get("campaign_id", None)
-        task_msg.task_id = zambeze_msg.get("activity_id", None)
-        task_msg.activity_id = zambeze_msg.get("name", None)
-        task_msg.dependencies = zambeze_msg.get("depends_on", None)
-        task_msg.custom_metadata = {"command": zambeze_msg.get("command", None)}
-        task_msg.status = get_status_from_str(zambeze_msg.get("activity_status", None))
-        task_msg.used = {
-            "args": zambeze_msg.get("arguments", None),
-            "kwargs": zambeze_msg.get("kwargs", None),
-            "files": zambeze_msg.get("files", None),
-        }
-        return task_msg
-
-    def start(self, bundle_exec_id) -> "ZambezeInterceptor":
-        """Start it."""
-        super().start(bundle_exec_id)
-        self._observer_thread = Thread(target=self.observe)
-        self._observer_thread.start()
-        return self
-
-    def stop(self) -> bool:
-        """Stop it."""
-        self.logger.debug("Interceptor stopping...")
-        super().stop()
-        try:
-            self._channel.stop_consuming()
-        except Exception as e:
-            self.logger.warning(f"This exception is expected to occur after channel.basic_cancel: {e}")
-        sleep(2)
-        self._observer_thread.join()
-        self.logger.debug("Interceptor stopped.")
-        return True
-
-    def observe(self):
-        """Observe it."""
-        connection = pika.BlockingConnection(
-            pika.ConnectionParameters(host=self.settings.host, port=self.settings.port)
-        )
-        self._channel = connection.channel()
-        for queue in self.settings.queue_names:
-            self._channel.queue_declare(queue=queue)
-
-        # self._consumer_tag =\
-        for queue in self.settings.queue_names:
-            self._channel.basic_consume(
-                queue=queue,
-                on_message_callback=self.callback,
-                auto_ack=True,
-            )
-            self.logger.debug(f"Waiting for Zambeze messages on queue {queue}")
-
-        try:
-            self._channel.start_consuming()
-        except Exception as e:
-            self.logger.warning(
-                f"If this exception happens after channel.start_consuming finishes, it is expected:\n {e}"
-            )
-
-    def _intercept(self, body_obj):
-        self.logger.debug(f"Zambeze interceptor needs to intercept this:\n\t{json.dumps(body_obj)}")
-        task_msg = self.prepare_task_msg(body_obj)
-        self.intercept(task_msg.to_dict())
-
-    def callback(self, ch, method, properties, body):
-        """Implement the callback."""
-        body_obj = json.loads(body)
-        if self.settings.key_values_to_filter is not None:
-            for key_value in self.settings.key_values_to_filter:
-                if key_value.key in body_obj:
-                    if body_obj[key_value.key] == key_value.value:
-                        self._intercept(body_obj)
-                        break
-        else:
-            self._intercept(body_obj)