flowcept 0.8.10__py3-none-any.whl → 0.8.11__py3-none-any.whl

flowcept/commons/settings_factory.py

@@ -7,9 +7,6 @@ from flowcept.commons.flowcept_dataclasses.base_settings_dataclasses import (
     BaseSettings,
     KeyValue,
 )
-from flowcept.flowceptor.adapters.zambeze.zambeze_dataclasses import (
-    ZambezeSettings,
-)
 from flowcept.flowceptor.adapters.mlflow.mlflow_dataclasses import (
     MLFlowSettings,
 )
@@ -22,7 +19,6 @@ from flowcept.flowceptor.adapters.dask.dask_dataclasses import (
 
 
 SETTINGS_CLASSES = {
-    Vocabulary.Settings.ZAMBEZE_KIND: ZambezeSettings,
     Vocabulary.Settings.MLFLOW_KIND: MLFlowSettings,
     Vocabulary.Settings.TENSORBOARD_KIND: TensorboardSettings,
     Vocabulary.Settings.DASK_KIND: DaskSettings,
@@ -30,6 +26,8 @@ SETTINGS_CLASSES = {
 
 
 def _build_base_settings(kind: str, settings_dict: dict) -> BaseSettings:
+    if kind not in SETTINGS_CLASSES:
+        return settings_dict
     settings_obj = SETTINGS_CLASSES[kind](**settings_dict)
     return settings_obj
 

flowcept/commons/task_data_preprocess.py

@@ -0,0 +1,200 @@
+"""
+The base of this code was generated using ChatGPT.
+
+Prompt:
+
+Here I have a list containing one real task.
+
+<Paste one real task here>
+
+I want to create a list of summarized task data, per task, containing:
+- activity_id
+- task_id
+- used
+- generated
+- task_duration (ended_at - started_at)
+- hostname
+- cpu_info
+- disk_info
+- mem_info
+- network_info
+<Consider adding GPU info too, if you have gpu in your task data>
+
+Where info about cpu, disk, mem, and network must consider telemetry_at_end and telemetry_at_start.
+
+We will use this summarized data as input for LLM questions to find patterns in the resource usage and how they relate
+to input (used) and output (generated) of each task.
+"""
+
+from typing import Dict, List
+
+
+def summarize_telemetry(task: Dict) -> Dict:
+    """
+    Extract and compute the telemetry summary for a task based on start and end telemetry snapshots.
+
+    Parameters
+    ----------
+    task : dict
+        The task dictionary containing telemetry_at_start and telemetry_at_end.
+
+    Returns
+    -------
+    dict
+        A summary of telemetry differences including CPU, disk, memory, and network metrics, and task duration.
+    """
+
+    def extract_cpu_info(start: Dict, end: Dict) -> Dict:
+        return {
+            "percent_all_diff": end["percent_all"] - start["percent_all"],
+            "user_time_diff": end["times_avg"]["user"] - start["times_avg"]["user"],
+            "system_time_diff": end["times_avg"]["system"] - start["times_avg"]["system"],
+            "idle_time_diff": end["times_avg"]["idle"] - start["times_avg"]["idle"],
+        }
+
+    def extract_disk_info(start: Dict, end: Dict) -> Dict:
+        io_start = start["io_sum"]
+        io_end = end["io_sum"]
+        return {
+            "read_bytes_diff": io_end["read_bytes"] - io_start["read_bytes"],
+            "write_bytes_diff": io_end["write_bytes"] - io_start["write_bytes"],
+            "read_count_diff": io_end["read_count"] - io_start["read_count"],
+            "write_count_diff": io_end["write_count"] - io_start["write_count"],
+        }
+
+    def extract_mem_info(start: Dict, end: Dict) -> Dict:
+        return {
+            "used_mem_diff": end["virtual"]["used"] - start["virtual"]["used"],
+            "percent_diff": end["virtual"]["percent"] - start["virtual"]["percent"],
+            "swap_used_diff": end["swap"]["used"] - start["swap"]["used"],
+        }
+
+    def extract_network_info(start: Dict, end: Dict) -> Dict:
+        net_start = start["netio_sum"]
+        net_end = end["netio_sum"]
+        return {
+            "bytes_sent_diff": net_end["bytes_sent"] - net_start["bytes_sent"],
+            "bytes_recv_diff": net_end["bytes_recv"] - net_start["bytes_recv"],
+            "packets_sent_diff": net_end["packets_sent"] - net_start["packets_sent"],
+            "packets_recv_diff": net_end["packets_recv"] - net_start["packets_recv"],
+        }
+
+    start_tele = task["telemetry_at_start"]
+    end_tele = task["telemetry_at_end"]
+
+    started_at = task["started_at"]
+    ended_at = task["ended_at"]
+    duration = ended_at - started_at
+
+    telemetry_summary = {
+        "duration_sec": duration,
+        "cpu_info": extract_cpu_info(start_tele["cpu"], end_tele["cpu"]),
+        "disk_info": extract_disk_info(start_tele["disk"], end_tele["disk"]),
+        "mem_info": extract_mem_info(start_tele["memory"], end_tele["memory"]),
+        "network_info": extract_network_info(start_tele["network"], end_tele["network"]),
+    }
+
+    return telemetry_summary
+
+
+def summarize_task(task: Dict, thresholds: Dict = None, logger=None) -> Dict:
+    """
+    Summarize key metadata and telemetry for a task, optionally tagging critical conditions.
+
+    Parameters
+    ----------
+    task : dict
+        The task dictionary containing metadata and telemetry snapshots.
+    thresholds : dict, optional
+        Threshold values used to tag abnormal resource usage.
+
+    Returns
+    -------
+    dict
+        Summary of the task including identifiers, telemetry summary, and optional critical tags.
+    """
+    task_summary = {
+        "workflow_id": task.get("workflow_id"),
+        "task_id": task.get("task_id"),
+        "activity_id": task.get("activity_id"),
+        "used": task.get("used"),
+        "generated": task.get("generated"),
+        "hostname": task.get("hostname"),
+        "status": task.get("status"),
+    }
+
+    try:
+        telemetry_summary = summarize_telemetry(task)
+        tags = tag_critical_task(
+            generated=task.get("generated", {}), telemetry_summary=telemetry_summary, thresholds=thresholds
+        )
+        if tags:
+            task_summary["tags"] = tags
+        task_summary["telemetry_summary"] = telemetry_summary
+    except Exception as e:
+        if logger:
+            logger.exception(e)
+        else:
+            print(e)
+
+    return task_summary
+
+
+def tag_critical_task(
+    generated: Dict, telemetry_summary: Dict, generated_keywords: List[str] = ["result"], thresholds: Dict = None
+) -> List[str]:
+    """
+    Tag a task with labels indicating abnormal or noteworthy resource usage or result anomalies.
+
+    Parameters
+    ----------
+    generated : dict
+        Dictionary of generated output values (e.g., results).
+    telemetry_summary : dict
+        Telemetry summary produced from summarize_telemetry().
+    generated_keywords : list of str, optional
+        List of keys in the generated output to check for anomalies.
+    thresholds : dict, optional
+        Custom thresholds for tagging high CPU, memory, disk, etc.
+
+    Returns
+    -------
+    list of str
+        Tags indicating abnormal patterns (e.g., "high_cpu", "low_output").
+    """
+    if thresholds is None:
+        thresholds = {
+            "high_cpu": 80,
+            "high_mem": 1e9,
+            "high_disk": 1e8,
+            "long_duration": 0.8,
+            "low_output": 0.1,
+            "high_output": 0.9,
+        }
+
+    cpu = abs(telemetry_summary["cpu_info"].get("percent_all_diff", 0))
+    mem = telemetry_summary["mem_info"].get("used_mem_diff", 0)
+    disk = telemetry_summary["disk_info"].get("read_bytes_diff", 0) + telemetry_summary["disk_info"].get(
+        "write_bytes_diff", 0
+    )
+    duration = telemetry_summary["duration_sec"]
+
+    tags = []
+
+    if cpu > thresholds["high_cpu"]:
+        tags.append("high_cpu")
+    if mem > thresholds["high_mem"]:
+        tags.append("high_mem")
+    if disk > thresholds["high_disk"]:
+        tags.append("high_disk")
+    if duration > thresholds["long_duration"]:
+        tags.append("long_duration")
+
+    for key in generated_keywords:
+        value = generated.get(key, 0)
+        if value < thresholds["low_output"]:
+            tags.append("low_output")
+        if value > thresholds["high_output"]:
+            tags.append("high_output")
+
+    return tags

flowcept/commons/utils.py

@@ -26,7 +26,7 @@ def get_utc_now() -> float:
 
 def get_utc_now_str() -> str:
     """Get UTC string."""
-    format_string = "%Y-%m-%dT%H:%M:%S.%f"
+    format_string = "%Y-%m-%dT%H:%M:%S.%f%z"
     now = datetime.now(timezone.utc)
     return now.strftime(format_string)
 

flowcept/configs.py

@@ -69,10 +69,11 @@ FLOWCEPT_USER = settings["experiment"].get("user", "blank_user")
 MQ_INSTANCES = settings["mq"].get("instances", None)
 MQ_SETTINGS = settings["mq"]
 MQ_TYPE = os.getenv("MQ_TYPE", settings["mq"].get("type", "redis"))
-MQ_CHANNEL = settings["mq"].get("channel", "interception")
+MQ_CHANNEL = os.getenv("MQ_CHANNEL", settings["mq"].get("channel", "interception"))
 MQ_PASSWORD = settings["mq"].get("password", None)
 MQ_HOST = os.getenv("MQ_HOST", settings["mq"].get("host", "localhost"))
 MQ_PORT = int(os.getenv("MQ_PORT", settings["mq"].get("port", "6379")))
+MQ_URI = os.getenv("MQ_URI", settings["mq"].get("uri", None))
 MQ_BUFFER_SIZE = settings["mq"].get("buffer_size", None)
 MQ_INSERTION_BUFFER_TIME = settings["mq"].get("insertion_buffer_time_secs", None)
 MQ_TIMING = settings["mq"].get("timing", False)
@@ -86,6 +87,7 @@ KVDB_PASSWORD = settings["kv_db"].get("password", None)
 KVDB_HOST = os.getenv("KVDB_HOST", settings["kv_db"].get("host", "localhost"))
 KVDB_PORT = int(os.getenv("KVDB_PORT", settings["kv_db"].get("port", "6379")))
 KVDB_URI = os.getenv("KVDB_URI", settings["kv_db"].get("uri", None))
+KVDB_ENABLED = settings["kv_db"].get("enabled", True)
 
 
 DATABASES = settings.get("databases", {})
@@ -118,9 +120,9 @@ if LMDB_SETTINGS:
     else:
         LMDB_ENABLED = LMDB_SETTINGS.get("enabled", False)
 
-if not LMDB_ENABLED and not MONGO_ENABLED:
-    # At least one of these variables need to be enabled.
-    LMDB_ENABLED = True
+# if not LMDB_ENABLED and not MONGO_ENABLED:
+#     # At least one of these variables need to be enabled.
+#     LMDB_ENABLED = True
 
 ##########################
 # DB Buffer Settings        #
@@ -219,6 +221,8 @@ ANALYTICS = settings.get("analytics", None)
 INSTRUMENTATION = settings.get("instrumentation", {})
 INSTRUMENTATION_ENABLED = INSTRUMENTATION.get("enabled", False)
 
+AGENT = settings.get("agent", {})
+
 ####################
 # Enabled ADAPTERS #
 ####################

flowcept/flowcept_api/flowcept_controller.py

@@ -9,7 +9,14 @@ from flowcept.commons.flowcept_dataclasses.workflow_object import (
 )
 from flowcept.commons.flowcept_logger import FlowceptLogger
 from flowcept.commons.utils import ClassProperty
-from flowcept.configs import MQ_INSTANCES, INSTRUMENTATION_ENABLED, MONGO_ENABLED, SETTINGS_PATH
+from flowcept.configs import (
+    MQ_INSTANCES,
+    INSTRUMENTATION_ENABLED,
+    MONGO_ENABLED,
+    SETTINGS_PATH,
+    LMDB_ENABLED,
+    KVDB_ENABLED,
+)
 from flowcept.flowceptor.adapters.base_interceptor import BaseInterceptor
 
 
@@ -39,6 +46,7 @@ class Flowcept(object):
         workflow_name: str = None,
         workflow_args: str = None,
         start_persistence=True,
+        check_safe_stops=True,  # TODO add to docstring
         save_workflow=True,
         *args,
         **kwargs,
@@ -86,6 +94,7 @@ class Flowcept(object):
         self.logger.debug(f"Using settings file: {SETTINGS_PATH}")
         self._enable_persistence = start_persistence
         self._db_inserters: List = []
+        self._check_safe_stops = check_safe_stops
         if bundle_exec_id is None:
             self._bundle_exec_id = id(self)
         else:
@@ -140,7 +149,7 @@ class Flowcept(object):
                 Flowcept.current_workflow_id = self.current_workflow_id
 
                 interceptor_inst = BaseInterceptor.build(interceptor)
-                interceptor_inst.start(bundle_exec_id=self._bundle_exec_id)
+                interceptor_inst.start(bundle_exec_id=self._bundle_exec_id, check_safe_stops=self._check_safe_stops)
                 self._interceptor_instances.append(interceptor_inst)
 
                 if self._should_save_workflow and not self._workflow_saved:
@@ -190,19 +199,20 @@ class Flowcept(object):
             else:
                 raise Exception("You must provide the argument `dask_client` so we can correctly link the workflow.")
 
-        interceptor_instance._mq_dao.set_campaign_id(Flowcept.campaign_id)
+        if KVDB_ENABLED:
+            interceptor_instance._mq_dao.set_campaign_id(Flowcept.campaign_id)
         interceptor_instance.send_workflow_message(wf_obj)
         self._workflow_saved = True
 
     def _init_persistence(self, mq_host=None, mq_port=None):
+        if not LMDB_ENABLED and not MONGO_ENABLED:
+            return
+
         from flowcept.flowceptor.consumers.document_inserter import DocumentInserter
 
-        self._db_inserters.append(
-            DocumentInserter(
-                check_safe_stops=True,
-                bundle_exec_id=self._bundle_exec_id,
-            ).start()
-        )
+        doc_inserter = DocumentInserter(check_safe_stops=self._check_safe_stops, bundle_exec_id=self._bundle_exec_id)
+        doc_inserter.start()
+        self._db_inserters.append(doc_inserter)
 
     def stop(self):
         """Stop it."""
@@ -214,7 +224,7 @@ class Flowcept(object):
             for interceptor in self._interceptor_instances:
                 if interceptor is None:
                     continue
-                interceptor.stop()
+                interceptor.stop(check_safe_stops=self._check_safe_stops)
 
         if len(self._db_inserters):
             self.logger.info("Stopping DB Inserters...")
@@ -259,17 +269,24 @@ class Flowcept(object):
         ...     print("One or more services are not ready.")
         """
         logger = FlowceptLogger()
-        if not MQDao.build().liveness_test():
+        mq = MQDao.build()
+        if not mq.liveness_test():
             logger.error("MQ Not Ready!")
             return False
 
+        if KVDB_ENABLED:
+            if not mq._keyvalue_dao.liveness_test():
+                logger.error("KVBD is enabled but is not ready!")
+                return False
+
+        logger.info("MQ is alive!")
         if MONGO_ENABLED:
             from flowcept.commons.daos.docdb_dao.mongodb_dao import MongoDBDAO
 
             if not MongoDBDAO(create_indices=False).liveness_test():
-                logger.error("DocDB Not Ready!")
+                logger.error("MongoDB is enabled but DocDB is not Ready!")
                 return False
-        logger.info("MQ and DocDB are alive!")
+            logger.info("DocDB is alive!")
         return True
 
     @staticmethod

flowcept/flowceptor/adapters/agents/__init__.py

@@ -0,0 +1 @@
+"""Agent adapter subpackage."""

flowcept/flowceptor/adapters/agents/agents_utils.py

@@ -0,0 +1,89 @@
+from typing import List, Union
+
+from langchain_community.llms.sambanova import SambaStudio
+from mcp.server.fastmcp.prompts import base
+from langchain_core.language_models import LLM
+from langchain_core.messages import HumanMessage, AIMessage
+
+from flowcept.configs import AGENT
+
+
+def build_llm_model() -> LLM:
+    """
+    Build and return an LLM instance using agent configuration.
+
+    This function retrieves the model name and keyword arguments from the AGENT configuration,
+    constructs a SambaStudio LLM instance, and returns it.
+
+    Returns
+    -------
+    LLM
+        An initialized LLM object configured using the `AGENT` settings.
+    """
+    model_kwargs = AGENT.get("model_kwargs").copy()
+    model_kwargs["model"] = AGENT.get("model")
+    llm = SambaStudio(model_kwargs=model_kwargs)
+
+    return llm
+
+
+def convert_mcp_messages_to_plain_text(messages: list[base.Message]) -> str:
+    """
+    Convert a list of MCP base.Message objects into a plain text dialogue.
+
+    Parameters
+    ----------
+    messages : list of BaseMessage
+        The list of messages, typically from HumanMessage, AIMessage, SystemMessage, etc.
+
+    Returns
+    -------
+    str
+        A plain text version of the conversation, with roles labeled.
+    """
+    lines = []
+    for message in messages:
+        role = message.role.capitalize()  # e.g., "human" → "Human"
+        line = f"{role}: {message.content.text}"
+        lines.append(line)
+    return "\n".join(lines)
+
+
+def convert_mcp_to_langchain(messages: list[base.Message]) -> List[Union[HumanMessage, AIMessage]]:
+    """
+    Convert a list of MCP-style messages to LangChain-compatible message objects.
+
+    Parameters
+    ----------
+    messages : list of base.Message
+        A list of messages in the MCP message format, each with a `role` and `content`.
+
+    Returns
+    -------
+    list of Union[HumanMessage, AIMessage]
+        A list of LangChain message objects, converted from the original MCP format.
+
+    Raises
+    ------
+    ValueError
+        If a message has a role that is not 'user' or 'assistant'.
+
+    Notes
+    -----
+    This function extracts the `text` attribute from message content if present, falling back to `str(content)`
+    otherwise. It maps MCP 'user' roles to LangChain `HumanMessage` and 'assistant' roles to `AIMessage`.
+    """
+    converted = []
+    for m in messages:
+        if hasattr(m.content, "text"):
+            content = m.content.text
+        else:
+            content = str(m.content)  # fallback if it's already a string
+
+        if m.role == "user":
+            converted.append(HumanMessage(content=content))
+        elif m.role == "assistant":
+            converted.append(AIMessage(content=content))
+        else:
+            raise ValueError(f"Unsupported role: {m.role}")
+    return converted