flowcept 0.8.10__py3-none-any.whl → 0.8.12__py3-none-any.whl

flowcept/commons/task_data_preprocess.py

@@ -0,0 +1,400 @@
+from datetime import datetime
+from typing import Dict, List
+import copy
+from collections import defaultdict
+from typing import Any
+
+import pytz
+
+
+def summarize_telemetry(task: Dict, logger) -> Dict:
+    """
+    Extract and compute the telemetry summary for a task based on start and end telemetry snapshots.
+
+    Parameters
+    ----------
+    task : dict
+        The task dictionary containing telemetry_at_start and telemetry_at_end.
+
+    Returns
+    -------
+    dict
+        A summary of telemetry differences including CPU, disk, memory, and network metrics, and task duration.
+    """
+
+    def extract_cpu_info(start: Dict, end: Dict) -> Dict:
+        return {
+            "percent_all_diff": end["percent_all"] - start["percent_all"],
+            "user_time_diff": end["times_avg"]["user"] - start["times_avg"]["user"],
+            "system_time_diff": end["times_avg"]["system"] - start["times_avg"]["system"],
+            "idle_time_diff": end["times_avg"]["idle"] - start["times_avg"]["idle"],
+        }
+
+    def extract_disk_info(start: Dict, end: Dict) -> Dict:
+        io_start = start["io_sum"]
+        io_end = end["io_sum"]
+        return {
+            "read_bytes_diff": io_end["read_bytes"] - io_start["read_bytes"],
+            "write_bytes_diff": io_end["write_bytes"] - io_start["write_bytes"],
+            "read_count_diff": io_end["read_count"] - io_start["read_count"],
+            "write_count_diff": io_end["write_count"] - io_start["write_count"],
+        }
+
+    def extract_mem_info(start: Dict, end: Dict) -> Dict:
+        return {
+            "used_mem_diff": end["virtual"]["used"] - start["virtual"]["used"],
+            "percent_diff": end["virtual"]["percent"] - start["virtual"]["percent"],
+            "swap_used_diff": end["swap"]["used"] - start["swap"]["used"],
+        }
+
+    def extract_network_info(start: Dict, end: Dict) -> Dict:
+        net_start = start["netio_sum"]
+        net_end = end["netio_sum"]
+        return {
+            "bytes_sent_diff": net_end["bytes_sent"] - net_start["bytes_sent"],
+            "bytes_recv_diff": net_end["bytes_recv"] - net_start["bytes_recv"],
+            "packets_sent_diff": net_end["packets_sent"] - net_start["packets_sent"],
+            "packets_recv_diff": net_end["packets_recv"] - net_start["packets_recv"],
+        }
+
+    tel_funcs = {
+        "cpu": extract_cpu_info,
+        "disk": extract_disk_info,
+        "memory": extract_mem_info,
+        "network": extract_network_info,
+    }
+
+    start_tele = task.get("telemetry_at_start", {})
+    end_tele = task.get("telemetry_at_end", {})
+
+    telemetry_summary = {}
+
+    try:
+        started_at = task.get("started_at", None)
+        ended_at = task.get("ended_at", None)
+        if started_at is None or ended_at is None:
+            logger.warning(f"We can't summarize telemetry for duration_sec for task {task}")
+        else:
+            duration = ended_at - started_at
+            telemetry_summary["duration_sec"] = duration
+    except Exception as e:
+        logger.error(f"Error to summarize telemetry for duration_sec in {task}")
+        logger.exception(e)
+
+    for key in start_tele.keys():
+        try:
+            if key not in tel_funcs:
+                continue
+            func = tel_funcs[key]
+            if key in end_tele:
+                telemetry_summary[key] = func(start_tele[key], end_tele[key])
+            else:
+                logger.warning(
+                    f"We can't summarize telemetry {key} for task {task} because the key is not in the end_tele"
+                )
+        except Exception as e:
+            logger.warning(f"Error to summarize telemetry for {key} for task {task}. Exception: {e}")
+            logger.exception(e)
+
+    return telemetry_summary
+
+
+def _safe_get(task, key):
+    try:
+        return task.get(key)
+    except Exception:
+        return None
+
+
+def summarize_task(task: Dict, thresholds: Dict = None, logger=None) -> Dict:
+    """
+    Summarize key metadata and telemetry for a task, optionally tagging critical conditions.
+
+    Parameters
+    ----------
+    task : dict
+        The task dictionary containing metadata and telemetry snapshots.
+    thresholds : dict, optional
+        Threshold values used to tag abnormal resource usage.
+
+    Returns
+    -------
+    dict
+        Summary of the task including identifiers, telemetry summary, and optional critical tags.
+    """
+    task_summary = {}
+
+    # Keys that can be copied directly
+    for key in [
+        "workflow_id",
+        "task_id",
+        "parent_task_id",
+        "activity_id",
+        "used",
+        "generated",
+        "hostname",
+        "status",
+        "agent_id",
+        "campaign_id",
+        "subtype",
+        "custom_metadata",
+    ]:
+        value = _safe_get(task, key)
+        if value is not None:
+            if "_id" in key:
+                task_summary[key] = str(value)
+            else:
+                task_summary[key] = value
+
+        # Special handling for timestamp field
+    try:
+        time_keys = ["started_at", "ended_at"]
+        for time_key in time_keys:
+            timestamp = _safe_get(task, time_key)
+            if timestamp is not None:
+                task_summary[time_key] = datetime.fromtimestamp(timestamp, pytz.utc)
+    except Exception as e:
+        if logger:
+            logger.exception(f"Error {e} converting timestamp for task {task.get('task_id', 'unknown')}")
+
+    try:
+        telemetry_summary = summarize_telemetry(task, logger)
+        try:
+            tags = tag_critical_task(
+                generated=task.get("generated", {}), telemetry_summary=telemetry_summary, thresholds=thresholds
+            )
+            if tags:
+                task_summary["tags"] = tags
+        except Exception as e:
+            logger.exception(e)
+        task_summary["telemetry_summary"] = telemetry_summary
+    except Exception as e:
+        if logger:
+            logger.exception(e)
+        else:
+            print(e)
+
+    return task_summary
+
+
+def tag_critical_task(
+    generated: Dict, telemetry_summary: Dict, generated_keywords: List[str] = ["result"], thresholds: Dict = None
+) -> List[str]:
+    """
+    Tag a task with labels indicating abnormal or noteworthy resource usage or result anomalies.
+
+    Parameters
+    ----------
+    generated : dict
+        Dictionary of generated output values (e.g., results).
+    telemetry_summary : dict
+        Telemetry summary produced from summarize_telemetry().
+    generated_keywords : list of str, optional
+        List of keys in the generated output to check for anomalies.
+    thresholds : dict, optional
+        Custom thresholds for tagging high CPU, memory, disk, etc.
+
+    Returns
+    -------
+    list of str
+        Tags indicating abnormal patterns (e.g., "high_cpu", "low_output").
+    """
+    if thresholds is None:
+        thresholds = {
+            "high_cpu": 80,
+            "high_mem": 1e9,
+            "high_disk": 1e8,
+            "long_duration": 0.8,
+            "low_output": 0.1,
+            "high_output": 0.9,
+        }
+
+    cpu = abs(telemetry_summary.get("cpu", {}).get("percent_all_diff", 0))
+    mem = telemetry_summary.get("mem", {}).get("used_mem_diff", 0)
+    disk = telemetry_summary.get("disk", {}).get("read_bytes_diff", 0) + telemetry_summary.get("disk", {}).get(
+        "write_bytes_diff", 0
+    )
+    # TODO gpu
+    duration = telemetry_summary.get("duration_sec", 0)
+
+    tags = []
+
+    if cpu > thresholds["high_cpu"]:
+        tags.append("high_cpu")
+    if mem > thresholds["high_mem"]:
+        tags.append("high_mem")
+    if disk > thresholds["high_disk"]:
+        tags.append("high_disk")
+    if duration > thresholds["long_duration"]:
+        tags.append("long_duration")
+
+    for key in generated_keywords:
+        value = generated.get(key, 0)
+        if value < thresholds["low_output"]:
+            tags.append("low_output")
+        if value > thresholds["high_output"]:
+            tags.append("high_output")
+
+    return tags
+
+
+sample_tasks = [
+    {
+        "task_id": "t1",
+        "activity_id": "train_model",
+        "used": {
+            "dataset": {"name": "MNIST", "size": 60000, "source": {"url": "http://example.com/mnist", "format": "csv"}},
+            "params": {"epochs": 5, "batch_size": 32, "shuffle": True},
+        },
+        "generated": {"model": {"accuracy": 0.98, "layers": [64, 64, 10], "saved_path": "/models/mnist_v1.pth"}},
+        "telemetry_summary": {"duration_sec": 42.7, "cpu_percent": 85.2},
+    },
+    {
+        "task_id": "t2",
+        "activity_id": "train_model",
+        "used": {
+            "dataset": {
+                "name": "CIFAR-10",
+                "size": 50000,
+                "source": {"url": "http://example.com/cifar", "format": "jpeg"},
+            },
+            "params": {"epochs": 10, "batch_size": 64, "shuffle": False},
+        },
+        "generated": {"model": {"accuracy": 0.91, "layers": [128, 128, 10], "saved_path": "/models/cifar_v1.pth"}},
+        "telemetry_summary": {"duration_sec": 120.5, "cpu_percent": 92.0},
+    },
+    {
+        "task_id": "t3",
+        "activity_id": "evaluate_model",
+        "used": {"model_path": "/models/mnist_v1.pth", "test_data": {"name": "MNIST-test", "samples": 10000}},
+        "generated": {"metrics": {"accuracy": 0.97, "confusion_matrix": [[8500, 100], [50, 1350]]}},
+        "telemetry_summary": {"duration_sec": 15.3},
+    },
+    {
+        "task_id": "t4",
+        "activity_id": "evaluate_model",
+        "used": {"model_path": "/models/cifar_v1.pth", "test_data": {"name": "CIFAR-test", "samples": 10000}},
+        "generated": {"metrics": {"accuracy": 0.88, "confusion_matrix": [[4000, 500], [300, 5200]]}},
+        "telemetry_summary": {"duration_sec": 18.9},
+    },
+]
+
+
+def infer_dtype(value: Any) -> str:
+    """Infer a simplified dtype label for the value."""
+    if isinstance(value, bool):
+        return "bool"
+    elif isinstance(value, int):
+        return "int"
+    elif isinstance(value, float):
+        return "float"
+    elif isinstance(value, str):
+        return "str"
+    elif isinstance(value, list):
+        return "list"
+    return "str"  # fallback for other types
+
+
+def flatten_dict(d: dict, parent_key: str = "", sep: str = ".") -> dict:
+    """Recursively flatten nested dicts using dot notation."""
+    items = {}
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.update(flatten_dict(v, new_key, sep=sep))
+        else:
+            items[new_key] = v
+    return items
+
+
+def update_schema(schema_section: list, flat_fields: dict):
+    """Update schema section with flattened fields and example values."""
+    field_map = {f["n"]: f for f in schema_section}
+
+    for key, value in flat_fields.items():
+        dtype = infer_dtype(value)
+        if isinstance(value, float):
+            val_repr = round(value, 2)
+        elif isinstance(value, (dict, list)):
+            val_repr = str(value)
+        else:
+            val_repr = value
+
+        if isinstance(val_repr, str) and len(val_repr) > 100:
+            val_repr = val_repr[:100] + "#TRUNCATED"
+
+        if key not in field_map:
+            field = {
+                "n": key,
+                "d": dtype,
+                "v": [val_repr] if val_repr is not None else [],
+            }
+            schema_section.append(field)
+            field_map[key] = field
+        else:
+            field = field_map[key]
+            if val_repr not in field["v"] and len(field["v"]) < 3:
+                field["v"].append(val_repr)
+
+
+def update_tasks_summary_schema(tasks: list[dict], schema) -> dict:
+    """Update tasks_summary schema."""
+    act_schema = update_activity_schema(tasks)
+    merged_schema = deep_merge_dicts(act_schema, schema)
+    return merged_schema
+
+
+def update_activity_schema(tasks: list[dict]) -> dict:
+    """Build schema for each activity_id from list of task dicts."""
+    schema = defaultdict(
+        lambda: {
+            "in": [],
+            "out": [],
+            # "tel": [],
+        }
+    )
+
+    for task in tasks:
+        activity_id = task.get("activity_id")
+        if not activity_id:
+            continue
+
+        activity_schema = schema[activity_id]
+
+        for section_key, schema_key in [
+            ("used", "in"),
+            ("generated", "out"),
+            #   ("telemetry_summary", "tel"),
+        ]:
+            section_data = task.get(section_key)
+            if isinstance(section_data, dict):
+                flat_fields = flatten_dict(section_data, parent_key=section_key)
+                update_schema(activity_schema[schema_key], flat_fields)
+
+    schema = dict(schema)
+    return schema
+
+
+def deep_merge_dicts(a: dict, b: dict) -> dict:
+    """
+    Recursively merge dict b into dict a:
+    - Does not overwrite existing values in a.
+    - If both values are dicts, merges recursively.
+    - If both values are lists, concatenates and deduplicates.
+    - Otherwise, keeps value from a.
+    Returns a new dict (does not mutate inputs).
+    """
+    result = copy.deepcopy(a)
+
+    for key, b_val in b.items():
+        if key not in result:
+            result[key] = copy.deepcopy(b_val)
+        else:
+            a_val = result[key]
+            if isinstance(a_val, dict) and isinstance(b_val, dict):
+                result[key] = deep_merge_dicts(a_val, b_val)
+            elif isinstance(a_val, list) and isinstance(b_val, list):
+                combined = a_val + [item for item in b_val if item not in a_val]
+                result[key] = combined
+            # preserve a_val otherwise
+    return result

flowcept/commons/utils.py

@@ -1,5 +1,6 @@
 """Utilities."""
 
+import argparse
 from datetime import datetime, timedelta, timezone
 import json
 from time import time, sleep
@@ -9,7 +10,6 @@ import platform
 import subprocess
 import types
 import numpy as np
-import pytz
 
 from flowcept import configs
 from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
@@ -19,14 +19,14 @@ from flowcept.commons.vocabulary import Status
 
 
 def get_utc_now() -> float:
-    """Get UTC time."""
+    """Get current UTC time as a timestamp (seconds since epoch)."""
     now = datetime.now(timezone.utc)
     return now.timestamp()
 
 
 def get_utc_now_str() -> str:
     """Get UTC string."""
-    format_string = "%Y-%m-%dT%H:%M:%S.%f"
+    format_string = "%Y-%m-%dT%H:%M:%S.%f%z"
     now = datetime.now(timezone.utc)
     return now.strftime(format_string)
 
@@ -159,11 +159,14 @@ class GenericJSONEncoder(json.JSONEncoder):
         return super().default(obj)
 
 
-def replace_non_serializable_times(obj, tz=pytz.utc):
-    """Replace non-serializable times in an object."""
+def replace_non_serializable_times(obj, tz=timezone.utc):
+    """Replace non-serializable datetimes in an object with ISO 8601 strings (ms precision)."""
     for time_field in TaskObject.get_time_field_names():
-        if time_field in obj:
-            obj[time_field] = obj[time_field].strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + f" {tz}"
+        if time_field in obj and isinstance(obj[time_field], datetime):
+            obj[time_field] = obj[time_field].astimezone(tz).isoformat(timespec="milliseconds")
+
+
+__DICT__CLASSES = (argparse.Namespace,)
 
 
 def replace_non_serializable(obj):
@@ -180,6 +183,8 @@ def replace_non_serializable(obj):
             return obj.to_flowcept_dict()
         elif hasattr(obj, "to_dict"):
             return obj.to_dict()
+        elif isinstance(obj, __DICT__CLASSES):
+            return obj.__dict__
         else:
             # Replace non-serializable values with id()
             return f"{obj.__class__.__name__}_instance_id_{id(obj)}"
@@ -262,6 +267,20 @@ class GenericJSONDecoder(json.JSONDecoder):
         return inst
 
 
+def get_git_info(path: str = "."):
+    """Get Git Repo metadata."""
+    from git import Repo
+
+    repo = Repo(path, search_parent_directories=True)
+    head = repo.head.commit.hexsha
+    short = repo.git.rev_parse(head, short=True)
+    branch = repo.active_branch.name if not repo.head.is_detached else "HEAD"
+    remote = next(iter(repo.remotes)).url if repo.remotes else None
+    dirty = "dirty" if repo.is_dirty() else "clean"
+    root = repo.working_tree_dir
+    return {"sha": head, "short_sha": short, "branch": branch, "root": root, "remote": remote, "dirty": dirty}
+
+
 class ClassProperty:
     """Wrapper to simulate property of class methods, removed in py313."""
 

flowcept/configs.py

@@ -4,29 +4,34 @@ import os
 import socket
 import getpass
 
+from flowcept.version import __version__
+
 PROJECT_NAME = "flowcept"
+
+DEFAULT_SETTINGS = {
+    "version": __version__,
+    "log": {"log_file_level": "disable", "log_stream_level": "disable"},
+    "project": {"dump_buffer_path": "flowcept_messages.jsonl"},
+    "telemetry_capture": {},
+    "instrumentation": {},
+    "experiment": {},
+    "mq": {"enabled": False},
+    "kv_db": {"enabled": False},
+    "web_server": {},
+    "sys_metadata": {},
+    "extra_metadata": {},
+    "analytics": {},
+    "db_buffer": {},
+    "databases": {"mongodb": {"enabled": False}, "lmdb": {"enabled": False}},
+    "adapters": {},
+    "agent": {},
+}
+
 USE_DEFAULT = os.getenv("FLOWCEPT_USE_DEFAULT", "False").lower() == "true"
-########################
-#   Project Settings   #
-########################
 
 if USE_DEFAULT:
-    settings = {
-        "log": {},
-        "project": {},
-        "telemetry_capture": {},
-        "instrumentation": {},
-        "experiment": {},
-        "mq": {},
-        "kv_db": {},
-        "web_server": {},
-        "sys_metadata": {},
-        "extra_metadata": {},
-        "analytics": {},
-        "buffer": {},
-        "databases": {},
-        "adapters": {},
-    }
+    settings = DEFAULT_SETTINGS.copy()
+
 else:
     from omegaconf import OmegaConf
 
@@ -42,7 +47,13 @@ else:
             settings = OmegaConf.load(f)
     else:
         settings = OmegaConf.load(SETTINGS_PATH)
-# print(SETTINGS_PATH)
+
+# Making sure all settings are in place.
+keys = DEFAULT_SETTINGS.keys() - settings.keys()
+if len(keys):
+    for k in keys:
+        settings[k] = DEFAULT_SETTINGS[k]
+
 ########################
 #   Log Settings       #
 ########################
@@ -68,11 +79,13 @@ FLOWCEPT_USER = settings["experiment"].get("user", "blank_user")
 
 MQ_INSTANCES = settings["mq"].get("instances", None)
 MQ_SETTINGS = settings["mq"]
+MQ_ENABLED = os.getenv("MQ_ENABLED", settings["mq"].get("enabled", True))
 MQ_TYPE = os.getenv("MQ_TYPE", settings["mq"].get("type", "redis"))
-MQ_CHANNEL = settings["mq"].get("channel", "interception")
+MQ_CHANNEL = os.getenv("MQ_CHANNEL", settings["mq"].get("channel", "interception"))
 MQ_PASSWORD = settings["mq"].get("password", None)
 MQ_HOST = os.getenv("MQ_HOST", settings["mq"].get("host", "localhost"))
 MQ_PORT = int(os.getenv("MQ_PORT", settings["mq"].get("port", "6379")))
+MQ_URI = os.getenv("MQ_URI", settings["mq"].get("uri", None))
 MQ_BUFFER_SIZE = settings["mq"].get("buffer_size", None)
 MQ_INSERTION_BUFFER_TIME = settings["mq"].get("insertion_buffer_time_secs", None)
 MQ_TIMING = settings["mq"].get("timing", False)
@@ -86,6 +99,7 @@ KVDB_PASSWORD = settings["kv_db"].get("password", None)
 KVDB_HOST = os.getenv("KVDB_HOST", settings["kv_db"].get("host", "localhost"))
 KVDB_PORT = int(os.getenv("KVDB_PORT", settings["kv_db"].get("port", "6379")))
 KVDB_URI = os.getenv("KVDB_URI", settings["kv_db"].get("uri", None))
+KVDB_ENABLED = settings["kv_db"].get("enabled", False)
 
 
 DATABASES = settings.get("databases", {})
@@ -118,9 +132,9 @@ if LMDB_SETTINGS:
     else:
         LMDB_ENABLED = LMDB_SETTINGS.get("enabled", False)
 
-if not LMDB_ENABLED and not MONGO_ENABLED:
-    # At least one of these variables need to be enabled.
-    LMDB_ENABLED = True
+# if not LMDB_ENABLED and not MONGO_ENABLED:
+#     # At least one of these variables need to be enabled.
+#     LMDB_ENABLED = True
 
 ##########################
 # DB Buffer Settings        #
@@ -134,19 +148,20 @@ DB_INSERTER_MAX_TRIALS_STOP = db_buffer_settings.get("stop_max_trials", 240)
 DB_INSERTER_SLEEP_TRIALS_STOP = db_buffer_settings.get("stop_trials_sleep", 0.01)
 
 
-######################
+###########################
 # PROJECT SYSTEM SETTINGS #
-######################
+###########################
 
-DB_FLUSH_MODE = settings["project"].get("db_flush_mode", "online")
+DB_FLUSH_MODE = settings["project"].get("db_flush_mode", "offline")
 # DEBUG_MODE = settings["project"].get("debug", False)
 PERF_LOG = settings["project"].get("performance_logging", False)
 JSON_SERIALIZER = settings["project"].get("json_serializer", "default")
 REPLACE_NON_JSON_SERIALIZABLE = settings["project"].get("replace_non_json_serializable", True)
 ENRICH_MESSAGES = settings["project"].get("enrich_messages", True)
-
+DUMP_BUFFER_PATH = settings["project"].get("dump_buffer_path", None)
 
 TELEMETRY_CAPTURE = settings.get("telemetry_capture", None)
+TELEMETRY_ENABLED = TELEMETRY_CAPTURE is not None and len(TELEMETRY_CAPTURE)
 
 ######################
 # SYS METADATA #
@@ -217,7 +232,11 @@ ANALYTICS = settings.get("analytics", None)
 ####################
 
 INSTRUMENTATION = settings.get("instrumentation", {})
-INSTRUMENTATION_ENABLED = INSTRUMENTATION.get("enabled", False)
+INSTRUMENTATION_ENABLED = INSTRUMENTATION.get("enabled", True)
+
+AGENT = settings.get("agent", {})
+AGENT_HOST = os.getenv("AGENT_HOST", settings["agent"].get("mcp_host", "localhost"))
+AGENT_PORT = int(os.getenv("AGENT_PORT", settings["agent"].get("mcp_port", "8000")))
 
 ####################
 # Enabled ADAPTERS #