flowcept 0.8.11__py3-none-any.whl → 0.9.1__py3-none-any.whl

flowcept/commons/flowcept_dataclasses/task_object.py

@@ -24,6 +24,7 @@ class TaskObject:
     utc_timestamp: float = None
     adapter_id: AnyStr = None
     user: AnyStr = None
+    data: Any = None
     used: Dict[AnyStr, Any] = None  # Used parameter and files
     campaign_id: AnyStr = None
     generated: Dict[AnyStr, Any] = None  # Generated results and files
@@ -53,6 +54,7 @@ class TaskObject:
     dependencies: List = None
     dependents: List = None
     tags: List = None
+    agent_id: str = None
 
     _DEFAULT_ENRICH_VALUES = {
         "node_name": NODE_NAME,
@@ -104,20 +106,16 @@ class TaskObject:
         if self.utc_timestamp is None:
             self.utc_timestamp = flowcept.commons.utils.get_utc_now()
 
-        if self.node_name is None and NODE_NAME is not None:
-            self.node_name = NODE_NAME
-
-        if self.login_name is None and LOGIN_NAME is not None:
-            self.login_name = LOGIN_NAME
-
-        if self.public_ip is None and PUBLIC_IP is not None:
-            self.public_ip = PUBLIC_IP
-
-        if self.private_ip is None and PRIVATE_IP is not None:
-            self.private_ip = PRIVATE_IP
+        for key, fallback_value in TaskObject._DEFAULT_ENRICH_VALUES.items():
+            if getattr(self, key) is None and fallback_value is not None:
+                setattr(self, key, fallback_value)
 
-        if self.hostname is None and HOSTNAME is not None:
-            self.hostname = HOSTNAME
+    @staticmethod
+    def enrich_task_dict(task_dict: dict):
+        """Enrich the task."""
+        for key, fallback_value in TaskObject._DEFAULT_ENRICH_VALUES.items():
+            if (key not in task_dict or task_dict[key] is None) and fallback_value is not None:
+                task_dict[key] = fallback_value
 
     def to_dict(self):
         """Convert to dictionary."""
@@ -139,13 +137,6 @@ class TaskObject:
         """Serialize it."""
         return msgpack.dumps(self.to_dict())
 
-    @staticmethod
-    def enrich_task_dict(task_dict: dict):
-        """Enrich the task."""
-        for key, fallback_value in TaskObject._DEFAULT_ENRICH_VALUES.items():
-            if (key not in task_dict or task_dict[key] is None) and fallback_value is not None:
-                task_dict[key] = fallback_value
-
     @staticmethod
     def from_dict(task_obj_dict: Dict[AnyStr, Any]) -> "TaskObject":
         """Create a TaskObject from a dictionary.
@@ -177,6 +168,10 @@ class TaskObject:
 
     def __repr__(self):
         """Return an unambiguous string representation of the TaskObject."""
-        attrs = ["task_id", "workflow_id", "campaign_id", "activity_id", "custom_metadata", "started_at", "ended_at"]
+        attrs = ["task_id", "workflow_id", "campaign_id", "activity_id", "started_at", "ended_at"]
+        optionals = ["subtype", "parent_task_id", "agent_id"]
+        for opt in optionals:
+            if getattr(self, opt) is not None:
+                attrs.append(opt)
         attr_str = ", ".join(f"{attr}={repr(getattr(self, attr))}" for attr in attrs)
         return f"TaskObject({attr_str})"

flowcept/commons/flowcept_dataclasses/workflow_object.py

@@ -5,7 +5,7 @@ import msgpack
 from omegaconf import OmegaConf, DictConfig
 
 from flowcept.version import __version__
-from flowcept.commons.utils import get_utc_now
+from flowcept.commons.utils import get_utc_now, get_git_info
 from flowcept.configs import (
     settings,
     FLOWCEPT_USER,
@@ -38,6 +38,7 @@ class WorkflowObject:
     sys_name: str = None
     extra_metadata: str = None
     used: Dict = None
+    code_repository: Dict = None
     generated: Dict = None
 
     def __init__(self, workflow_id=None, name=None, used=None, generated=None):
@@ -93,6 +94,13 @@ class WorkflowObject:
             )
             self.extra_metadata = _extra_metadata
 
+        if self.code_repository is None:
+            try:
+                self.code_repository = get_git_info()
+            except Exception as e:
+                print(e)
+                pass
+
         if self.flowcept_version is None:
             self.flowcept_version = __version__
 

flowcept/commons/task_data_preprocess.py

@@ -1,35 +1,13 @@
-"""
-The base of this code was generated using ChatGPT.
-
-Prompt:
-
-Here I have a list containing one real task.
-
-<Paste one real task here>
-
-I want to create a list of summarized task data, per task, containing:
-- activity_id
-- task_id
-- used
-- generated
-- task_duration (ended_at - started_at)
-- hostname
-- cpu_info
-- disk_info
-- mem_info
-- network_info
-<Consider adding GPU info too, if you have gpu in your task data>
-
-Where info about cpu, disk, mem, and network must consider telemetry_at_end and telemetry_at_start.
-
-We will use this summarized data as input for LLM questions to find patterns in the resource usage and how they relate
-to input (used) and output (generated) of each task.
-"""
-
+from datetime import datetime
 from typing import Dict, List
+import copy
+from collections import defaultdict
+from typing import Any
+
+import pytz
 
 
-def summarize_telemetry(task: Dict) -> Dict:
+def summarize_telemetry(task: Dict, logger) -> Dict:
     """
     Extract and compute the telemetry summary for a task based on start and end telemetry snapshots.
 
@@ -79,24 +57,55 @@ def summarize_telemetry(task: Dict) -> Dict:
             "packets_recv_diff": net_end["packets_recv"] - net_start["packets_recv"],
         }
 
-    start_tele = task["telemetry_at_start"]
-    end_tele = task["telemetry_at_end"]
+    tel_funcs = {
+        "cpu": extract_cpu_info,
+        "disk": extract_disk_info,
+        "memory": extract_mem_info,
+        "network": extract_network_info,
+    }
+
+    start_tele = task.get("telemetry_at_start", {})
+    end_tele = task.get("telemetry_at_end", {})
 
-    started_at = task["started_at"]
-    ended_at = task["ended_at"]
-    duration = ended_at - started_at
+    telemetry_summary = {}
 
-    telemetry_summary = {
-        "duration_sec": duration,
-        "cpu_info": extract_cpu_info(start_tele["cpu"], end_tele["cpu"]),
-        "disk_info": extract_disk_info(start_tele["disk"], end_tele["disk"]),
-        "mem_info": extract_mem_info(start_tele["memory"], end_tele["memory"]),
-        "network_info": extract_network_info(start_tele["network"], end_tele["network"]),
-    }
+    try:
+        started_at = task.get("started_at", None)
+        ended_at = task.get("ended_at", None)
+        if started_at is None or ended_at is None:
+            logger.warning(f"We can't summarize telemetry for duration_sec for task {task}")
+        else:
+            duration = ended_at - started_at
+            telemetry_summary["duration_sec"] = duration
+    except Exception as e:
+        logger.error(f"Error to summarize telemetry for duration_sec in {task}")
+        logger.exception(e)
+
+    for key in start_tele.keys():
+        try:
+            if key not in tel_funcs:
+                continue
+            func = tel_funcs[key]
+            if key in end_tele:
+                telemetry_summary[key] = func(start_tele[key], end_tele[key])
+            else:
+                logger.warning(
+                    f"We can't summarize telemetry {key} for task {task} because the key is not in the end_tele"
+                )
+        except Exception as e:
+            logger.warning(f"Error to summarize telemetry for {key} for task {task}. Exception: {e}")
+            logger.exception(e)
 
     return telemetry_summary
 
 
+def _safe_get(task, key):
+    try:
+        return task.get(key)
+    except Exception:
+        return None
+
+
 def summarize_task(task: Dict, thresholds: Dict = None, logger=None) -> Dict:
     """
     Summarize key metadata and telemetry for a task, optionally tagging critical conditions.
@@ -113,23 +122,51 @@ def summarize_task(task: Dict, thresholds: Dict = None, logger=None) -> Dict:
     dict
         Summary of the task including identifiers, telemetry summary, and optional critical tags.
     """
-    task_summary = {
-        "workflow_id": task.get("workflow_id"),
-        "task_id": task.get("task_id"),
-        "activity_id": task.get("activity_id"),
-        "used": task.get("used"),
-        "generated": task.get("generated"),
-        "hostname": task.get("hostname"),
-        "status": task.get("status"),
-    }
+    task_summary = {}
+
+    # Keys that can be copied directly
+    for key in [
+        "workflow_id",
+        "task_id",
+        "parent_task_id",
+        "activity_id",
+        "used",
+        "generated",
+        "hostname",
+        "status",
+        "agent_id",
+        "campaign_id",
+        "subtype",
+        "custom_metadata",
+    ]:
+        value = _safe_get(task, key)
+        if value is not None:
+            if "_id" in key:
+                task_summary[key] = str(value)
+            else:
+                task_summary[key] = value
+
+        # Special handling for timestamp field
+    try:
+        time_keys = ["started_at", "ended_at"]
+        for time_key in time_keys:
+            timestamp = _safe_get(task, time_key)
+            if timestamp is not None:
+                task_summary[time_key] = datetime.fromtimestamp(timestamp, pytz.utc)
+    except Exception as e:
+        if logger:
+            logger.exception(f"Error {e} converting timestamp for task {task.get('task_id', 'unknown')}")
 
     try:
-        telemetry_summary = summarize_telemetry(task)
-        tags = tag_critical_task(
-            generated=task.get("generated", {}), telemetry_summary=telemetry_summary, thresholds=thresholds
-        )
-        if tags:
-            task_summary["tags"] = tags
+        telemetry_summary = summarize_telemetry(task, logger)
+        try:
+            tags = tag_critical_task(
+                generated=task.get("generated", {}), telemetry_summary=telemetry_summary, thresholds=thresholds
+            )
+            if tags:
+                task_summary["tags"] = tags
+        except Exception as e:
+            logger.exception(e)
         task_summary["telemetry_summary"] = telemetry_summary
     except Exception as e:
         if logger:
@@ -172,12 +209,13 @@ def tag_critical_task(
             "high_output": 0.9,
         }
 
-    cpu = abs(telemetry_summary["cpu_info"].get("percent_all_diff", 0))
-    mem = telemetry_summary["mem_info"].get("used_mem_diff", 0)
-    disk = telemetry_summary["disk_info"].get("read_bytes_diff", 0) + telemetry_summary["disk_info"].get(
+    cpu = abs(telemetry_summary.get("cpu", {}).get("percent_all_diff", 0))
+    mem = telemetry_summary.get("mem", {}).get("used_mem_diff", 0)
+    disk = telemetry_summary.get("disk", {}).get("read_bytes_diff", 0) + telemetry_summary.get("disk", {}).get(
         "write_bytes_diff", 0
     )
-    duration = telemetry_summary["duration_sec"]
+    # TODO gpu
+    duration = telemetry_summary.get("duration_sec", 0)
 
     tags = []
 
@@ -198,3 +236,165 @@ def tag_critical_task(
             tags.append("high_output")
 
     return tags
+
+
+sample_tasks = [
+    {
+        "task_id": "t1",
+        "activity_id": "train_model",
+        "used": {
+            "dataset": {"name": "MNIST", "size": 60000, "source": {"url": "http://example.com/mnist", "format": "csv"}},
+            "params": {"epochs": 5, "batch_size": 32, "shuffle": True},
+        },
+        "generated": {"model": {"accuracy": 0.98, "layers": [64, 64, 10], "saved_path": "/models/mnist_v1.pth"}},
+        "telemetry_summary": {"duration_sec": 42.7, "cpu_percent": 85.2},
+    },
+    {
+        "task_id": "t2",
+        "activity_id": "train_model",
+        "used": {
+            "dataset": {
+                "name": "CIFAR-10",
+                "size": 50000,
+                "source": {"url": "http://example.com/cifar", "format": "jpeg"},
+            },
+            "params": {"epochs": 10, "batch_size": 64, "shuffle": False},
+        },
+        "generated": {"model": {"accuracy": 0.91, "layers": [128, 128, 10], "saved_path": "/models/cifar_v1.pth"}},
+        "telemetry_summary": {"duration_sec": 120.5, "cpu_percent": 92.0},
+    },
+    {
+        "task_id": "t3",
+        "activity_id": "evaluate_model",
+        "used": {"model_path": "/models/mnist_v1.pth", "test_data": {"name": "MNIST-test", "samples": 10000}},
+        "generated": {"metrics": {"accuracy": 0.97, "confusion_matrix": [[8500, 100], [50, 1350]]}},
+        "telemetry_summary": {"duration_sec": 15.3},
+    },
+    {
+        "task_id": "t4",
+        "activity_id": "evaluate_model",
+        "used": {"model_path": "/models/cifar_v1.pth", "test_data": {"name": "CIFAR-test", "samples": 10000}},
+        "generated": {"metrics": {"accuracy": 0.88, "confusion_matrix": [[4000, 500], [300, 5200]]}},
+        "telemetry_summary": {"duration_sec": 18.9},
+    },
+]
+
+
+def infer_dtype(value: Any) -> str:
+    """Infer a simplified dtype label for the value."""
+    if isinstance(value, bool):
+        return "bool"
+    elif isinstance(value, int):
+        return "int"
+    elif isinstance(value, float):
+        return "float"
+    elif isinstance(value, str):
+        return "str"
+    elif isinstance(value, list):
+        return "list"
+    return "str"  # fallback for other types
+
+
+def flatten_dict(d: dict, parent_key: str = "", sep: str = ".") -> dict:
+    """Recursively flatten nested dicts using dot notation."""
+    items = {}
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.update(flatten_dict(v, new_key, sep=sep))
+        else:
+            items[new_key] = v
+    return items
+
+
+def update_schema(schema_section: list, flat_fields: dict):
+    """Update schema section with flattened fields and example values."""
+    field_map = {f["n"]: f for f in schema_section}
+
+    for key, value in flat_fields.items():
+        dtype = infer_dtype(value)
+        if isinstance(value, float):
+            val_repr = round(value, 2)
+        elif isinstance(value, (dict, list)):
+            val_repr = str(value)
+        else:
+            val_repr = value
+
+        if isinstance(val_repr, str) and len(val_repr) > 100:
+            val_repr = val_repr[:100] + "#TRUNCATED"
+
+        if key not in field_map:
+            field = {
+                "n": key,
+                "d": dtype,
+                "v": [val_repr] if val_repr is not None else [],
+            }
+            schema_section.append(field)
+            field_map[key] = field
+        else:
+            field = field_map[key]
+            if val_repr not in field["v"] and len(field["v"]) < 3:
+                field["v"].append(val_repr)
+
+
+def update_tasks_summary_schema(tasks: list[dict], schema) -> dict:
+    """Update tasks_summary schema."""
+    act_schema = update_activity_schema(tasks)
+    merged_schema = deep_merge_dicts(act_schema, schema)
+    return merged_schema
+
+
+def update_activity_schema(tasks: list[dict]) -> dict:
+    """Build schema for each activity_id from list of task dicts."""
+    schema = defaultdict(
+        lambda: {
+            "in": [],
+            "out": [],
+            # "tel": [],
+        }
+    )
+
+    for task in tasks:
+        activity_id = task.get("activity_id")
+        if not activity_id:
+            continue
+
+        activity_schema = schema[activity_id]
+
+        for section_key, schema_key in [
+            ("used", "in"),
+            ("generated", "out"),
+            #   ("telemetry_summary", "tel"),
+        ]:
+            section_data = task.get(section_key)
+            if isinstance(section_data, dict):
+                flat_fields = flatten_dict(section_data, parent_key=section_key)
+                update_schema(activity_schema[schema_key], flat_fields)
+
+    schema = dict(schema)
+    return schema
+
+
+def deep_merge_dicts(a: dict, b: dict) -> dict:
+    """
+    Recursively merge dict b into dict a:
+    - Does not overwrite existing values in a.
+    - If both values are dicts, merges recursively.
+    - If both values are lists, concatenates and deduplicates.
+    - Otherwise, keeps value from a.
+    Returns a new dict (does not mutate inputs).
+    """
+    result = copy.deepcopy(a)
+
+    for key, b_val in b.items():
+        if key not in result:
+            result[key] = copy.deepcopy(b_val)
+        else:
+            a_val = result[key]
+            if isinstance(a_val, dict) and isinstance(b_val, dict):
+                result[key] = deep_merge_dicts(a_val, b_val)
+            elif isinstance(a_val, list) and isinstance(b_val, list):
+                combined = a_val + [item for item in b_val if item not in a_val]
+                result[key] = combined
+            # preserve a_val otherwise
+    return result

flowcept/commons/utils.py

@@ -1,5 +1,6 @@
 """Utilities."""
 
+import argparse
 from datetime import datetime, timedelta, timezone
 import json
 from time import time, sleep
@@ -9,7 +10,6 @@ import platform
 import subprocess
 import types
 import numpy as np
-import pytz
 
 from flowcept import configs
 from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
@@ -19,7 +19,7 @@ from flowcept.commons.vocabulary import Status
 
 
 def get_utc_now() -> float:
-    """Get UTC time."""
+    """Get current UTC time as a timestamp (seconds since epoch)."""
     now = datetime.now(timezone.utc)
     return now.timestamp()
 
@@ -159,11 +159,14 @@ class GenericJSONEncoder(json.JSONEncoder):
         return super().default(obj)
 
 
-def replace_non_serializable_times(obj, tz=pytz.utc):
-    """Replace non-serializable times in an object."""
+def replace_non_serializable_times(obj, tz=timezone.utc):
+    """Replace non-serializable datetimes in an object with ISO 8601 strings (ms precision)."""
     for time_field in TaskObject.get_time_field_names():
-        if time_field in obj:
-            obj[time_field] = obj[time_field].strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + f" {tz}"
+        if time_field in obj and isinstance(obj[time_field], datetime):
+            obj[time_field] = obj[time_field].astimezone(tz).isoformat(timespec="milliseconds")
+
+
+__DICT__CLASSES = (argparse.Namespace,)
 
 
 def replace_non_serializable(obj):
@@ -180,6 +183,8 @@ def replace_non_serializable(obj):
             return obj.to_flowcept_dict()
         elif hasattr(obj, "to_dict"):
             return obj.to_dict()
+        elif isinstance(obj, __DICT__CLASSES):
+            return obj.__dict__
         else:
             # Replace non-serializable values with id()
             return f"{obj.__class__.__name__}_instance_id_{id(obj)}"
@@ -262,6 +267,20 @@ class GenericJSONDecoder(json.JSONDecoder):
         return inst
 
 
+def get_git_info(path: str = "."):
+    """Get Git Repo metadata."""
+    from git import Repo
+
+    repo = Repo(path, search_parent_directories=True)
+    head = repo.head.commit.hexsha
+    short = repo.git.rev_parse(head, short=True)
+    branch = repo.active_branch.name if not repo.head.is_detached else "HEAD"
+    remote = next(iter(repo.remotes)).url if repo.remotes else None
+    dirty = "dirty" if repo.is_dirty() else "clean"
+    root = repo.working_tree_dir
+    return {"sha": head, "short_sha": short, "branch": branch, "root": root, "remote": remote, "dirty": dirty}
+
+
 class ClassProperty:
     """Wrapper to simulate property of class methods, removed in py313."""
 

flowcept/configs.py

@@ -4,29 +4,34 @@ import os
 import socket
 import getpass
 
+from flowcept.version import __version__
+
 PROJECT_NAME = "flowcept"
+
+DEFAULT_SETTINGS = {
+    "version": __version__,
+    "log": {"log_file_level": "disable", "log_stream_level": "disable"},
+    "project": {"dump_buffer_path": "flowcept_messages.jsonl"},
+    "telemetry_capture": {},
+    "instrumentation": {},
+    "experiment": {},
+    "mq": {"enabled": False},
+    "kv_db": {"enabled": False},
+    "web_server": {},
+    "sys_metadata": {},
+    "extra_metadata": {},
+    "analytics": {},
+    "db_buffer": {},
+    "databases": {"mongodb": {"enabled": False}, "lmdb": {"enabled": False}},
+    "adapters": {},
+    "agent": {},
+}
+
 USE_DEFAULT = os.getenv("FLOWCEPT_USE_DEFAULT", "False").lower() == "true"
-########################
-#   Project Settings   #
-########################
 
 if USE_DEFAULT:
-    settings = {
-        "log": {},
-        "project": {},
-        "telemetry_capture": {},
-        "instrumentation": {},
-        "experiment": {},
-        "mq": {},
-        "kv_db": {},
-        "web_server": {},
-        "sys_metadata": {},
-        "extra_metadata": {},
-        "analytics": {},
-        "buffer": {},
-        "databases": {},
-        "adapters": {},
-    }
+    settings = DEFAULT_SETTINGS.copy()
+
 else:
     from omegaconf import OmegaConf
 
@@ -42,7 +47,13 @@ else:
             settings = OmegaConf.load(f)
     else:
         settings = OmegaConf.load(SETTINGS_PATH)
-# print(SETTINGS_PATH)
+
+# Making sure all settings are in place.
+keys = DEFAULT_SETTINGS.keys() - settings.keys()
+if len(keys):
+    for k in keys:
+        settings[k] = DEFAULT_SETTINGS[k]
+
 ########################
 #   Log Settings       #
 ########################
@@ -68,6 +79,7 @@ FLOWCEPT_USER = settings["experiment"].get("user", "blank_user")
 
 MQ_INSTANCES = settings["mq"].get("instances", None)
 MQ_SETTINGS = settings["mq"]
+MQ_ENABLED = os.getenv("MQ_ENABLED", settings["mq"].get("enabled", True))
 MQ_TYPE = os.getenv("MQ_TYPE", settings["mq"].get("type", "redis"))
 MQ_CHANNEL = os.getenv("MQ_CHANNEL", settings["mq"].get("channel", "interception"))
 MQ_PASSWORD = settings["mq"].get("password", None)
@@ -87,7 +99,7 @@ KVDB_PASSWORD = settings["kv_db"].get("password", None)
 KVDB_HOST = os.getenv("KVDB_HOST", settings["kv_db"].get("host", "localhost"))
 KVDB_PORT = int(os.getenv("KVDB_PORT", settings["kv_db"].get("port", "6379")))
 KVDB_URI = os.getenv("KVDB_URI", settings["kv_db"].get("uri", None))
-KVDB_ENABLED = settings["kv_db"].get("enabled", True)
+KVDB_ENABLED = settings["kv_db"].get("enabled", False)
 
 
 DATABASES = settings.get("databases", {})
@@ -136,19 +148,20 @@ DB_INSERTER_MAX_TRIALS_STOP = db_buffer_settings.get("stop_max_trials", 240)
 DB_INSERTER_SLEEP_TRIALS_STOP = db_buffer_settings.get("stop_trials_sleep", 0.01)
 
 
-######################
+###########################
 # PROJECT SYSTEM SETTINGS #
-######################
+###########################
 
-DB_FLUSH_MODE = settings["project"].get("db_flush_mode", "online")
+DB_FLUSH_MODE = settings["project"].get("db_flush_mode", "offline")
 # DEBUG_MODE = settings["project"].get("debug", False)
 PERF_LOG = settings["project"].get("performance_logging", False)
 JSON_SERIALIZER = settings["project"].get("json_serializer", "default")
 REPLACE_NON_JSON_SERIALIZABLE = settings["project"].get("replace_non_json_serializable", True)
 ENRICH_MESSAGES = settings["project"].get("enrich_messages", True)
-
+DUMP_BUFFER_PATH = settings["project"].get("dump_buffer_path", None)
 
 TELEMETRY_CAPTURE = settings.get("telemetry_capture", None)
+TELEMETRY_ENABLED = TELEMETRY_CAPTURE is not None and len(TELEMETRY_CAPTURE)
 
 ######################
 # SYS METADATA #
@@ -219,9 +232,11 @@ ANALYTICS = settings.get("analytics", None)
 ####################
 
 INSTRUMENTATION = settings.get("instrumentation", {})
-INSTRUMENTATION_ENABLED = INSTRUMENTATION.get("enabled", False)
+INSTRUMENTATION_ENABLED = INSTRUMENTATION.get("enabled", True)
 
 AGENT = settings.get("agent", {})
+AGENT_HOST = os.getenv("AGENT_HOST", settings["agent"].get("mcp_host", "localhost"))
+AGENT_PORT = int(os.getenv("AGENT_PORT", settings["agent"].get("mcp_port", "8000")))
 
 ####################
 # Enabled ADAPTERS #