flowcept 0.8.11__py3-none-any.whl → 0.9.1__py3-none-any.whl

flowcept/__init__.py

@@ -2,10 +2,6 @@
 
 from flowcept.version import __version__
 
-from flowcept.commons.flowcept_dataclasses.workflow_object import (
-    WorkflowObject,
-)
-
 
 def __getattr__(name):
     if name == "Flowcept":
@@ -13,6 +9,13 @@ def __getattr__(name):
 
         return Flowcept
 
+    elif name == "WorkflowObject":
+        from flowcept.commons.flowcept_dataclasses.workflow_object import (
+            WorkflowObject,
+        )
+
+        return WorkflowObject
+
     elif name == "flowcept_task":
         from flowcept.instrumentation.flowcept_task import flowcept_task
 

flowcept/agents/__init__.py

@@ -0,0 +1,5 @@
+# flake8: noqa: F403
+"""Agents subpackage."""
+
+from flowcept.agents.tools.general_tools import *
+from flowcept.agents.tools.in_memory_queries.in_memory_queries_tools import *

flowcept/{flowceptor/consumers/agent/client_agent.py → agents/agent_client.py}

@@ -1,17 +1,15 @@
-from typing import Dict, List
+import asyncio
+from typing import Dict, List, Callable
 
-from flowcept.configs import AGENT
+from flowcept.configs import AGENT_HOST, AGENT_PORT
 from mcp import ClientSession
 from mcp.client.streamable_http import streamablehttp_client
 from mcp.types import TextContent
 
-MCP_HOST = AGENT.get("mcp_host", "0.0.0.0")
-MCP_PORT = AGENT.get("mcp_port", 8000)
 
-MCP_URL = f"http://{MCP_HOST}:{MCP_PORT}/mcp"
-
-
-def run_tool(tool_name: str, kwargs: Dict = None) -> List[TextContent]:
+def run_tool(
+    tool_name: str | Callable, kwargs: Dict = None, host: str = AGENT_HOST, port: int = AGENT_PORT
+) -> List[str]:
     """
     Run a tool using an MCP client session via a local streamable HTTP connection.
 
@@ -36,13 +34,25 @@ def run_tool(tool_name: str, kwargs: Dict = None) -> List[TextContent]:
     This function uses `asyncio.run`, so it must not be called from an already-running
     event loop (e.g., inside another async function in environments like Jupyter).
     """
-    import asyncio
+    if isinstance(tool_name, Callable):
+        tool_name = tool_name.__name__
 
     async def _run():
-        async with streamablehttp_client(MCP_URL) as (read, write, _):
+        mcp_url = f"http://{host}:{port}/mcp"
+        print(mcp_url)
+        print(tool_name)
+
+        async with streamablehttp_client(mcp_url) as (read, write, _):
             async with ClientSession(read, write) as session:
                 await session.initialize()
-                result = await session.call_tool(tool_name, arguments=kwargs)
-                return result.content
+                result: List[TextContent] = await session.call_tool(tool_name, arguments=kwargs)
+                actual_result = []
+                for r in result.content:
+                    if isinstance(r, str):
+                        actual_result.append(r)
+                    else:
+                        actual_result.append(r.text)
+
+                return actual_result
 
     return asyncio.run(_run())

flowcept/agents/agents_utils.py

@@ -0,0 +1,181 @@
+import os
+from typing import Union, Dict
+
+from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager
+from flowcept.instrumentation.flowcept_agent_task import FlowceptLLM, get_current_context_task
+
+from flowcept.configs import AGENT
+from pydantic import BaseModel
+
+
+class ToolResult(BaseModel):
+    """
+    ToolResult is a standardized wrapper for tool outputs, encapsulating
+    status codes, results, and optional metadata.
+
+    This class provides conventions for interpreting the output of tools
+    (e.g., LLM calls, DataFrame operations, plotting functions) and ensures
+    consistent handling of both successes and errors.
+
+    Conventions
+    -----------
+    - **2xx: Success (string result)**
+      - Result is the expected output as a string.
+      - Example: ``201`` → operation completed successfully.
+
+    - **3xx: Success (dict result)**
+      - Result is the expected output as a dictionary.
+      - Example: ``301`` → operation completed successfully.
+
+    - **4xx: Error (string message)**
+      - System or agent internal error, returned as a string message.
+      - ``400``: LLM call problem (e.g., server connection or token issues).
+      - ``404``: Empty or ``None`` result.
+      - ``405``: LLM responded, but format was wrong.
+      - ``406``: Error executing Python code.
+      - ``499``: Other uncategorized error.
+
+    - **5xx: Error (dict result)**
+      - System or agent internal error, returned as a structured dictionary.
+
+    - **None**
+      - Result not yet set or tool did not return anything.
+
+    Attributes
+    ----------
+    code : int or None
+        Status code indicating success or error category.
+    result : str or dict, optional
+        The main output of the tool (string, dict, or error message).
+    extra : dict or str or None
+        Additional metadata or debugging information.
+    tool_name : str or None
+        Name of the tool that produced this result.
+
+    Methods
+    -------
+    result_is_str() -> bool
+        Return True if the result should be interpreted as a string.
+    is_success() -> bool
+        Return True if the result represents any type of success.
+    is_success_string() -> bool
+        Return True if the result is a success with a string output (2xx).
+    is_error_string() -> bool
+        Return True if the result is an error with a string message (4xx).
+    is_success_dict() -> bool
+        Return True if the result is a success with a dict output (3xx).
+
+    Examples
+    --------
+    >>> ToolResult(code=201, result="Operation successful")
+    ToolResult(code=201, result='Operation successful')
+
+    >>> ToolResult(code=301, result={"data": [1, 2, 3]})
+    ToolResult(code=301, result={'data': [1, 2, 3]})
+
+    >>> ToolResult(code=405, result="Invalid format from LLM")
+    ToolResult(code=405, result='Invalid format from LLM')
+    """
+
+    code: int | None = None
+    result: Union[str, Dict] = None
+    extra: Dict | str | None = None
+    tool_name: str | None = None
+
+    def result_is_str(self) -> bool:
+        """Returns True if the result is a string."""
+        return (200 <= self.code < 300) or (400 <= self.code < 500)
+
+    def is_success(self):
+        """Returns True if the result is a success."""
+        return self.is_success_string() or self.is_success_dict()
+
+    def is_success_string(self):
+        """Returns True if the result is a success string."""
+        return 200 <= self.code < 300
+
+    def is_error_string(self):
+        """Returns True if the result is an error string."""
+        return 400 <= self.code < 500
+
+    def is_success_dict(self) -> bool:
+        """Returns True if the result is a success dictionary."""
+        return 300 <= self.code < 400
+
+
+def build_llm_model(
+    model_name=None,
+    model_kwargs=None,
+    service_provider=None,
+    agent_id=BaseAgentContextManager.agent_id,
+    track_tools=True,
+) -> FlowceptLLM:
+    """
+    Build and return an LLM instance using agent configuration.
+
+    This function retrieves the model name and keyword arguments from the AGENT configuration,
+    constructs a SambaStudio LLM instance, and returns it.
+
+    Returns
+    -------
+    LLM
+        An initialized LLM object configured using the `AGENT` settings.
+    """
+    _model_kwargs = AGENT.get("model_kwargs", {}).copy()
+    if model_kwargs is not None:
+        for k in model_kwargs:
+            _model_kwargs[k] = model_kwargs[k]
+
+    if "model" not in _model_kwargs:
+        _model_kwargs["model"] = AGENT.get("model", model_name)
+
+    if service_provider:
+        _service_provider = service_provider
+    else:
+        _service_provider = AGENT.get("service_provider")
+
+    if _service_provider == "sambanova":
+        from langchain_community.llms.sambanova import SambaStudio
+
+        os.environ["SAMBASTUDIO_URL"] = AGENT.get("llm_server_url")
+        os.environ["SAMBASTUDIO_API_KEY"] = AGENT.get("api_key")
+
+        llm = SambaStudio(model_kwargs=_model_kwargs)
+    elif _service_provider == "azure":
+        from langchain_openai.chat_models.azure import AzureChatOpenAI
+
+        api_key = os.environ.get("AZURE_OPENAI_API_KEY", AGENT.get("api_key", None))
+        service_url = os.environ.get("AZURE_OPENAI_API_ENDPOINT", AGENT.get("llm_server_url", None))
+        llm = AzureChatOpenAI(
+            azure_deployment=_model_kwargs.get("model"), azure_endpoint=service_url, api_key=api_key, **_model_kwargs
+        )
+    elif _service_provider == "openai":
+        from langchain_openai import ChatOpenAI
+
+        api_key = os.environ.get("OPENAI_API_KEY", AGENT.get("api_key", None))
+        llm = ChatOpenAI(openai_api_key=api_key, **model_kwargs)
+    elif _service_provider == "google":
+        if "claude" in _model_kwargs["model"]:
+            api_key = os.environ.get("GOOGLE_API_KEY", AGENT.get("api_key", None))
+            _model_kwargs["model_id"] = _model_kwargs.pop("model")
+            _model_kwargs["google_token_auth"] = api_key
+            from flowcept.agents.llms.claude_gcp import ClaudeOnGCPLLM
+
+            llm = ClaudeOnGCPLLM(**_model_kwargs)
+        elif "gemini" in _model_kwargs["model"]:
+            from flowcept.agents.llms.gemini25 import Gemini25LLM
+
+            llm = Gemini25LLM(**_model_kwargs)
+
+    else:
+        raise Exception("Currently supported providers are sambanova, openai, azure, and google.")
+    if track_tools:
+        llm = FlowceptLLM(llm)
+        if agent_id is None:
+            agent_id = BaseAgentContextManager.agent_id
+        llm.agent_id = agent_id
+        if track_tools:
+            tool_task = get_current_context_task()
+            if tool_task:
+                llm.parent_task_id = tool_task.task_id
+    return llm

flowcept/agents/dynamic_schema_tracker.py

@@ -0,0 +1,191 @@
+import json
+
+
+class DynamicSchemaTracker:
+    """
+    DynamicSchemaTracker maintains and updates a dynamic schema of tasks,
+    tracking both input and output fields along with example values and type
+    information. It is designed to help build lightweight, evolving schemas
+    from observed task executions.
+
+    The tracker flattens nested structures, deduplicates fields, and captures
+    examples of values (truncated when necessary) to provide insight into
+    the shape and types of task data over time.
+
+    Parameters
+    ----------
+    max_examples : int, default=3
+        Maximum number of example values to store for each field.
+    max_str_len : int, default=70
+        Maximum string length for stored example values. Longer values are
+        truncated with ellipsis.
+
+    Attributes
+    ----------
+    schema : dict
+        Maps activity IDs to dictionaries containing lists of input ("i")
+        and output ("o") fields. Example:
+        ``{"train_model": {"i": ["used.dataset"], "o": ["generated.metrics"]}}``.
+    values : dict
+        Maps normalized field names to metadata about their values, including:
+        - ``v`` : list of example values (up to ``max_examples``).
+        - ``t`` : type of the field ("int", "float", "list", "str", etc.).
+        - ``s`` : shape of lists (if applicable).
+        - ``et`` : element type for lists (if applicable).
+    max_examples : int
+        Maximum number of examples per field.
+    max_str_len : int
+        Maximum stored string length for example values.
+
+    Methods
+    -------
+    update_with_tasks(tasks)
+        Update the schema and value examples with a list of tasks.
+    get_schema()
+        Retrieve the current schema with prefixed "used." and "generated." fields.
+    get_example_values()
+        Retrieve deduplicated example values and type information for fields.
+
+    Examples
+    --------
+    >>> tracker = DynamicSchemaTracker(max_examples=2, max_str_len=20)
+    >>> tasks = [
+    ...     {"activity_id": "task1",
+    ...      "used": {"input": [1, 2, 3]},
+    ...      "generated": {"output": {"score": 0.95}}}
+    ... ]
+    >>> tracker.update_with_tasks(tasks)
+    >>> tracker.get_schema()
+    {'task1': {'i': ['used.input'], 'o': ['generated.output.score']}}
+    >>> tracker.get_example_values()
+    {'input': {'v': [[1, 2, 3]], 't': 'list', 's': [3], 'et': 'int'},
+     'output.score': {'v': [0.95], 't': 'float'}}
+    """
+
+    def __init__(self, max_examples=3, max_str_len=70):
+        self.schema = {}  # {activity_id: {"i": [...], "o": [...]}}
+
+        # {normalized_field: {"v": [...], "t": ..., "s": ..., "et": ...}}
+        self.values = {}
+
+        self.max_examples = max_examples
+        self.max_str_len = max_str_len
+
+    def _flatten_dict(self, d, parent_key="", sep="."):
+        """Flatten dictionary but preserve lists as single units."""
+        items = []
+        for k, v in d.items():
+            new_key = f"{parent_key}{sep}{k}" if parent_key else k
+            if isinstance(v, dict):
+                items.extend(self._flatten_dict(v, new_key, sep=sep).items())
+            else:
+                items.append((new_key, v))
+        return dict(items)
+
+    def _truncate_if_needed(self, val):
+        """Truncate if stringified length exceeds max_str_len."""
+        try:
+            s = json.dumps(val)
+        except Exception:
+            s = str(val)
+
+        if len(s) > self.max_str_len:
+            return s[: self.max_str_len] + "..."
+        return val
+
+    def _get_type(self, val):
+        if isinstance(val, bool):
+            return "bool"
+        elif isinstance(val, int):
+            return "int"
+        elif isinstance(val, float):
+            return "float"
+        elif isinstance(val, list):
+            return "list"
+        else:
+            return "str"
+
+    def _get_shape(self, val):
+        if not isinstance(val, list):
+            return None
+        shape = []
+        while isinstance(val, list):
+            shape.append(len(val))
+            if not val:  # Empty list -> stop
+                break
+            val = val[0]
+        return shape
+
+    def _get_list_element_type(self, val):
+        if not isinstance(val, list):
+            return None
+
+        def describe(elem):
+            if isinstance(elem, list):
+                return f"list[{describe(elem[0])}]" if elem else "list[unknown]"
+            elif isinstance(elem, dict):
+                return "dict"
+            elif isinstance(elem, bool):
+                return "bool"
+            elif isinstance(elem, int):
+                return "int"
+            elif isinstance(elem, float):
+                return "float"
+            elif isinstance(elem, str):
+                return "str"
+            else:
+                return "unknown"
+
+        return describe(val[0]) if val else "unknown"
+
+    def _add_schema_field(self, activity_id, field_name, direction):
+        key = "i" if direction == "used" else "o"
+        if field_name not in self.schema[activity_id][key]:
+            self.schema[activity_id][key].append(field_name)
+
+    def _add_value_info(self, normalized_field, val):
+        val_type = self._get_type(val)
+        truncated_val = self._truncate_if_needed(val)
+
+        entry = self.values.setdefault(normalized_field, {"v": [], "t": val_type})
+
+        # Always reflect latest observed type
+        entry["t"] = val_type
+
+        if val_type == "list":
+            entry["s"] = self._get_shape(val)
+            entry["et"] = self._get_list_element_type(val)
+        else:
+            entry.pop("s", None)
+            entry.pop("et", None)
+
+        if truncated_val not in entry["v"]:
+            entry["v"].append(truncated_val)
+
+        if len(entry["v"]) > self.max_examples:
+            entry["v"] = sorted(entry["v"], key=lambda x: str(x))[: self.max_examples]
+
+    def update_with_tasks(self, tasks):
+        """Update the schema with tasks."""
+        for task in tasks:
+            activity = task.get("activity_id")
+            if activity not in self.schema:
+                self.schema[activity] = {"i": [], "o": []}
+
+            for direction in ["used", "generated"]:
+                data = task.get(direction, {})
+                flat_data = self._flatten_dict(data)
+                for field, val in flat_data.items():
+                    prefixed_field = f"{direction}.{field}"
+                    normalized_field = field  # role-agnostic key for value descriptions
+
+                    self._add_schema_field(activity, prefixed_field, direction)
+                    self._add_value_info(normalized_field, val)
+
+    def get_schema(self):
+        """Get the current schema."""
+        return self.schema  # fields with 'used.' or 'generated.' prefix
+
+    def get_example_values(self):
+        """Get example values."""
+        return self.values  # deduplicated field schemas

flowcept/agents/flowcept_agent.py

@@ -0,0 +1,30 @@
+from threading import Thread
+from time import sleep
+
+from flowcept.agents import check_liveness
+from flowcept.agents.agent_client import run_tool
+from flowcept.agents.flowcept_ctx_manager import mcp_flowcept
+from flowcept.configs import AGENT_HOST, AGENT_PORT
+from flowcept.flowcept_api.flowcept_controller import Flowcept
+
+import uvicorn
+
+
+def main():
+    """
+    Start the MCP server.
+    """
+    f = Flowcept(start_persistence=False, save_workflow=False, check_safe_stops=False).start()
+    f.logger.info(f"This section's workflow_id={Flowcept.current_workflow_id}")
+
+    def run():
+        uvicorn.run(mcp_flowcept.streamable_http_app, host=AGENT_HOST, port=AGENT_PORT, lifespan="on")
+
+    Thread(target=run).start()
+    sleep(2)
+    # Wake up tool call
+    print(run_tool(check_liveness, host=AGENT_HOST, port=AGENT_PORT)[0])
+
+
+if __name__ == "__main__":
+    main()

flowcept/agents/flowcept_ctx_manager.py

@@ -0,0 +1,175 @@
+from flowcept.agents.dynamic_schema_tracker import DynamicSchemaTracker
+from flowcept.agents.tools.in_memory_queries.pandas_agent_utils import load_saved_df
+from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
+from mcp.server.fastmcp import FastMCP
+
+import json
+import os.path
+from dataclasses import dataclass
+from typing import Dict, List
+
+import pandas as pd
+
+from flowcept.flowceptor.consumers.agent.base_agent_context_manager import BaseAgentContextManager, BaseAppContext
+
+
+from flowcept.agents import agent_client
+from flowcept.commons.task_data_preprocess import summarize_task
+
+
+@dataclass
+class FlowceptAppContext(BaseAppContext):
+    """
+    Context object for holding flowcept-specific state (e.g., tasks data) during the agent's lifecycle.
+
+    Attributes
+    ----------
+    task_summaries : List[Dict]
+        List of summarized task dictionaries.
+    critical_tasks : List[Dict]
+        List of critical task summaries with tags or anomalies.
+    """
+
+    tasks: List[Dict] | None
+    task_summaries: List[Dict] | None
+    critical_tasks: List[Dict] | None
+    df: pd.DataFrame | None
+    tasks_schema: Dict | None  # TODO: we dont need to keep the tasks_schema in context, just in the manager's memory.
+    value_examples: Dict | None
+    tracker_config: Dict | None
+
+
+class FlowceptAgentContextManager(BaseAgentContextManager):
+    """
+    Manages agent context and operations for Flowcept's intelligent task monitoring.
+
+    This class extends BaseAgentContextManager and maintains a rolling buffer of task messages.
+    It summarizes and tags tasks, builds a QA index over them, and uses LLM tools to analyze
+    task batches periodically.
+
+    Attributes
+    ----------
+    context : FlowceptAppContext
+        Current application context holding task state and QA components.
+    msgs_counter : int
+        Counter tracking how many task messages have been processed.
+    context_size : int
+        Number of task messages to collect before triggering QA index building and LLM analysis.
+    qa_manager : FlowceptQAManager
+        Utility for constructing QA chains from task summaries.
+    """
+
+    def __init__(self):
+        self.context: FlowceptAppContext = None
+        self.tracker_config = dict(max_examples=3, max_str_len=50)
+        self.schema_tracker = DynamicSchemaTracker(**self.tracker_config)
+        self.msgs_counter = 0
+        self.context_size = 1
+        super().__init__()
+
+    def message_handler(self, msg_obj: Dict):
+        """
+        Handle an incoming message and update context accordingly.
+
+        Parameters
+        ----------
+        msg_obj : Dict
+            The incoming message object.
+
+        Returns
+        -------
+        bool
+            True if the message was handled successfully.
+        """
+        print("Received:", msg_obj)
+        msg_type = msg_obj.get("type", None)
+        if msg_type == "task":
+            task_msg = TaskObject.from_dict(msg_obj)
+            if task_msg.subtype == "llm_task" and task_msg.agent_id == self.agent_id:
+                self.logger.info(f"Going to ignore our own LLM messages: {task_msg}")
+                return True
+
+            self.msgs_counter += 1
+            self.logger.debug("Received task msg!")
+            self.context.tasks.append(msg_obj)
+
+            task_summary = summarize_task(msg_obj, logger=self.logger)
+            self.context.task_summaries.append(task_summary)
+            if len(task_summary.get("tags", [])):
+                self.context.critical_tasks.append(task_summary)
+
+            if self.msgs_counter > 0 and self.msgs_counter % self.context_size == 0:
+                self.logger.debug(
+                    f"Going to add to index! {(self.msgs_counter - self.context_size, self.msgs_counter)}"
+                )
+                try:
+                    self.update_schema_and_add_to_df(
+                        tasks=self.context.task_summaries[self.msgs_counter - self.context_size : self.msgs_counter]
+                    )
+                except Exception as e:
+                    self.logger.error(
+                        f"Could not add these tasks to buffer!\n"
+                        f"{self.context.task_summaries[self.msgs_counter - self.context_size : self.msgs_counter]}"
+                    )
+                    self.logger.exception(e)
+
+                # self.monitor_chunk()
+
+        return True
+
+    def update_schema_and_add_to_df(self, tasks: List[Dict]):
+        """Update the schema and add to the DataFrame in context."""
+        self.schema_tracker.update_with_tasks(tasks)
+        self.context.tasks_schema = self.schema_tracker.get_schema()
+        self.context.value_examples = self.schema_tracker.get_example_values()
+
+        _df = pd.json_normalize(tasks)
+        self.context.df = pd.concat([self.context.df, pd.DataFrame(_df)], ignore_index=True)
+
+    def monitor_chunk(self):
+        """
+        Perform LLM-based analysis on the current chunk of task messages and send the results.
+        """
+        self.logger.debug(f"Going to begin LLM job! {self.msgs_counter}")
+        result = agent_client.run_tool("analyze_task_chunk")
+        if len(result):
+            content = result[0].text
+            if content != "Error executing tool":
+                msg = {"type": "flowcept_agent", "info": "monitor", "content": content}
+                self._mq_dao.send_message(msg)
+                self.logger.debug(str(content))
+            else:
+                self.logger.error(content)
+
+    def reset_context(self):
+        """
+        Reset the agent's context to a clean state, initializing a new QA setup.
+        """
+        self.context = FlowceptAppContext(
+            tasks=[],
+            task_summaries=[],
+            critical_tasks=[],
+            df=pd.DataFrame(),
+            tasks_schema={},
+            value_examples={},
+            tracker_config=self.tracker_config,
+        )
+        DEBUG = True  # TODO debugging!
+        if DEBUG:
+            self.logger.warning("Running agent in DEBUG mode!")
+            df_path = "/tmp/current_agent_df.csv"
+            if os.path.exists(df_path):
+                self.logger.warning("Going to load df into context")
+                df = load_saved_df(df_path)
+                self.context.df = df
+            if os.path.exists("/tmp/current_tasks_schema.json"):
+                with open("/tmp/current_tasks_schema.json") as f:
+                    self.context.tasks_schema = json.load(f)
+            if os.path.exists("/tmp/value_examples.json"):
+                with open("/tmp/value_examples.json") as f:
+                    self.context.value_examples = json.load(f)
+
+
+# Exporting the ctx_manager and the mcp_flowcept
+ctx_manager = FlowceptAgentContextManager()
+mcp_flowcept = FastMCP("FlowceptAgent", require_session=False, lifespan=ctx_manager.lifespan, stateless_http=True)

flowcept/agents/gui/__init__.py

@@ -0,0 +1,5 @@
+"""Streamlit Subpackage."""
+
+PAGE_TITLE = "Flowcept Agent Chat"
+DEFAULT_AGENT_NAME = "FlowceptAgent"
+AI = "🤖"