fast-agent-mcp 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

fast_agent/__init__.py

@@ -27,8 +27,11 @@ from fast_agent.config import (
     XAISettings,
 )
 
+# Prompt helpers (safe - no heavy dependencies)
+from fast_agent.mcp.prompt import Prompt
+
 # Type definitions and enums (safe - no dependencies)
-from fast_agent.types import LlmStopReason, RequestParams
+from fast_agent.types import LlmStopReason, PromptMessageExtended, RequestParams
 
 
 def __getattr__(name: str):
@@ -91,6 +94,8 @@ def __getattr__(name: str):
 if TYPE_CHECKING:  # pragma: no cover - typing aid only
     # Provide a concrete import path for type checkers/IDEs
     from fast_agent.core.fastagent import FastAgent as FastAgent  # noqa: F401
+    from fast_agent.mcp.prompt import Prompt as Prompt  # noqa: F401
+    from fast_agent.types import PromptMessageExtended as PromptMessageExtended  # noqa: F401
 
 
 __all__ = [
@@ -127,6 +132,9 @@ __all__ = [
     # Type definitions and enums (eagerly loaded)
     "LlmStopReason",
     "RequestParams",
+    "PromptMessageExtended",
+    # Prompt helpers (eagerly loaded)
+    "Prompt",
     # Agents (lazy loaded)
     "LlmAgent",
     "LlmDecorator",

fast_agent/agents/agent_types.py

@@ -3,7 +3,7 @@ Type definitions for agents and agent configurations.
 """
 
 from dataclasses import dataclass, field
-from enum import Enum
+from enum import StrEnum, auto
 from typing import Dict, List, Optional
 
 from mcp.client.session import ElicitationFnT
@@ -12,18 +12,18 @@ from mcp.client.session import ElicitationFnT
 from fast_agent.types import RequestParams
 
 
-class AgentType(Enum):
+class AgentType(StrEnum):
     """Enumeration of supported agent types."""
 
-    LLM = "llm"  # simple llm delegator
-    BASIC = "agent"
-    CUSTOM = "custom"
-    ORCHESTRATOR = "orchestrator"
-    PARALLEL = "parallel"
-    EVALUATOR_OPTIMIZER = "evaluator_optimizer"
-    ROUTER = "router"
-    CHAIN = "chain"
-    ITERATIVE_PLANNER = "iterative_planner"
+    LLM = auto()
+    BASIC = auto()
+    CUSTOM = auto()
+    ORCHESTRATOR = auto()
+    PARALLEL = auto()
+    EVALUATOR_OPTIMIZER = auto()
+    ROUTER = auto()
+    CHAIN = auto()
+    ITERATIVE_PLANNER = auto()
 
 
 @dataclass

fast_agent/agents/llm_agent.py

@@ -45,7 +45,7 @@ class LlmAgent(LlmDecorator):
 
     This class provides LLM-specific functionality including UI display methods,
     tool call tracking, and chat interaction patterns while delegating core
-    LLM operations to the attached AugmentedLLMProtocol.
+    LLM operations to the attached FastAgentLLMProtocol.
     """
 
     def __init__(
@@ -81,56 +81,71 @@ class LlmAgent(LlmDecorator):
         """
 
         # Determine display content based on stop reason if not provided
-        if additional_message is None:
-            # Generate additional message based on stop reason
-            match message.stop_reason:
-                case LlmStopReason.END_TURN:
-                    # No additional message needed for normal end turn
-                    additional_message_text = None
-
-                case LlmStopReason.MAX_TOKENS:
-                    additional_message_text = Text(
+        additional_segments: List[Text] = []
+
+        # Generate additional message based on stop reason
+        match message.stop_reason:
+            case LlmStopReason.END_TURN:
+                pass
+
+            case LlmStopReason.MAX_TOKENS:
+                additional_segments.append(
+                    Text(
                         "\n\nMaximum output tokens reached - generation stopped.",
                         style="dim red italic",
                     )
+                )
 
-                case LlmStopReason.SAFETY:
-                    additional_message_text = Text(
-                        "\n\nContent filter activated - generation stopped.", style="dim red italic"
+            case LlmStopReason.SAFETY:
+                additional_segments.append(
+                    Text(
+                        "\n\nContent filter activated - generation stopped.",
+                        style="dim red italic",
                     )
+                )
 
-                case LlmStopReason.PAUSE:
-                    additional_message_text = Text(
-                        "\n\nLLM has requested a pause.", style="dim green italic"
-                    )
+            case LlmStopReason.PAUSE:
+                additional_segments.append(
+                    Text("\n\nLLM has requested a pause.", style="dim green italic")
+                )
 
-                case LlmStopReason.STOP_SEQUENCE:
-                    additional_message_text = Text(
-                        "\n\nStop Sequence activated - generation stopped.", style="dim red italic"
+            case LlmStopReason.STOP_SEQUENCE:
+                additional_segments.append(
+                    Text(
+                        "\n\nStop Sequence activated - generation stopped.",
+                        style="dim red italic",
                     )
+                )
 
-                case LlmStopReason.TOOL_USE:
-                    if None is message.last_text():
-                        additional_message_text = Text(
-                            "The assistant requested tool calls", style="dim green italic"
-                        )
-                    else:
-                        additional_message_text = None
+            case LlmStopReason.TOOL_USE:
+                if None is message.last_text():
+                    additional_segments.append(
+                        Text("The assistant requested tool calls", style="dim green italic")
+                    )
 
-                case _:
-                    if message.stop_reason:
-                        additional_message_text = Text(
+            case _:
+                if message.stop_reason:
+                    additional_segments.append(
+                        Text(
                             f"\n\nGeneration stopped for an unhandled reason ({message.stop_reason})",
                             style="dim red italic",
                         )
-                    else:
-                        additional_message_text = None
-        else:
-            # Use provided additional message
-            additional_message_text = (
-                additional_message if isinstance(additional_message, Text) else None
+                    )
+
+        if additional_message is not None:
+            additional_segments.append(
+                additional_message
+                if isinstance(additional_message, Text)
+                else Text(str(additional_message))
             )
 
+        additional_message_text = None
+        if additional_segments:
+            combined = Text()
+            for segment in additional_segments:
+                combined += segment
+            additional_message_text = combined
+
         message_text = message.last_text() or ""
 
         # Use provided name/model or fall back to defaults
@@ -182,9 +197,11 @@ class LlmAgent(LlmDecorator):
 
         # TODO -- we should merge the request parameters here with the LLM defaults?
         # TODO - manage error catch, recovery, pause
-        result = await super().generate_impl(messages, request_params, tools)
+        result, summary = await self._generate_with_summary(messages, request_params, tools)
+
+        summary_text = Text(f"\n\n{summary.message}", style="dim red italic") if summary else None
 
-        await self.show_assistant_message(result)
+        await self.show_assistant_message(result, additional_message=summary_text)
         return result
 
     async def structured_impl(
@@ -196,8 +213,11 @@ class LlmAgent(LlmDecorator):
         if "user" == messages[-1].role:
             self.show_user_message(message=messages[-1])
 
-        result, message = await super().structured_impl(messages, model, request_params)
-        await self.show_assistant_message(message=message)
+        (result, message), summary = await self._structured_with_summary(
+            messages, model, request_params
+        )
+        summary_text = Text(f"\n\n{summary.message}", style="dim red italic") if summary else None
+        await self.show_assistant_message(message=message, additional_message=summary_text)
         return result, message
 
     # async def show_prompt_loaded(

fast_agent/agents/llm_decorator.py

@@ -2,6 +2,9 @@
 Decorator for LlmAgent, normalizes PromptMessageExtended, allows easy extension of Agents
 """
 
+import json
+from collections import Counter, defaultdict
+from dataclasses import dataclass
 from typing import (
     TYPE_CHECKING,
     Dict,
@@ -21,35 +24,64 @@ if TYPE_CHECKING:
 from a2a.types import AgentCard
 from mcp import Tool
 from mcp.types import (
+    CallToolResult,
+    ContentBlock,
+    EmbeddedResource,
     GetPromptResult,
+    ImageContent,
     Prompt,
     PromptMessage,
     ReadResourceResult,
+    ResourceLink,
+    TextContent,
+    TextResourceContents,
 )
 from opentelemetry import trace
 from pydantic import BaseModel
 
 from fast_agent.agents.agent_types import AgentConfig, AgentType
+from fast_agent.constants import FAST_AGENT_ERROR_CHANNEL, FAST_AGENT_REMOVED_METADATA_CHANNEL
 from fast_agent.context import Context
-from fast_agent.core.logging.logger import get_logger
 from fast_agent.interfaces import (
     AgentProtocol,
     FastAgentLLMProtocol,
     LLMFactoryProtocol,
 )
+from fast_agent.llm.model_database import ModelDatabase
 from fast_agent.llm.provider_types import Provider
 from fast_agent.llm.usage_tracking import UsageAccumulator
-from fast_agent.mcp.helpers.content_helpers import normalize_to_extended_list
+from fast_agent.mcp.helpers.content_helpers import normalize_to_extended_list, text_content
+from fast_agent.mcp.mime_utils import is_text_mime_type
 from fast_agent.types import PromptMessageExtended, RequestParams
 
-logger = get_logger(__name__)
 # Define a TypeVar for models
 ModelT = TypeVar("ModelT", bound=BaseModel)
 
-# Define a TypeVar for AugmentedLLM and its subclasses
 LLM = TypeVar("LLM", bound=FastAgentLLMProtocol)
 
 
+@dataclass
+class _RemovedBlock:
+    """Internal representation of a removed content block."""
+
+    category: str
+    mime_type: str | None
+    source: str
+    tool_id: str | None
+    block: ContentBlock
+
+
+@dataclass(frozen=True)
+class RemovedContentSummary:
+    """Summary information about removed content for the last turn."""
+
+    model_name: str | None
+    counts: Dict[str, int]
+    category_mimes: Dict[str, Tuple[str, ...]]
+    alert_flags: frozenset[str]
+    message: str
+
+
 class LlmDecorator(AgentProtocol):
     """
     A pure delegation wrapper around LlmAgent instances.
@@ -236,8 +268,8 @@ class LlmDecorator(AgentProtocol):
         Returns:
             The LLM's response as a PromptMessageExtended
         """
-        assert self._llm, "LLM is not attached"
-        return await self._llm.generate(messages, request_params, tools)
+        response, _ = await self._generate_with_summary(messages, request_params, tools)
+        return response
 
     async def apply_prompt_template(self, prompt_result: GetPromptResult, prompt_name: str) -> str:
         """
@@ -338,8 +370,320 @@ class LlmDecorator(AgentProtocol):
         Returns:
             A tuple of (parsed model instance or None, assistant response message)
         """
+        result, _ = await self._structured_with_summary(messages, model, request_params)
+        return result
+
+    async def _generate_with_summary(
+        self,
+        messages: List[PromptMessageExtended],
+        request_params: RequestParams | None = None,
+        tools: List[Tool] | None = None,
+    ) -> Tuple[PromptMessageExtended, RemovedContentSummary | None]:
         assert self._llm, "LLM is not attached"
-        return await self._llm.structured(messages, model, request_params)
+        sanitized_messages, summary = self._sanitize_messages_for_llm(messages)
+        response = await self._llm.generate(sanitized_messages, request_params, tools)
+        return response, summary
+
+    async def _structured_with_summary(
+        self,
+        messages: List[PromptMessageExtended],
+        model: Type[ModelT],
+        request_params: RequestParams | None = None,
+    ) -> Tuple[Tuple[ModelT | None, PromptMessageExtended], RemovedContentSummary | None]:
+        assert self._llm, "LLM is not attached"
+        sanitized_messages, summary = self._sanitize_messages_for_llm(messages)
+        structured_result = await self._llm.structured(sanitized_messages, model, request_params)
+        return structured_result, summary
+
+    def _sanitize_messages_for_llm(
+        self, messages: List[PromptMessageExtended]
+    ) -> Tuple[List[PromptMessageExtended], RemovedContentSummary | None]:
+        """Filter out content blocks that the current model cannot tokenize."""
+        if not messages:
+            return [], None
+
+        removed_blocks: List[_RemovedBlock] = []
+        sanitized_messages: List[PromptMessageExtended] = []
+
+        for message in messages:
+            sanitized, removed = self._sanitize_message_for_llm(message)
+            sanitized_messages.append(sanitized)
+            removed_blocks.extend(removed)
+
+        summary = self._build_removed_summary(removed_blocks)
+        if summary:
+            # Attach metadata to the last user message for downstream UI usage
+            for msg in reversed(sanitized_messages):
+                if msg.role == "user":
+                    channels = dict(msg.channels or {})
+                    meta_entries = list(channels.get(FAST_AGENT_REMOVED_METADATA_CHANNEL, []))
+                    meta_entries.extend(self._build_metadata_entries(removed_blocks))
+                    channels[FAST_AGENT_REMOVED_METADATA_CHANNEL] = meta_entries
+                    msg.channels = channels
+                    break
+
+        return sanitized_messages, summary
+
+    def _sanitize_message_for_llm(
+        self, message: PromptMessageExtended
+    ) -> Tuple[PromptMessageExtended, List[_RemovedBlock]]:
+        """Return a sanitized copy of a message and any removed content blocks."""
+        msg_copy = message.model_copy(deep=True)
+        removed: List[_RemovedBlock] = []
+
+        msg_copy.content = self._filter_block_list(
+            list(msg_copy.content or []), removed, source="message"
+        )
+
+        if msg_copy.tool_results:
+            new_tool_results: Dict[str, CallToolResult] = {}
+            for tool_id, tool_result in msg_copy.tool_results.items():
+                original_blocks = list(tool_result.content or [])
+                filtered_blocks = self._filter_block_list(
+                    original_blocks,
+                    removed,
+                    source="tool_result",
+                    tool_id=tool_id,
+                )
+
+                if filtered_blocks != original_blocks:
+                    try:
+                        updated_result = tool_result.model_copy(update={"content": filtered_blocks})
+                    except AttributeError:
+                        updated_result = CallToolResult(
+                            content=filtered_blocks, isError=getattr(tool_result, "isError", False)
+                        )
+                else:
+                    updated_result = tool_result
+
+                new_tool_results[tool_id] = updated_result
+
+            msg_copy.tool_results = new_tool_results
+
+        if removed:
+            channels = dict(msg_copy.channels or {})
+            error_entries = list(channels.get(FAST_AGENT_ERROR_CHANNEL, []))
+            error_entries.extend(self._build_error_channel_entries(removed))
+            channels[FAST_AGENT_ERROR_CHANNEL] = error_entries
+            msg_copy.channels = channels
+
+        return msg_copy, removed
+
+    def _filter_block_list(
+        self,
+        blocks: Sequence[ContentBlock],
+        removed: List[_RemovedBlock],
+        *,
+        source: str,
+        tool_id: str | None = None,
+    ) -> List[ContentBlock]:
+        kept: List[ContentBlock] = []
+        for block in blocks or []:
+            mime_type, category = self._extract_block_metadata(block)
+            if self._block_supported(mime_type, category):
+                kept.append(block)
+            else:
+                removed.append(
+                    _RemovedBlock(
+                        category=category,
+                        mime_type=mime_type,
+                        source=source,
+                        tool_id=tool_id,
+                        block=block,
+                    )
+                )
+        return kept
+
+    def _block_supported(self, mime_type: str | None, category: str) -> bool:
+        """Determine if the current model can process a content block."""
+        if category == "text":
+            return True
+
+        model_name = self._llm.model_name if self._llm else None
+        if not model_name:
+            return False
+
+        if mime_type:
+            return ModelDatabase.supports_mime(model_name, mime_type)
+
+        if category == "vision":
+            return ModelDatabase.supports_any_mime(
+                model_name, ["image/jpeg", "image/png", "image/webp"]
+            )
+
+        if category == "document":
+            return ModelDatabase.supports_mime(model_name, "application/pdf")
+
+        return False
+
+    def _extract_block_metadata(self, block: ContentBlock) -> Tuple[str | None, str]:
+        """Infer the MIME type and high-level category for a content block."""
+        if isinstance(block, TextContent):
+            return "text/plain", "text"
+
+        if isinstance(block, TextResourceContents):
+            mime = getattr(block, "mimeType", None) or "text/plain"
+            return mime, "text"
+
+        if isinstance(block, ImageContent):
+            mime = getattr(block, "mimeType", None) or "image/*"
+            return mime, "vision"
+
+        if isinstance(block, EmbeddedResource):
+            resource = getattr(block, "resource", None)
+            mime = getattr(resource, "mimeType", None)
+            if isinstance(resource, TextResourceContents) or (mime and is_text_mime_type(mime)):
+                return mime or "text/plain", "text"
+            if mime and mime.startswith("image/"):
+                return mime, "vision"
+            return mime, "document"
+
+        if isinstance(block, ResourceLink):
+            mime = getattr(block, "mimeType", None)
+            if mime and mime.startswith("image/"):
+                return mime, "vision"
+            if mime and is_text_mime_type(mime):
+                return mime, "text"
+            return mime, "document"
+
+        return None, "document"
+
+    def _build_error_channel_entries(self, removed: List[_RemovedBlock]) -> List[ContentBlock]:
+        """Create informative entries for the error channel."""
+        entries: List[ContentBlock] = []
+        model_name = self._llm.model_name if self._llm else None
+        model_display = model_name or "current model"
+
+        for item in removed:
+            mime_display = item.mime_type or "unknown"
+            category_label = self._category_label(item.category)
+            if item.source == "message":
+                source_label = "user content"
+            elif item.tool_id:
+                source_label = f"tool result '{item.tool_id}'"
+            else:
+                source_label = "tool result"
+
+            message = (
+                f"Removed unsupported {category_label} {source_label} ({mime_display}) "
+                f"before sending to {model_display}."
+            )
+            entries.append(text_content(message))
+            entries.append(item.block)
+
+        return entries
+
+    def _build_metadata_entries(self, removed: List[_RemovedBlock]) -> List[ContentBlock]:
+        entries: List[ContentBlock] = []
+        for item in removed:
+            metadata_text = text_content(
+                json.dumps(
+                    {
+                        "type": "fast-agent-removed",
+                        "category": item.category,
+                        "mime_type": item.mime_type,
+                        "source": item.source,
+                        "tool_id": item.tool_id,
+                    }
+                )
+            )
+            entries.append(metadata_text)
+        return entries
+
+    def _build_removed_summary(self, removed: List[_RemovedBlock]) -> RemovedContentSummary | None:
+        if not removed:
+            return None
+
+        counts = Counter(item.category for item in removed)
+        category_mimes: Dict[str, Tuple[str, ...]] = {}
+        mime_accumulator: Dict[str, set[str]] = defaultdict(set)
+
+        for item in removed:
+            mime_accumulator[item.category].add(item.mime_type or "unknown")
+
+        for category, mimes in mime_accumulator.items():
+            category_mimes[category] = tuple(sorted(mimes))
+
+        alert_flags = frozenset(
+            flag
+            for category in counts
+            for flag in (self._category_to_flag(category),)
+            if flag is not None
+        )
+
+        model_name = self._llm.model_name if self._llm else None
+        model_display = model_name or "current model"
+
+        category_order = ["vision", "document", "other", "text"]
+        segments: List[str] = []
+        for category in category_order:
+            if category not in counts:
+                continue
+            count = counts[category]
+            mime_list = ", ".join(category_mimes.get(category, ()))
+            label = self._category_label(category)
+            plural = "s" if count != 1 else ""
+            if mime_list:
+                segments.append(f"{count} {label} block{plural} ({mime_list})")
+            else:
+                segments.append(f"{count} {label} block{plural}")
+
+        # Append any remaining categories not covered in the preferred order
+        for category, count in counts.items():
+            if category in category_order:
+                continue
+            mime_list = ", ".join(category_mimes.get(category, ()))
+            label = self._category_label(category)
+            plural = "s" if count != 1 else ""
+            if mime_list:
+                segments.append(f"{count} {label} block{plural} ({mime_list})")
+            else:
+                segments.append(f"{count} {label} block{plural}")
+
+        detail = "; ".join(segments) if segments else "unknown content"
+
+        capability_labels = []
+        for flag in alert_flags:
+            match flag:
+                case "V":
+                    capability_labels.append("vision")
+                case "D":
+                    capability_labels.append("document")
+                case "T":
+                    capability_labels.append("text")
+
+        capability_note = ""
+        if capability_labels:
+            unique_caps = ", ".join(sorted(set(capability_labels)))
+            capability_note = f" Missing capability: {unique_caps}."
+
+        message = (
+            f"Removed unsupported content before sending to {model_display}: {detail}."
+            f"{capability_note} Stored original content in '{FAST_AGENT_ERROR_CHANNEL}'."
+        )
+
+        return RemovedContentSummary(
+            model_name=model_name,
+            counts=dict(counts),
+            category_mimes=category_mimes,
+            alert_flags=alert_flags,
+            message=message,
+        )
+
+    @staticmethod
+    def _category_to_flag(category: str) -> str | None:
+        mapping = {"text": "T", "document": "D", "vision": "V"}
+        return mapping.get(category)
+
+    @staticmethod
+    def _category_label(category: str) -> str:
+        if category == "vision":
+            return "vision"
+        if category == "document":
+            return "document"
+        if category == "text":
+            return "text"
+        return "content"
 
     @property
     def message_history(self) -> List[PromptMessageExtended]: