fast-agent-mcp 0.1.11__py3-none-any.whl → 0.1.12__py3-none-any.whl

mcp_agent/workflows/llm/providers/multipart_converter_openai.py

@@ -1,10 +1,11 @@
-from typing import List, Union, Optional, Dict, Any, Tuple
+from typing import List, Union, Optional, Tuple, Dict, Any
 
 from mcp.types import (
     TextContent,
     ImageContent,
     EmbeddedResource,
     CallToolResult,
+    PromptMessage,
 )
 from mcp_agent.mcp.prompt_message_multipart import PromptMessageMultipart
 from mcp_agent.mcp.mime_utils import (
@@ -14,18 +15,36 @@ from mcp_agent.mcp.mime_utils import (
 )
 from mcp_agent.mcp.resource_utils import extract_title_from_uri
 
+
 from mcp_agent.logging.logger import get_logger
 
 _logger = get_logger("multipart_converter_openai")
 
-# Define the types for OpenAI API
-OpenAIContentBlock = Dict[str, Any]
+# Define type aliases for content blocks
+ContentBlock = Dict[str, Any]
 OpenAIMessage = Dict[str, Any]
 
 
 class OpenAIConverter:
     """Converts MCP message types to OpenAI API format."""
 
+    @staticmethod
+    def _is_supported_image_type(mime_type: str) -> bool:
+        """
+        Check if the given MIME type is supported by OpenAI's image API.
+
+        Args:
+            mime_type: The MIME type to check
+
+        Returns:
+            True if the MIME type is generally supported, False otherwise
+        """
+        return (
+            mime_type is not None
+            and is_image_mime_type(mime_type)
+            and mime_type != "image/svg+xml"
+        )
+
     @staticmethod
     def convert_to_openai(
         multipart_msg: PromptMessageMultipart, concatenate_text_blocks: bool = False
@@ -57,13 +76,23 @@ class OpenAIConverter:
 
             return {"role": role, "content": content_text}
 
+        # System messages also only support string content
+        if role == "system":
+            # Extract text from all text content blocks
+            content_text = ""
+            for item in multipart_msg.content:
+                if isinstance(item, TextContent):
+                    content_text += item.text
+
+            return {"role": role, "content": content_text}
+
         # For user messages, convert each content block
-        content_blocks = []
+        content_blocks: List[ContentBlock] = []
 
         for item in multipart_msg.content:
             try:
                 if isinstance(item, TextContent):
-                    content_blocks.append(OpenAIConverter._convert_text_content(item))
+                    content_blocks.append({"type": "text", "text": item.text})
 
                 elif isinstance(item, ImageContent):
                     content_blocks.append(OpenAIConverter._convert_image_content(item))
@@ -75,23 +104,11 @@ class OpenAIConverter:
 
                 # Handle input_audio if implemented
                 elif hasattr(item, "type") and getattr(item, "type") == "input_audio":
-                    # This assumes an InputAudioContent class structure with input_audio attribute
-                    if hasattr(item, "input_audio"):
-                        content_blocks.append(
-                            {
-                                "type": "input_audio",
-                                "input_audio": {
-                                    "data": item.input_audio.get("data", ""),
-                                    "format": item.input_audio.get("format", "wav"),
-                                },
-                            }
-                        )
-                    else:
-                        _logger.warning(
-                            "InputAudio content missing input_audio attribute"
-                        )
-                        fallback_text = "[Audio content missing data]"
-                        content_blocks.append({"type": "text", "text": fallback_text})
+                    _logger.warning(
+                        "Input audio content not supported in standard OpenAI types"
+                    )
+                    fallback_text = "[Audio content not directly supported]"
+                    content_blocks.append({"type": "text", "text": fallback_text})
 
                 else:
                     _logger.warning(f"Unsupported content type: {type(item)}")
@@ -119,46 +136,76 @@ class OpenAIConverter:
 
         # If concatenate_text_blocks is True, combine adjacent text blocks
         if concatenate_text_blocks:
-            combined_blocks = []
-            current_text = ""
-
-            for block in content_blocks:
-                if block["type"] == "text":
-                    # Add to current text accumulator
-                    if current_text:
-                        current_text += " " + block["text"]
-                    else:
-                        current_text = block["text"]
-                else:
-                    # Non-text block found, flush accumulated text if any
-                    if current_text:
-                        combined_blocks.append({"type": "text", "text": current_text})
-                        current_text = ""
-                    # Add the non-text block
-                    combined_blocks.append(block)
+            content_blocks = OpenAIConverter._concatenate_text_blocks(content_blocks)
+
+        # Return user message with content blocks
+        return {"role": role, "content": content_blocks}
 
-            # Don't forget any remaining text
-            if current_text:
-                combined_blocks.append({"type": "text", "text": current_text})
+    @staticmethod
+    def _concatenate_text_blocks(blocks: List[ContentBlock]) -> List[ContentBlock]:
+        """
+        Combine adjacent text blocks into single blocks.
 
-            content_blocks = combined_blocks
+        Args:
+            blocks: List of content blocks
 
-        return {"role": role, "content": content_blocks}
+        Returns:
+            List with adjacent text blocks combined
+        """
+        if not blocks:
+            return []
+
+        combined_blocks: List[ContentBlock] = []
+        current_text = ""
+
+        for block in blocks:
+            if block["type"] == "text":
+                # Add to current text accumulator
+                if current_text:
+                    current_text += " " + block["text"]
+                else:
+                    current_text = block["text"]
+            else:
+                # Non-text block found, flush accumulated text if any
+                if current_text:
+                    combined_blocks.append({"type": "text", "text": current_text})
+                    current_text = ""
+                # Add the non-text block
+                combined_blocks.append(block)
+
+        # Don't forget any remaining text
+        if current_text:
+            combined_blocks.append({"type": "text", "text": current_text})
+
+        return combined_blocks
 
     @staticmethod
-    def _convert_text_content(content: TextContent) -> OpenAIContentBlock:
-        """Convert TextContent to OpenAI text content block."""
-        return {"type": "text", "text": content.text}
+    def convert_prompt_message_to_openai(
+        message: PromptMessage, concatenate_text_blocks: bool = False
+    ) -> OpenAIMessage:
+        """
+        Convert a standard PromptMessage to OpenAI API format.
+
+        Args:
+            message: The PromptMessage to convert
+            concatenate_text_blocks: If True, adjacent text blocks will be combined
+
+        Returns:
+            An OpenAI API message object
+        """
+        # Convert the PromptMessage to a PromptMessageMultipart containing a single content item
+        multipart = PromptMessageMultipart(role=message.role, content=[message.content])
+
+        # Use the existing conversion method with the specified concatenation option
+        return OpenAIConverter.convert_to_openai(multipart, concatenate_text_blocks)
 
     @staticmethod
-    def _convert_image_content(content: ImageContent) -> OpenAIContentBlock:
+    def _convert_image_content(content: ImageContent) -> ContentBlock:
         """Convert ImageContent to OpenAI image_url content block."""
         # OpenAI requires image URLs or data URIs for images
         image_url = {"url": f"data:{content.mimeType};base64,{content.data}"}
 
         # Check if the image has annotations for detail level
-        # This would depend on your ImageContent implementation
-        # If annotations are available, use them for the detail parameter
         if hasattr(content, "annotations") and content.annotations:
             if hasattr(content.annotations, "detail"):
                 detail = content.annotations.detail
@@ -167,52 +214,72 @@ class OpenAIConverter:
 
         return {"type": "image_url", "image_url": image_url}
 
+    @staticmethod
+    def _determine_mime_type(resource_content) -> str:
+        """
+        Determine the MIME type of a resource.
+
+        Args:
+            resource_content: The resource content to check
+
+        Returns:
+            The determined MIME type as a string
+        """
+        if hasattr(resource_content, "mimeType") and resource_content.mimeType:
+            return resource_content.mimeType
+
+        if hasattr(resource_content, "uri") and resource_content.uri:
+            mime_type = guess_mime_type(str(resource_content.uri))
+            return mime_type
+
+        if hasattr(resource_content, "blob"):
+            return "application/octet-stream"
+
+        return "text/plain"
+
     @staticmethod
     def _convert_embedded_resource(
         resource: EmbeddedResource,
-    ) -> Optional[OpenAIContentBlock]:
-        """Convert EmbeddedResource to appropriate OpenAI content block."""
-        resource_content = resource.resource
-        uri = resource_content.uri
+    ) -> Optional[ContentBlock]:
+        """
+        Convert EmbeddedResource to appropriate OpenAI content block.
 
-        # Use mime_utils to guess MIME type if not provided
-        if resource_content.mimeType is None and uri:
-            mime_type = guess_mime_type(str(uri))
-            _logger.info(f"MIME type not provided, guessed {mime_type} for {uri}")
-        else:
-            mime_type = resource_content.mimeType or "application/octet-stream"
+        Args:
+            resource: The embedded resource to convert
 
-        is_url: bool = str(uri).startswith(("http://", "https://"))
+        Returns:
+            An appropriate OpenAI content block or None if conversion failed
+        """
+        resource_content = resource.resource
+        uri = getattr(resource_content, "uri", None)
+        is_url = uri and str(uri).startswith(("http://", "https://"))
         title = extract_title_from_uri(uri) if uri else "resource"
+        mime_type = OpenAIConverter._determine_mime_type(resource_content)
 
-        # Handle image resources
-        if is_image_mime_type(mime_type) and mime_type != "image/svg+xml":
-            image_url = {}
+        # Handle different resource types based on MIME type
 
+        # Handle images
+        if OpenAIConverter._is_supported_image_type(mime_type):
             if is_url:
-                image_url["url"] = str(uri)
+                return {"type": "image_url", "image_url": {"url": str(uri)}}
             elif hasattr(resource_content, "blob"):
-                image_url["url"] = f"data:{mime_type};base64,{resource_content.blob}"
+                return {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:{mime_type};base64,{resource_content.blob}"
+                    },
+                }
             else:
-                _logger.warning(f"Image resource missing both URL and blob data: {uri}")
                 return {"type": "text", "text": f"[Image missing data: {title}]"}
 
-            # Check for detail level in annotations if available
-            if hasattr(resource, "annotations") and resource.annotations:
-                if hasattr(resource.annotations, "detail"):
-                    detail = resource.annotations.detail
-                    if detail in ("auto", "low", "high"):
-                        image_url["detail"] = detail
-
-            return {"type": "image_url", "image_url": image_url}
-
-        # Handle PDF resources - OpenAI has specific file format for PDFs
+        # Handle PDFs
         elif mime_type == "application/pdf":
             if is_url:
-                # OpenAI doesn't directly support PDF URLs, only file_id or base64
-                _logger.warning(f"PDF URL not directly supported in OpenAI API: {uri}")
-                fallback_text = f"[PDF URL: {uri}]\nOpenAI requires PDF files to be uploaded or provided as base64 data."
-                return {"type": "text", "text": fallback_text}
+                # OpenAI doesn't directly support PDF URLs, explain this limitation
+                return {
+                    "type": "text",
+                    "text": f"[PDF URL: {uri}]\nOpenAI requires PDF files to be uploaded or provided as base64 data.",
+                }
             elif hasattr(resource_content, "blob"):
                 return {
                     "type": "file",
@@ -222,65 +289,82 @@ class OpenAIConverter:
                     },
                 }
 
-        # Handle SVG as text with fastagent:file tags
-        elif mime_type == "image/svg+xml":
-            if hasattr(resource_content, "text"):
-                file_text = (
-                    f'<fastagent:file title="{title}" mimetype="{mime_type}">\n'
-                    f"{resource_content.text}\n"
-                    f"</fastagent:file>"
-                )
-                return {"type": "text", "text": file_text}
-
-        # Handle text resources with fastagent:file tags
-        elif is_text_mime_type(mime_type):
-            if hasattr(resource_content, "text"):
-                # Wrap in fastagent:file tags for text resources
-                file_text = (
-                    f'<fastagent:file title="{title}" mimetype="{mime_type}">\n'
-                    f"{resource_content.text}\n"
-                    f"</fastagent:file>"
-                )
-                return {"type": "text", "text": file_text}
-
-        # Handle other binary formats that OpenAI supports with file type
-        # Currently, OpenAI supports PDFs for comprehensive viewing, but we can try
-        # to use the file type for other binary formats as well for future compatibility
-        elif hasattr(resource_content, "blob"):
-            # For now, we'll use file type for PDFs only, and use fallback for others
-            if mime_type == "application/pdf":
-                return {
-                    "type": "file",
-                    "file": {"file_name": title, "file_data": resource_content.blob},
-                }
-            else:
-                # For other binary formats, create a text message mentioning the resource
-                return {
-                    "type": "text",
-                    "text": f"[Binary resource: {title} ({mime_type})]",
-                }
+        # Handle SVG (convert to text)
+        elif mime_type == "image/svg+xml" and hasattr(resource_content, "text"):
+            file_text = (
+                f'<fastagent:file title="{title}" mimetype="{mime_type}">\n'
+                f"{resource_content.text}\n"
+                f"</fastagent:file>"
+            )
+            return {"type": "text", "text": file_text}
+
+        # Handle text files
+        elif is_text_mime_type(mime_type) and hasattr(resource_content, "text"):
+            file_text = (
+                f'<fastagent:file title="{title}" mimetype="{mime_type}">\n'
+                f"{resource_content.text}\n"
+                f"</fastagent:file>"
+            )
+            return {"type": "text", "text": file_text}
 
-        # Default fallback - convert to text if possible
-        if hasattr(resource_content, "text"):
-            # For anything with text content that isn't handled specially above,
-            # use the raw text without special formatting
+        # Default fallback for text resources
+        elif hasattr(resource_content, "text"):
             return {"type": "text", "text": resource_content.text}
 
-        _logger.warning(f"Unable to convert resource with MIME type: {mime_type}")
+        # Default fallback for binary resources
+        elif hasattr(resource_content, "blob"):
+            return {
+                "type": "text",
+                "text": f"[Binary resource: {title} ({mime_type})]",
+            }
+
+        # Last resort fallback
         return {
             "type": "text",
             "text": f"[Unsupported resource: {title} ({mime_type})]",
         }
 
+    @staticmethod
+    def _extract_text_from_content_blocks(
+        content: Union[str, List[ContentBlock]],
+    ) -> str:
+        """
+        Extract and combine text from content blocks.
+
+        Args:
+            content: Content blocks or string
+
+        Returns:
+            Combined text as a string
+        """
+        if isinstance(content, str):
+            return content
+
+        if not content:
+            return ""
+
+        # Extract only text blocks
+        text_parts = []
+        for block in content:
+            if block.get("type") == "text":
+                text_parts.append(block.get("text", ""))
+
+        return (
+            " ".join(text_parts)
+            if text_parts
+            else "[Complex content converted to text]"
+        )
+
     @staticmethod
     def convert_tool_result_to_openai(
         tool_result: CallToolResult,
         tool_call_id: str,
         concatenate_text_blocks: bool = False,
-    ) -> Union[OpenAIMessage, Tuple[OpenAIMessage, List[OpenAIMessage]]]:
+    ) -> Union[Dict[str, Any], Tuple[Dict[str, Any], List[Dict[str, Any]]]]:
         """
         Convert a CallToolResult to an OpenAI tool message.
-        If the result contains non-text elements, those are converted to separate messages
+
+        If the result contains non-text elements, those are converted to separate user messages
         since OpenAI tool messages can only contain text.
 
         Args:
@@ -300,7 +384,7 @@ class OpenAIConverter:
                 "content": "[No content in tool result]",
             }
 
-        # First, separate text and non-text content
+        # Separate text and non-text content
         text_content = []
         non_text_content = []
 
@@ -310,57 +394,19 @@ class OpenAIConverter:
             else:
                 non_text_content.append(item)
 
-        # If we only have text content, process as before
-        if not non_text_content:
-            # Create a temporary PromptMessageMultipart to reuse the conversion logic
-            temp_multipart = PromptMessageMultipart(role="user", content=text_content)
-
-            # Convert using the same logic as user messages
-            converted = OpenAIConverter.convert_to_openai(
-                temp_multipart, concatenate_text_blocks=concatenate_text_blocks
-            )
-
-            # For tool messages, we need to extract and combine all text content
-            if isinstance(converted["content"], str):
-                content = converted["content"]
-            else:
-                # For compatibility with OpenAI's tool message format, combine all text blocks
-                all_text = all(
-                    block.get("type") == "text" for block in converted["content"]
-                )
-
-                if all_text and len(converted["content"]) > 0:
-                    # Combine all text blocks
-                    content = " ".join(
-                        block.get("text", "") for block in converted["content"]
-                    )
-                else:
-                    # Fallback for unexpected cases
-                    content = "[Complex content converted to text]"
-
-            # Create a tool message with the converted content
-            return {"role": "tool", "tool_call_id": tool_call_id, "content": content}
-
-        # If we have mixed content or only non-text content
-
-        # Process text content for the tool message
+        # Create tool message with text content
         tool_message_content = ""
         if text_content:
+            # Convert text content to OpenAI format
             temp_multipart = PromptMessageMultipart(role="user", content=text_content)
             converted = OpenAIConverter.convert_to_openai(
-                temp_multipart, concatenate_text_blocks=True
+                temp_multipart, concatenate_text_blocks=concatenate_text_blocks
             )
 
-            if isinstance(converted["content"], str):
-                tool_message_content = converted["content"]
-            else:
-                # Combine all text blocks
-                all_text = [
-                    block.get("text", "")
-                    for block in converted["content"]
-                    if block.get("type") == "text"
-                ]
-                tool_message_content = " ".join(all_text)
+            # Extract text from content blocks
+            tool_message_content = OpenAIConverter._extract_text_from_content_blocks(
+                converted.get("content", "")
+            )
 
         if not tool_message_content:
             tool_message_content = "[Tool returned non-text content]"
@@ -372,31 +418,30 @@ class OpenAIConverter:
             "content": tool_message_content,
         }
 
+        # If there's no non-text content, return just the tool message
+        if not non_text_content:
+            return tool_message
+
         # Process non-text content as a separate user message
-        if non_text_content:
-            # Create a multipart message with the non-text content
-            non_text_multipart = PromptMessageMultipart(
-                role="user", content=non_text_content
-            )
+        non_text_multipart = PromptMessageMultipart(
+            role="user", content=non_text_content
+        )
 
-            # Convert to OpenAI format
-            user_message = OpenAIConverter.convert_to_openai(non_text_multipart)
-            # Add tool_call_id to associate with the tool call
-            user_message["tool_call_id"] = tool_call_id
+        # Convert to OpenAI format
+        user_message = OpenAIConverter.convert_to_openai(non_text_multipart)
 
-            return (tool_message, [user_message])
+        # We need to add tool_call_id manually
+        user_message["tool_call_id"] = tool_call_id
 
-        return tool_message
+        return (tool_message, [user_message])
 
     @staticmethod
     def convert_function_results_to_openai(
         results: List[Tuple[str, CallToolResult]],
         concatenate_text_blocks: bool = False,
-    ) -> List[OpenAIMessage]:
+    ) -> List[Dict[str, Any]]:
         """
         Convert a list of function call results to OpenAI messages.
-        Handles cases where tool results contain non-text content by creating
-        additional user messages as needed.
 
         Args:
             results: List of (tool_call_id, result) tuples