fast-agent-mcp 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

mcp_agent/workflows/llm/augmented_llm_openai.py

@@ -1,37 +1,34 @@
-import json
 import os
-from typing import Iterable, List, Type
-from mcp.types import PromptMessage
-import instructor
+from typing import List, Type, TYPE_CHECKING
+
+from pydantic_core import from_json
+
+from mcp_agent.workflows.llm.providers.multipart_converter_openai import OpenAIConverter
+from mcp_agent.workflows.llm.providers.sampling_converter_openai import (
+    OpenAISamplingConverter,
+)
+
+if TYPE_CHECKING:
+    from mcp_agent.mcp.prompt_message_multipart import PromptMessageMultipart
 from openai import OpenAI, AuthenticationError
+
+# from openai.types.beta.chat import
 from openai.types.chat import (
-    ChatCompletionAssistantMessageParam,
-    ChatCompletionContentPartParam,
-    ChatCompletionContentPartTextParam,
-    ChatCompletionContentPartRefusalParam,
-    ChatCompletionMessage,
     ChatCompletionMessageParam,
+    ChatCompletionMessage,
     ChatCompletionSystemMessageParam,
     ChatCompletionToolParam,
-    ChatCompletionToolMessageParam,
     ChatCompletionUserMessageParam,
 )
 from mcp.types import (
     CallToolRequestParams,
     CallToolRequest,
     CallToolResult,
-    EmbeddedResource,
-    ImageContent,
-    TextContent,
-    TextResourceContents,
 )
 
 from mcp_agent.workflows.llm.augmented_llm import (
     AugmentedLLM,
     ModelT,
-    MCPMessageParam,
-    MCPMessageResult,
-    ProviderToMCPConverter,
     RequestParams,
 )
 from mcp_agent.core.exceptions import ProviderKeyError
@@ -56,7 +53,7 @@ class OpenAIAugmentedLLM(
     def __init__(self, *args, **kwargs):
         # Set type_converter before calling super().__init__
         if "type_converter" not in kwargs:
-            kwargs["type_converter"] = MCPOpenAITypeConverter
+            kwargs["type_converter"] = OpenAISamplingConverter
 
         super().__init__(*args, **kwargs)
 
@@ -128,7 +125,12 @@ class OpenAIAugmentedLLM(
             self.context.config.openai.base_url if self.context.config.openai else None
         )
 
-    async def generate(self, message, request_params: RequestParams | None = None):
+    async def generate(
+        self,
+        message,
+        request_params: RequestParams | None = None,
+        response_model: Type[ModelT] | None = None,
+    ) -> List[ChatCompletionMessage]:
         """
         Process a query using an LLM and available tools.
         The default implementation uses OpenAI's ChatCompletion as the LLM.
@@ -152,7 +154,7 @@ class OpenAIAugmentedLLM(
                 ChatCompletionSystemMessageParam(role="system", content=system_prompt)
             )
 
-        # Always include prompt messages, but only include conversation history 
+        # Always include prompt messages, but only include conversation history
         # if use_history is True
         messages.extend(self.history.get(include_history=params.use_history))
 
@@ -179,7 +181,7 @@ class OpenAIAugmentedLLM(
             for tool in response.tools
         ]
         if not available_tools:
-            available_tools = None
+            available_tools = []
 
         responses: List[ChatCompletionMessage] = []
         model = await self.select_model(params)
@@ -191,11 +193,11 @@ class OpenAIAugmentedLLM(
         else:
             self.show_user_message(str(message), model, chat_turn)
 
+        # we do NOT send stop sequences as this causes errors with mutlimodal processing
         for i in range(params.max_iterations):
             arguments = {
                 "model": model or "gpt-4o",
                 "messages": messages,
-                "stop": params.stopSequences,
                 "tools": available_tools,
             }
             if self._reasoning:
@@ -215,9 +217,16 @@ class OpenAIAugmentedLLM(
             self.logger.debug(f"{arguments}")
             self._log_chat_progress(chat_turn, model=model)
 
-            executor_result = await self.executor.execute(
-                openai_client.chat.completions.create, **arguments
-            )
+            if response_model is None:
+                executor_result = await self.executor.execute(
+                    openai_client.chat.completions.create, **arguments
+                )
+            else:
+                executor_result = await self.executor.execute(
+                    openai_client.beta.chat.completions.parse,
+                    **arguments,
+                    response_format=response_model,
+                )
 
             response = executor_result[0]
 
@@ -271,30 +280,34 @@ class OpenAIAugmentedLLM(
                         message.tool_calls[0].function.name,
                     )
 
-                # Execute all tool calls in parallel.
-                tool_tasks = []
+                tool_results = []
                 for tool_call in message.tool_calls:
                     self.show_tool_call(
                         available_tools,
                         tool_call.function.name,
                         tool_call.function.arguments,
                     )
-                    tool_tasks.append(self.execute_tool_call(tool_call))
-                # Wait for all tool calls to complete.
-                tool_results = await self.executor.execute(*tool_tasks)
+                    tool_call_request = CallToolRequest(
+                        method="tools/call",
+                        params=CallToolRequestParams(
+                            name=tool_call.function.name,
+                            arguments=from_json(
+                                tool_call.function.arguments, allow_partial=True
+                            ),
+                        ),
+                    )
+                    result = await self.call_tool(tool_call_request, tool_call.id)
+                    self.show_oai_tool_result(str(result))
+
+                    tool_results.append((tool_call.id, result))
+
+                messages.extend(
+                    OpenAIConverter.convert_function_results_to_openai(tool_results)
+                )
+
                 self.logger.debug(
                     f"Iteration {i}: Tool call results: {str(tool_results) if tool_results else 'None'}"
                 )
-                # Add non-None results to messages.
-                for result in tool_results:
-                    if isinstance(result, BaseException):
-                        self.logger.error(
-                            f"Warning: Unexpected error during tool execution: {result}. Continuing..."
-                        )
-                        continue
-                    if result is not None:
-                        self.show_oai_tool_result(str(result["content"]))
-                        messages.append(result)
             elif choice.finish_reason == "length":
                 # We have reached the max tokens limit
                 self.logger.debug(
@@ -334,10 +347,10 @@ class OpenAIAugmentedLLM(
         if params.use_history:
             # Get current prompt messages
             prompt_messages = self.history.get(include_history=False)
-            
+
             # Calculate new conversation messages (excluding prompts)
-            new_messages = messages[len(prompt_messages):]
-            
+            new_messages = messages[len(prompt_messages) :]
+
             # Update conversation history
             self.history.set(new_messages)
 
@@ -354,7 +367,15 @@ class OpenAIAugmentedLLM(
         Process a query using an LLM and available tools.
         The default implementation uses OpenAI's ChatCompletion as the LLM.
         Override this method to use a different LLM.
+
+        Special commands:
+        - "***SAVE_HISTORY <filename.md>" - Saves the conversation history to the specified file
+          in MCP prompt format with user/assistant delimiters.
         """
+        # Check if this is a special command to save history
+        if isinstance(message, str) and message.startswith("***SAVE_HISTORY "):
+            return await self._save_history_to_file(message)
+
         responses = await self.generate(
             message=message,
             request_params=request_params,
@@ -373,46 +394,128 @@ class OpenAIAugmentedLLM(
 
         return "\n".join(final_text)
 
+    async def _apply_prompt_template_provider_specific(
+        self, multipart_messages: List["PromptMessageMultipart"]
+    ) -> str:
+        """
+        OpenAI-specific implementation of apply_prompt_template that handles
+        multimodal content natively.
+
+        Args:
+            multipart_messages: List of PromptMessageMultipart objects parsed from the prompt template
+
+        Returns:
+            String representation of the assistant's response if generated,
+            or the last assistant message in the prompt
+        """
+
+        # TODO -- this is very similar to Anthropic (just the converter class changes).
+        # TODO -- potential refactor to base class, standardize Converter interface
+        # Check the last message role
+        last_message = multipart_messages[-1]
+
+        # Add all previous messages to history (or all messages if last is from assistant)
+        messages_to_add = (
+            multipart_messages[:-1]
+            if last_message.role == "user"
+            else multipart_messages
+        )
+        converted = []
+        for msg in messages_to_add:
+            converted.append(OpenAIConverter.convert_to_openai(msg))
+        self.history.extend(converted, is_prompt=True)
+
+        if last_message.role == "user":
+            # For user messages: Generate response to the last one
+            self.logger.debug(
+                "Last message in prompt is from user, generating assistant response"
+            )
+            message_param = OpenAIConverter.convert_to_openai(last_message)
+            return await self.generate_str(message_param)
+        else:
+            # For assistant messages: Return the last message content as text
+            self.logger.debug(
+                "Last message in prompt is from assistant, returning it directly"
+            )
+            return str(last_message)
+
+    async def _save_history_to_file(self, command: str) -> str:
+        """
+        Save the conversation history to a file in MCP prompt format.
+
+        Args:
+            command: The command string, expected format: "***SAVE_HISTORY <filename.md>"
+
+        Returns:
+            Success or error message
+        """
+        try:
+            # Extract the filename from the command
+            parts = command.split(" ", 1)
+            if len(parts) != 2 or not parts[1].strip():
+                return "Error: Invalid format. Expected '***SAVE_HISTORY <filename.md>'"
+
+            filename = parts[1].strip()
+
+            # Get all messages from history
+            messages = self.history.get(include_history=True)
+
+            # Import required utilities
+            from mcp_agent.workflows.llm.openai_utils import (
+                openai_message_param_to_prompt_message_multipart,
+            )
+            from mcp_agent.mcp.prompt_serialization import (
+                multipart_messages_to_delimited_format,
+            )
+
+            # Convert message params to PromptMessageMultipart objects
+            multipart_messages = []
+            for msg in messages:
+                # Skip system messages - PromptMessageMultipart only supports user and assistant roles
+                if isinstance(msg, dict) and msg.get("role") == "system":
+                    continue
+
+                # Convert the message to a multipart message
+                multipart_messages.append(
+                    openai_message_param_to_prompt_message_multipart(msg)
+                )
+
+            # Convert to delimited format
+            delimited_content = multipart_messages_to_delimited_format(
+                multipart_messages,
+                user_delimiter="---USER",
+                assistant_delimiter="---ASSISTANT",
+            )
+
+            # Write to file
+            with open(filename, "w", encoding="utf-8") as f:
+                f.write("\n\n".join(delimited_content))
+
+            self.logger.info(f"Saved conversation history to {filename}")
+            return f"Done. Saved conversation history to {filename}"
+
+        except Exception as e:
+            self.logger.error(f"Error saving history: {str(e)}")
+            return f"Error saving history: {str(e)}"
+
     async def generate_structured(
         self,
         message,
         response_model: Type[ModelT],
         request_params: RequestParams | None = None,
     ) -> ModelT:
-        # First we invoke the LLM to generate a string response
-        # We need to do this in a two-step process because Instructor doesn't
-        # know how to invoke MCP tools via call_tool, so we'll handle all the
-        # processing first and then pass the final response through Instructor
-        response = await self.generate_str(
+        responses = await self.generate(
             message=message,
             request_params=request_params,
-        )
-
-        # Next we pass the text through instructor to extract structured data
-        client = instructor.from_openai(
-            OpenAI(
-                api_key=self._api_key(),
-                base_url=self._base_url(),
-            ),
-            mode=instructor.Mode.TOOLS_STRICT,
-        )
-
-        params = self.get_request_params(request_params)
-        model = await self.select_model(params)
-
-        # Extract structured data from natural language
-        structured_response = client.chat.completions.create(
-            model=model,
             response_model=response_model,
-            messages=[
-                {"role": "user", "content": response},
-            ],
-        )
-        await self.show_assistant_message(
-            str(structured_response), title="ASSISTANT/STRUCTURED"
         )
+        return responses[0].parsed
 
-        return structured_response
+    async def generate_prompt(
+        self, prompt: "PromptMessageMultipart", request_params: RequestParams | None
+    ) -> str:
+        converted_prompt = OpenAIConverter.convert_to_openai(prompt)
+        return await self.generate_str(converted_prompt, request_params)
 
     async def pre_tool_call(self, tool_call_id: str | None, request: CallToolRequest):
         return request
@@ -422,47 +525,6 @@ class OpenAIAugmentedLLM(
     ):
         return result
 
-    async def execute_tool_call(
-        self,
-        tool_call: ChatCompletionToolParam,
-    ) -> ChatCompletionToolMessageParam | None:
-        """
-        Execute a single tool call and return the result message.
-        Returns None if there's no content to add to messages.
-        """
-        tool_name = tool_call.function.name
-        tool_args_str = tool_call.function.arguments
-        tool_call_id = tool_call.id
-        tool_args = {}
-
-        try:
-            if tool_args_str:
-                tool_args = json.loads(tool_args_str)
-        except json.JSONDecodeError as e:
-            return ChatCompletionToolMessageParam(
-                role="tool",
-                tool_call_id=tool_call_id,
-                content=f"Invalid JSON provided in tool call arguments for '{tool_name}'. Failed to load JSON: {str(e)}",
-            )
-
-        tool_call_request = CallToolRequest(
-            method="tools/call",
-            params=CallToolRequestParams(name=tool_name, arguments=tool_args),
-        )
-
-        result = await self.call_tool(
-            request=tool_call_request, tool_call_id=tool_call_id
-        )
-
-        if result.content:
-            return ChatCompletionToolMessageParam(
-                role="tool",
-                tool_call_id=tool_call_id,
-                content=[mcp_content_to_openai_content(c) for c in result.content],
-            )
-
-        return None
-
     def message_param_str(self, message: ChatCompletionMessageParam) -> str:
         """Convert an input message to a string representation."""
         if message.get("content"):
@@ -489,227 +551,3 @@ class OpenAIAugmentedLLM(
             return content
 
         return str(message)
-
-
-class MCPOpenAITypeConverter(
-    ProviderToMCPConverter[ChatCompletionMessageParam, ChatCompletionMessage]
-):
-    """
-    Convert between OpenAI and MCP types.
-    """
-
-    @classmethod
-    def from_mcp_message_result(cls, result: MCPMessageResult) -> ChatCompletionMessage:
-        # MCPMessageResult -> ChatCompletionMessage
-        if result.role != "assistant":
-            raise ValueError(
-                f"Expected role to be 'assistant' but got '{result.role}' instead."
-            )
-
-        return ChatCompletionMessage(
-            role="assistant",
-            content=result.content.text or str(result.context),
-            # Lossy conversion for the following fields:
-            # result.model
-            # result.stopReason
-        )
-
-    @classmethod
-    def to_mcp_message_result(cls, result: ChatCompletionMessage) -> MCPMessageResult:
-        # ChatCompletionMessage -> MCPMessageResult
-        return MCPMessageResult(
-            role=result.role,
-            content=TextContent(type="text", text=result.content),
-            model=None,
-            stopReason=None,
-            # extras for ChatCompletionMessage fields
-            **result.model_dump(exclude={"role", "content"}),
-        )
-
-    @classmethod
-    def from_mcp_message_param(
-        cls, param: MCPMessageParam
-    ) -> ChatCompletionMessageParam:
-        # MCPMessageParam -> ChatCompletionMessageParam
-        if param.role == "assistant":
-            extras = param.model_dump(exclude={"role", "content"})
-            return ChatCompletionAssistantMessageParam(
-                role="assistant",
-                content=mcp_content_to_openai_content(param.content),
-                **extras,
-            )
-        elif param.role == "user":
-            extras = param.model_dump(exclude={"role", "content"})
-            return ChatCompletionUserMessageParam(
-                role="user",
-                content=mcp_content_to_openai_content(param.content),
-                **extras,
-            )
-        else:
-            raise ValueError(
-                f"Unexpected role: {param.role}, MCP only supports 'assistant' and 'user'"
-            )
-
-    @classmethod
-    def to_mcp_message_param(cls, param: ChatCompletionMessageParam) -> MCPMessageParam:
-        # ChatCompletionMessage -> MCPMessageParam
-
-        contents = openai_content_to_mcp_content(param.content)
-
-        # TODO: saqadri - the mcp_content can have multiple elements
-        # while sampling message content has a single content element
-        # Right now we error out if there are > 1 elements in mcp_content
-        # We need to handle this case properly going forward
-        if len(contents) > 1:
-            raise NotImplementedError(
-                "Multiple content elements in a single message are not supported"
-            )
-        mcp_content: TextContent | ImageContent | EmbeddedResource = contents[0]
-
-        if param.role == "assistant":
-            return MCPMessageParam(
-                role="assistant",
-                content=mcp_content,
-                **typed_dict_extras(param, ["role", "content"]),
-            )
-        elif param.role == "user":
-            return MCPMessageParam(
-                role="user",
-                content=mcp_content,
-                **typed_dict_extras(param, ["role", "content"]),
-            )
-        elif param.role == "tool":
-            raise NotImplementedError(
-                "Tool messages are not supported in SamplingMessage yet"
-            )
-        elif param.role == "system":
-            raise NotImplementedError(
-                "System messages are not supported in SamplingMessage yet"
-            )
-        elif param.role == "developer":
-            raise NotImplementedError(
-                "Developer messages are not supported in SamplingMessage yet"
-            )
-        elif param.role == "function":
-            raise NotImplementedError(
-                "Function messages are not supported in SamplingMessage yet"
-            )
-        else:
-            raise ValueError(
-                f"Unexpected role: {param.role}, MCP only supports 'assistant', 'user', 'tool', 'system', 'developer', and 'function'"
-            )
-
-    @classmethod
-    def from_mcp_prompt_message(
-        cls, message: PromptMessage
-    ) -> ChatCompletionMessageParam:
-        """Convert an MCP PromptMessage to an OpenAI ChatCompletionMessageParam."""
-
-        # Extract content
-        content = None
-        if hasattr(message.content, "text"):
-            content = message.content.text
-        else:
-            content = str(message.content)
-
-        # Extract extras
-        extras = message.model_dump(exclude={"role", "content"})
-
-        if message.role == "user":
-            return ChatCompletionUserMessageParam(
-                role="user", content=content, **extras
-            )
-        elif message.role == "assistant":
-            return ChatCompletionAssistantMessageParam(
-                role="assistant", content=content, **extras
-            )
-        else:
-            # Fall back to user for any unrecognized role, including "system"
-            _logger.warning(
-                f"Unsupported role '{message.role}' in PromptMessage. Falling back to 'user' role."
-            )
-            return ChatCompletionUserMessageParam(
-                role="user", content=f"[{message.role.upper()}] {content}", **extras
-            )
-
-
-def mcp_content_to_openai_content(
-    content: TextContent | ImageContent | EmbeddedResource,
-) -> ChatCompletionContentPartTextParam:
-    if isinstance(content, list):
-        # Handle list of content items
-        return ChatCompletionContentPartTextParam(
-            type="text",
-            text="\n".join(mcp_content_to_openai_content(c) for c in content),
-        )
-
-    if isinstance(content, TextContent):
-        return ChatCompletionContentPartTextParam(type="text", text=content.text)
-    elif isinstance(content, ImageContent):
-        # Best effort to convert an image to text
-        return ChatCompletionContentPartTextParam(
-            type="text", text=f"{content.mimeType}:{content.data}"
-        )
-    elif isinstance(content, EmbeddedResource):
-        if isinstance(content.resource, TextResourceContents):
-            return ChatCompletionContentPartTextParam(
-                type="text", text=content.resource.text
-            )
-        else:  # BlobResourceContents
-            return ChatCompletionContentPartTextParam(
-                type="text", text=f"{content.resource.mimeType}:{content.resource.blob}"
-            )
-    else:
-        # Last effort to convert the content to a string
-        return ChatCompletionContentPartTextParam(type="text", text=str(content))
-
-
-def openai_content_to_mcp_content(
-    content: str
-    | Iterable[ChatCompletionContentPartParam | ChatCompletionContentPartRefusalParam],
-) -> Iterable[TextContent | ImageContent | EmbeddedResource]:
-    mcp_content = []
-
-    if isinstance(content, str):
-        mcp_content = [TextContent(type="text", text=content)]
-    else:
-        # TODO: saqadri - this is a best effort conversion, we should handle all possible content types
-        for c in content:
-            if c.type == "text":  # isinstance(c, ChatCompletionContentPartTextParam):
-                mcp_content.append(
-                    TextContent(
-                        type="text", text=c.text, **typed_dict_extras(c, ["text"])
-                    )
-                )
-            elif (
-                c.type == "image_url"
-            ):  # isinstance(c, ChatCompletionContentPartImageParam):
-                raise NotImplementedError("Image content conversion not implemented")
-                # TODO: saqadri - need to download the image into a base64-encoded string
-                # Download image from c.image_url
-                # return ImageContent(
-                #     type="image",
-                #     data=downloaded_image,
-                #     **c
-                # )
-            elif (
-                c.type == "input_audio"
-            ):  # isinstance(c, ChatCompletionContentPartInputAudioParam):
-                raise NotImplementedError("Audio content conversion not implemented")
-            elif (
-                c.type == "refusal"
-            ):  # isinstance(c, ChatCompletionContentPartRefusalParam):
-                mcp_content.append(
-                    TextContent(
-                        type="text", text=c.refusal, **typed_dict_extras(c, ["refusal"])
-                    )
-                )
-            else:
-                raise ValueError(f"Unexpected content type: {c.type}")
-
-    return mcp_content
-
-
-def typed_dict_extras(d: dict, exclude: List[str]):
-    extras = {k: v for k, v in d.items() if k not in exclude}
-    return extras