meshagent-openai 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

meshagent/openai/tools/completions_adapter.py

@@ -4,7 +4,7 @@ from meshagent.api import WebSocketClientProtocol, RoomClient, RoomException
 from meshagent.tools.blob import Blob, BlobStorage
 from meshagent.tools import Toolkit, ToolContext, Tool
 from meshagent.api.messaging import Response, LinkResponse, FileResponse, JsonResponse, TextResponse, EmptyResponse
-from meshagent.agents.schema import prompt_schema
+from meshagent.api.schema_util import prompt_schema
 from meshagent.agents.adapter import ToolResponseAdapter, LLMAdapter
 from uuid import uuid4
 import json
@@ -198,7 +198,7 @@ class OpenAICompletionsToolResponseAdapter(ToolResponseAdapter):
         else:
             raise Exception("unexpected return type: {type}".format(type=type(response)))
 
-    async def append_messages(self, *, context: AgentChatContext, tool_call: Any, room: RoomClient, response: Response) -> list:
+    async def create_messages(self, *, context: AgentChatContext, tool_call: Any, room: RoomClient, response: Response) -> list:
 
         message = {
             "role" : "tool",
@@ -209,7 +209,7 @@ class OpenAICompletionsToolResponseAdapter(ToolResponseAdapter):
 
         room.developer.log_nowait(type="llm.message", data={ "context" : context.id,  "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "message" : message })
                             
-        context.messages.append(message) 
+        return [ message ]
         
 
 
@@ -330,16 +330,17 @@ class OpenAICompletionsAdapter(LLMAdapter):
                             )
                             tool_response = await tool_bundle.execute(context=tool_context, tool_call=tool_call)
                             logger.info(f"tool response {tool_response}")
-                            await tool_adapter.append_messages(context=context, tool_call=tool_call, room=room, response=tool_response)
+                            return await tool_adapter.create_messages(context=context, tool_call=tool_call, room=room, response=tool_response)
+                        
                         except Exception as e:
                             logger.error(f"unable to complete tool call {tool_call}", exc_info=e)
                             room.developer.log_nowait(type="llm.error", data={ "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "error" : f"{e}" })
                 
-                            return {
+                            return [{
                                 "role" : "tool",
                                 "content" : json.dumps({"error":f"unable to complete tool call: {e}"}),
                                 "tool_call_id" : tool_call.id,
-                            }
+                            }]
 
 
                     for tool_call in message.tool_calls:

meshagent/openai/tools/responses_adapter.py

@@ -3,8 +3,8 @@ from meshagent.agents.agent import Agent, AgentChatContext, AgentCallContext
 from meshagent.api import WebSocketClientProtocol, RoomClient, RoomException
 from meshagent.tools.blob import Blob, BlobStorage
 from meshagent.tools import Toolkit, ToolContext, Tool
-from meshagent.api.messaging import Response, LinkResponse, FileResponse, JsonResponse, TextResponse, EmptyResponse
-from meshagent.agents.schema import prompt_schema
+from meshagent.api.messaging import Response, LinkResponse, FileResponse, JsonResponse, TextResponse, EmptyResponse, RawOutputs
+from meshagent.api.schema_util import prompt_schema
 from meshagent.agents.adapter import ToolResponseAdapter, LLMAdapter
 from uuid import uuid4
 import json
@@ -12,14 +12,13 @@ from jsonschema import validate
 from typing import List, Dict
 
 from openai import AsyncOpenAI, APIStatusError, NOT_GIVEN
-from openai.types.chat import ChatCompletionMessageToolCall
-from openai.types.responses import ResponseFunctionToolCall
+from openai.types.responses import ResponseFunctionToolCall, ResponseStreamEvent
 
 from copy import deepcopy
 from abc import abstractmethod, ABC
 import os
 import jsonschema
-from typing import Optional, Any
+from typing import Optional, Any, Callable
 
 import logging
 import re
@@ -65,12 +64,13 @@ class ResponsesToolBundle:
         self._toolkits = toolkits
         self._executors = dict[str, Toolkit]()
         self._safe_names = {}
+        self._tools_by_name = {}
 
         open_ai_tools = []
         
         for toolkit in toolkits:                
             for v in toolkit.tools:
-
+          
                 k = v.name
 
                 name = safe_tool_name(k)
@@ -81,22 +81,29 @@ class ResponsesToolBundle:
                 self._executors[k] = toolkit
 
                 self._safe_names[name] = k
+                self._tools_by_name[name] = v
+
+                if v.name != "computer_call":
+
+                    fn = {
+                        "type" : "function",
+                        "name" : name,
+                        "description" : v.description,
+                        "parameters" : {
+                            **v.input_schema,
+                        },
+                        "strict": True,
+                    }
 
-                fn = {
-                    "type" : "function",
-                    "name" : name,
-                    "description" : v.description,
-                    "parameters" : {
-                        **v.input_schema,
-                    },
-                    "strict": True,
-                }
 
+                    if v.defs != None:
+                        fn["parameters"]["$defs"] = v.defs
+            
+                    open_ai_tools.append(fn)
 
-                if v.defs != None:
-                    fn["parameters"]["$defs"] = v.defs
-          
-                open_ai_tools.append(fn)
+                else:
+
+                    open_ai_tools.append(v.options)
 
         if len(open_ai_tools) == 0:
             open_ai_tools = None
@@ -127,7 +134,10 @@ class ResponsesToolBundle:
         except Exception as e:
             logger.error("failed calling %s %s", tool_call.id, name, exc_info=e)
             raise
-
+    
+    def get_tool(self, name: str) -> Tool | None:
+        return self._tools_by_name.get(name, None)
+    
     def contains(self, name: str) -> bool:
         return name in self._open_ai_tools
 
@@ -150,7 +160,8 @@ class OpenAIResponsesToolResponseAdapter(ToolResponseAdapter):
                 "url" : response.url,
             })
            
-        elif isinstance(response, JsonResponse):                                            
+        elif isinstance(response, JsonResponse):   
+                                                     
             return json.dumps(response.json)
         
         elif isinstance(response, TextResponse):
@@ -193,34 +204,45 @@ class OpenAIResponsesToolResponseAdapter(ToolResponseAdapter):
         else:
             raise Exception("unexpected return type: {type}".format(type=type(response)))
 
-    async def append_messages(self, *, context: AgentChatContext, tool_call: ResponseFunctionToolCall, room: RoomClient, response: Response) -> list:
+    async def create_messages(self, *, context: AgentChatContext, tool_call: ResponseFunctionToolCall, room: RoomClient, response: Response) -> list:
 
-        message = {
-            "output" : await self.to_plain_text(room=room, response=response),
-            "call_id" : tool_call.call_id,
-            "type" : "function_call_output"
-        }                                 
-      
+        if isinstance(response, RawOutputs):
+            
+            for output in response.outputs:
 
-        room.developer.log_nowait(type="llm.message", data={ "context" : context.id,  "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "message" : message })
-                            
-        context.messages.append(message) 
+                room.developer.log_nowait(type="llm.message", data={ "context" : context.id,  "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "message" : output })
+                                    
+            return response.outputs
+        else:
+            output = await self.to_plain_text(room=room, response=response)
+                
+            message = {
+                "output" : output,
+                "call_id" : tool_call.call_id,
+                "type" : "function_call_output"
+            }                                 
         
 
+            room.developer.log_nowait(type="llm.message", data={ "context" : context.id,  "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "message" : message })
+                                
+            return [ message ]
+            
+
 
 
-class OpenAIResponsesAdapter(LLMAdapter):
+class OpenAIResponsesAdapter(LLMAdapter[ResponsesToolBundle]):
     def __init__(self,      
         model: str = os.getenv("OPENAI_MODEL"),
         parallel_tool_calls : Optional[bool] = None,
         client: Optional[AsyncOpenAI] = None,
         reasoning_effort: Optional[str] = None,
+        retries : int = 10,
     ):
         self._model = model
         self._parallel_tool_calls = parallel_tool_calls
         self._client = client
         self._reasoning_effort = reasoning_effort
-
+        self._retries = retries
 
     def create_chat_context(self):
         system_role = "system"
@@ -228,6 +250,8 @@ class OpenAIResponsesAdapter(LLMAdapter):
             system_role = "developer"
         elif self._model.startswith("o3"):
             system_role = "developer"
+        elif self._model.startswith("computer-use"):
+            system_role = "developer"
 
         context = AgentChatContext(
             system_role=system_role
@@ -235,6 +259,40 @@ class OpenAIResponsesAdapter(LLMAdapter):
 
         return context
     
+    async def check_for_termination(self, *, context: AgentChatContext, room: RoomClient) -> bool:
+
+        if len(context.previous_messages) > 0:
+            last_message = context.previous_messages[-1]
+            logger.info(f"last_message {last_message}")
+
+        for message in context.messages:
+
+            if message.get("type", "message") != "message":
+                logger.info(f"found {message.get("type", "message")}")
+
+                return False
+
+        return True
+
+    def _get_client(self, *, room: RoomClient) -> AsyncOpenAI:
+        if self._client != None:
+            openai = self._client
+        else:
+            token : str = room.protocol.token
+            url : str = room.room_url
+            
+            room_proxy_url = f"{url}/v1"
+        
+            openai=AsyncOpenAI(
+                api_key=token,
+                base_url=room_proxy_url,
+                default_headers={
+                    "Meshagent-Session" : room.session_id
+                }
+            )
+       
+        return openai
+    
     # Takes the current chat context, executes a completion request and processes the response.
     # If a tool calls are requested, invokes the tools, processes the tool calls results, and appends the tool call results to the context
     async def next(self,
@@ -244,28 +302,14 @@ class OpenAIResponsesAdapter(LLMAdapter):
         toolkits: Toolkit,
         tool_adapter: Optional[ToolResponseAdapter] = None,
         output_schema: Optional[dict] = None,
+        event_handler: Optional[Callable[[ResponseStreamEvent],None]] = None
     ):  
         if tool_adapter == None:
             tool_adapter = OpenAIResponsesToolResponseAdapter()
             
         try:
-            if self._client != None:
-                openai = self._client
-            else:
-                
-               
-                token : str = room.protocol.token
-                url : str = room.room_url
-                
-                room_proxy_url = f"{url}/v1"
             
-                openai=AsyncOpenAI(
-                    api_key=token,
-                    base_url=room_proxy_url,
-                    default_headers={
-                        "Meshagent-Session" : room.session_id
-                    }
-                )
+            openai = self._get_client(room=room)
 
             tool_bundle = ResponsesToolBundle(toolkits=[
                 *toolkits,
@@ -276,17 +320,23 @@ class OpenAIResponsesAdapter(LLMAdapter):
                 logger.info("OpenAI Tools: %s", json.dumps(open_ai_tools))
             else:
                 logger.info("OpenAI Tools: Empty")
+                open_ai_tools = NOT_GIVEN
             
             response_schema = output_schema
             response_name = "response"
             
-        
-            while context.messages[-1].get("role") != "assistant" if context.messages else True:
+
+            while True:
+  
                 logger.info("model: %s, context: %s, output_schema: %s", self._model, context.messages, output_schema)
                 ptc = self._parallel_tool_calls
                 extra = {}
                 if ptc != None and self._model.startswith("o") == False:
                     extra["parallel_tool_calls"] = ptc 
+                
+                trunc = NOT_GIVEN
+                if self._model == "computer-use-preview":
+                    trunc = "auto"
 
                 text = NOT_GIVEN
                 if output_schema != None:
@@ -305,19 +355,36 @@ class OpenAIResponsesAdapter(LLMAdapter):
                         "effort" : self._reasoning_effort
                     }
 
-                response = await openai.responses.create(
-                    model = self._model,
-                    input = context.messages,
-                    tools = open_ai_tools,
-                    text = text,
-                    reasoning=reasoning,
-                )
+                previous_response_id = NOT_GIVEN
+                if context.previous_response_id != None:
+                    previous_response_id = context.previous_response_id
                 
-                room.developer.log_nowait(type="llm.message", data={ "context" : context.id, "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "response" : response.to_dict() })
+                stream = event_handler != None
                 
-                for message in response.output:
-                    context.messages.append(message)
+                for i in range(self._retries + 1):
+                    if range == self._retries:
+                        raise RoomException("exceeded maximum attempts calling openai")
+                    try:
+                        response = await openai.responses.create(
+                            stream=stream,
+                            model = self._model,
+                            input = context.messages,
+                            tools = open_ai_tools,
+                            text = text,
+                            previous_response_id=previous_response_id,
+                            reasoning=reasoning,
+                            truncation=trunc
+                        )
+                        break
+                    except Exception as e:
+                        logger.error(f"error calling openai attempt: {i+1}", exc_info=e)
+                        if i == self._retries:
+                            raise
                 
+
+                async def handle_message(message):
+                    
+
                     if message.type == "function_call":
                     
                         tasks = []
@@ -330,56 +397,139 @@ class OpenAIResponsesAdapter(LLMAdapter):
                                 )
                                 tool_response = await tool_bundle.execute(context=tool_context, tool_call=tool_call)
                                 logger.info(f"tool response {tool_response}")
-                                await tool_adapter.append_messages(context=context, tool_call=tool_call, room=room, response=tool_response)
+                                return await tool_adapter.create_messages(context=context, tool_call=tool_call, room=room, response=tool_response)
                             except Exception as e:
                                 logger.error(f"unable to complete tool call {tool_call}", exc_info=e)
                                 room.developer.log_nowait(type="llm.error", data={ "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "error" : f"{e}" })
                     
-                                return {
+                                return [{
                                     "output" : json.dumps({"error":f"unable to complete tool call: {e}"}),
                                     "call_id" : tool_call.call_id,
                                     "type" : "function_call_output"
-                                }
+                                }]
 
 
                         tasks.append(asyncio.create_task(do_tool_call(message)))
 
                         results = await asyncio.gather(*tasks)
 
+                        all_results = []
                         for result in results:
-                            if result != None:
-                                room.developer.log_nowait(type="llm.message", data={ "context" : context.id, "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "message" : result })
-                                context.messages.append(result)
-                        
-                    elif response.output_text != None:
+                            room.developer.log_nowait(type="llm.message", data={ "context" : context.id, "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "message" : result })
+                            all_results.extend(result)
+
+                        return all_results, False
 
-                        content = response.output_text
+                    elif message.type == "computer_call" and tool_bundle.get_tool("computer_call"):
+                        tool_context = ToolContext(
+                            room=room,
+                            caller=room.local_participant,
+                        )
+                        outputs = (await tool_bundle.get_tool("computer_call").execute(context=tool_context, arguments=message.to_dict(mode="json"))).outputs
+
+                        return outputs, False
+                    
+                    elif message.type == "reasoning":
+                        reasoning = tool_bundle.get_tool("reasoning_tool")
+                        if reasoning != None:
+                            await tool_bundle.get_tool("reasoning_tool").execute(context=tool_context, arguments=message.to_dict(mode="json"))
                         
-                        logger.info("RESPONSE FROM OPENAI %s", content)
-                        if response_schema == None:
-                            return content
+                    elif message.type == "message":
                         
-                        # First try to parse the result
-                        try:
-                            full_response = json.loads(content)
-                        # sometimes open ai packs two JSON chunks seperated by newline, check if that's why we couldn't parse
-                        except json.decoder.JSONDecodeError as e:
-                            for part in content.splitlines():
-                                if len(part.strip()) > 0:
-                                    full_response = json.loads(part)
+                        contents = message.content
+                        if response_schema == None:
+                            return [], False
+                        else:
+                            for content in contents:
+                                # First try to parse the result
+                                try:
+                                    full_response = json.loads(content.text)
+                                # sometimes open ai packs two JSON chunks seperated by newline, check if that's why we couldn't parse
+                                except json.decoder.JSONDecodeError as e:
+                                    for part in content.text.splitlines():
+                                        if len(part.strip()) > 0:
+                                            full_response = json.loads(part)
+                                            
+                                            try:
+                                                self.validate(response=full_response, output_schema=response_schema)
+                                            except Exception as e:
+                                                logger.error("recieved invalid response, retrying", exc_info=e)
+                                                error = { "role" : "user", "content" : "encountered a validation error with the output: {error}".format(error=e)}
+                                                room.developer.log_nowait(type="llm.message", data={ "context" : message.id, "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "message" : error })
+                                                context.messages.append(error)
+                                                continue
                                     
-                                    try:
-                                        self.validate(response=full_response, output_schema=response_schema)
-                                    except Exception as e:
-                                        logger.error("recieved invalid response, retrying", exc_info=e)
-                                        error = { "role" : "user", "content" : "encountered a validation error with the output: {error}".format(error=e)}
-                                        room.developer.log_nowait(type="llm.message", data={ "context" : context.id, "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "message" : error })
-                                        context.messages.append(error)
-                                        continue
-                            
-                        return full_response
+                                return [ full_response ], True
                     else:
                         raise RoomException("Unexpected response from OpenAI {response}".format(response=message))
+                    
+                    return [], False
+                    
+                if stream == False:
+                    room.developer.log_nowait(type="llm.message", data={ "context" : context.id, "participant_id" : room.local_participant.id, "participant_name" : room.local_participant.get_attribute("name"), "response" : response.to_dict() })
+                
+                    context.create_response(response.id)
+
+                    final_outputs = []
+                    
+                    for message in response.output:
+                        outputs, done = await handle_message(message=message)
+                        if done:
+                            final_outputs.extend(outputs)
+                        else:
+                            for output in outputs:
+                                context.messages.append(output)
+
+                    if len(final_outputs) > 0:
+
+                        return final_outputs[0]
+
+                else:
+                    
+                    final_outputs = []
+                    all_outputs = []
+                    async for e in response:
+                        
+                        event : ResponseStreamEvent = e
+
+                        event_handler(event)
+
+                        if event.type == "response.completed":
+                            context.create_response(event.response.id)
+                         
+                            context.messages.extend(all_outputs)
+
+                            term = await self.check_for_termination(context=context, room=room)
+                            if term:
+                                text = ""
+                                for output in event.response.output:
+                                    if output.type == "message":
+                                        for content in output.content:
+                                            text += content.text
+
+                                return text
+
+
+                            all_outputs = []
+
+                        elif event.type == "response.output_item.done":
+                          
+                            context.previous_messages.append(event.item.to_dict())
+                        
+                            outputs, done = await handle_message(message=event.item)
+                            if done:
+                                final_outputs.extend(outputs)
+                            else:
+                                for output in outputs:
+                                    all_outputs.append(output)
+
+                        if len(final_outputs) > 0:
+
+                            return final_outputs[0]
+
+                                        
+                    
+                        
         except APIStatusError as e:
             raise RoomException(f"Error from OpenAI: {e}")
             

{meshagent_openai-0.0.3.dist-info → meshagent_openai-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: meshagent-openai
-Version: 0.0.3
+Version: 0.0.4
 Summary: OpenAI Building Blocks for Meshagent
 Home-page: 
 License: Apache License 2.0
@@ -14,9 +14,9 @@ Requires-Dist: pyjwt>=2.0.0
 Requires-Dist: pytest>=8.3.4
 Requires-Dist: pytest-asyncio>=0.24.0
 Requires-Dist: openai>=1.66.2
-Requires-Dist: meshagent-api>=0.0.3
-Requires-Dist: meshagent-agents>=0.0.3
-Requires-Dist: meshagent-tools>=0.0.3
+Requires-Dist: meshagent-api>=0.0.4
+Requires-Dist: meshagent-agents>=0.0.4
+Requires-Dist: meshagent-tools>=0.0.4
 Dynamic: description-content-type
 Dynamic: license
 Dynamic: project-url

meshagent_openai-0.0.4.dist-info/RECORD

@@ -0,0 +1,9 @@
+meshagent/openai/__init__.py,sha256=qAqthmkQBsTqmrnVbtGOeaWCz9o5N48uEUCX7irzEsg,141
+meshagent/openai/tools/__init__.py,sha256=RBU_J4qRDuBaaUdi6jpgpuMlIbvT30QmTBrZrYLwsUU,185
+meshagent/openai/tools/completions_adapter.py,sha256=I74OC6JMbGb_C7qCYc0lWpJPs3uwuX9TfIDyn9Jou6U,15340
+meshagent/openai/tools/responses_adapter.py,sha256=WGUCHYXq_ZzvjjUOlD59G0lazRSntQqg6BgU7N18W5M,21766
+meshagent_openai-0.0.4.dist-info/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
+meshagent_openai-0.0.4.dist-info/METADATA,sha256=MVMIQmzR7Vi_wXmNDvFOyoh-JzzYQh-48YMce6tYB2A,752
+meshagent_openai-0.0.4.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+meshagent_openai-0.0.4.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
+meshagent_openai-0.0.4.dist-info/RECORD,,

meshagent_openai-0.0.3.dist-info/RECORD

@@ -1,9 +0,0 @@
-meshagent/openai/__init__.py,sha256=qAqthmkQBsTqmrnVbtGOeaWCz9o5N48uEUCX7irzEsg,141
-meshagent/openai/tools/__init__.py,sha256=RBU_J4qRDuBaaUdi6jpgpuMlIbvT30QmTBrZrYLwsUU,185
-meshagent/openai/tools/completions_adapter.py,sha256=sotgpBusJ-pkTOLWqvavokit3bysYBj4nNBat4PFnBw,15319
-meshagent/openai/tools/responses_adapter.py,sha256=ybvIMRpIrwDr3O1CW7m_Efma926IqFv5jPawZX-WYtQ,15609
-meshagent_openai-0.0.3.dist-info/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
-meshagent_openai-0.0.3.dist-info/METADATA,sha256=vgZd2DmRaQAu8cokGl9N6Jn6BFE88kqoNbHKHBC0rP4,752
-meshagent_openai-0.0.3.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-meshagent_openai-0.0.3.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
-meshagent_openai-0.0.3.dist-info/RECORD,,