meshagent-openai 0.0.37__tar.gz → 0.0.39__tar.gz

{meshagent_openai-0.0.37 → meshagent_openai-0.0.39}/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [0.0.39]
+- Stability
+
+## [0.0.38]
+- Stability
+
 ## [0.0.37]
 - Stability
 

meshagent_openai-0.0.39/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: meshagent-openai
+Version: 0.0.39
+Summary: OpenAI Building Blocks for Meshagent
+License-Expression: Apache-2.0
+Project-URL: Documentation, https://docs.meshagent.com
+Project-URL: Website, https://www.meshagent.com
+Project-URL: Source, https://www.meshagent.com
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyjwt~=2.10
+Requires-Dist: pytest~=8.4
+Requires-Dist: pytest-asyncio~=0.26
+Requires-Dist: openai~=1.84
+Requires-Dist: meshagent-api~=0.0.39
+Requires-Dist: meshagent-agents~=0.0.39
+Requires-Dist: meshagent-tools~=0.0.39
+Dynamic: license-file
+
+## MeshAgent OpenAI
+
+## MeshAgent OpenAI
+The ``meshagent.openai`` package provides adapters to integrate OpenAI models with MeshAgent tools and agents. 
+
+### Completions Adapter and Responses Adapter
+MeshAgent supports both the OpenAI Chat Completions API and Responses API. It is recommended to use the Responses adapter given the newer OpenAI models and functionality use the Responses adapter.
+
+- ``OpenAICompletionsAdapter``: wraps the OpenAI Chat Completions API. It turns Toolkit objects into OpenAI-style tool definitions and processes tool calls appropriately.
+- ``OpenAIResponsesAdapter``: wraps the newer OpenAI Responses API. It collects tools, handles streaming events, and provides callbacks for advanced features like image generation or web search. 
+
+```Python Python
+from meshagent.openai import OpenAIResponsesAdapter
+from openai import AsyncOpenAI
+
+# Use an OpenAI client inside a MeshAgent LLMAdapter
+adapter = OpenAIResponsesAdapter(client=AsyncOpenAI(api_key="sk-..."))
+```
+
+### Tool Response Adapter
+The ``OpenAICompletionsToolResponseAdapter`` and ``OpenAIResponsesToolResponseAdapter``convert a tool's structured response into plain text or JSOn that can beinserted into an OpenAI chat context. 
+
+---
+### Learn more about MeshAgent on our website or check out the docs for additional examples!
+
+**Website**: [www.meshagent.com](https://www.meshagent.com/)
+
+**Documentation**: [docs.meshagent.com](https://docs.meshagent.com/)
+
+---

meshagent_openai-0.0.39/README.md

@@ -0,0 +1,30 @@
+## MeshAgent OpenAI
+
+## MeshAgent OpenAI
+The ``meshagent.openai`` package provides adapters to integrate OpenAI models with MeshAgent tools and agents. 
+
+### Completions Adapter and Responses Adapter
+MeshAgent supports both the OpenAI Chat Completions API and Responses API. It is recommended to use the Responses adapter given the newer OpenAI models and functionality use the Responses adapter.
+
+- ``OpenAICompletionsAdapter``: wraps the OpenAI Chat Completions API. It turns Toolkit objects into OpenAI-style tool definitions and processes tool calls appropriately.
+- ``OpenAIResponsesAdapter``: wraps the newer OpenAI Responses API. It collects tools, handles streaming events, and provides callbacks for advanced features like image generation or web search. 
+
+```Python Python
+from meshagent.openai import OpenAIResponsesAdapter
+from openai import AsyncOpenAI
+
+# Use an OpenAI client inside a MeshAgent LLMAdapter
+adapter = OpenAIResponsesAdapter(client=AsyncOpenAI(api_key="sk-..."))
+```
+
+### Tool Response Adapter
+The ``OpenAICompletionsToolResponseAdapter`` and ``OpenAIResponsesToolResponseAdapter``convert a tool's structured response into plain text or JSOn that can beinserted into an OpenAI chat context. 
+
+---
+### Learn more about MeshAgent on our website or check out the docs for additional examples!
+
+**Website**: [www.meshagent.com](https://www.meshagent.com/)
+
+**Documentation**: [docs.meshagent.com](https://docs.meshagent.com/)
+
+---

meshagent_openai-0.0.39/meshagent/openai/__init__.py

@@ -0,0 +1,16 @@
+from .tools import (
+    OpenAICompletionsAdapter,
+    OpenAIResponsesAdapter,
+    OpenAICompletionsToolResponseAdapter,
+    OpenAIResponsesToolResponseAdapter,
+)
+from .version import __version__
+
+
+__all__ = [
+    __version__,
+    OpenAICompletionsAdapter,
+    OpenAIResponsesAdapter,
+    OpenAICompletionsToolResponseAdapter,
+    OpenAIResponsesToolResponseAdapter,
+]

{meshagent_openai-0.0.37 → meshagent_openai-0.0.39}/meshagent/openai/proxy/__init__.py

@@ -1 +1,3 @@
 from .proxy import get_client
+
+__all__ = [get_client]

{meshagent_openai-0.0.37 → meshagent_openai-0.0.39}/meshagent/openai/proxy/proxy.py

@@ -1,26 +1,23 @@
-import os
 from meshagent.api import RoomClient
 from openai import AsyncOpenAI
 
+
 def get_client(*, room: RoomClient) -> AsyncOpenAI:
+    token: str = room.protocol.token
 
-    token : str = room.protocol.token
-    
     # when running inside the room pod, the room.room_url currently points to the external url
     # so we need to use url off the protocol (if available).
     # TODO: room_url should be set properly, but may need a claim in the token to be set during call to say it is local
-    url : str = getattr(room.protocol, "url", room.room_url)
-    
+    url: str = getattr(room.protocol, "url", room.room_url)
+
     room_proxy_url = f"{url}/v1"
 
     if room_proxy_url.startswith("ws:") or room_proxy_url.startswith("wss:"):
-        room_proxy_url = room_proxy_url.replace("ws","http",1)
+        room_proxy_url = room_proxy_url.replace("ws", "http", 1)
 
-    openai=AsyncOpenAI(
+    openai = AsyncOpenAI(
         api_key=token,
         base_url=room_proxy_url,
-        default_headers={
-            "Meshagent-Session" : room.session_id
-        }
+        default_headers={"Meshagent-Session": room.session_id},
     )
     return openai

meshagent_openai-0.0.39/meshagent/openai/tools/__init__.py

@@ -0,0 +1,18 @@
+from .responses_adapter import (
+    OpenAIResponsesAdapter,
+    OpenAIResponsesToolResponseAdapter,
+)
+from .completions_adapter import (
+    OpenAICompletionsAdapter,
+    OpenAICompletionsToolResponseAdapter,
+)
+from .stt import OpenAIAudioFileSTT, OpenAISTTToolkit
+
+__all__ = [
+    OpenAIResponsesAdapter,
+    OpenAIResponsesToolResponseAdapter,
+    OpenAICompletionsAdapter,
+    OpenAICompletionsToolResponseAdapter,
+    OpenAIAudioFileSTT,
+    OpenAISTTToolkit,
+]

meshagent_openai-0.0.39/meshagent/openai/tools/completions_adapter.py

@@ -0,0 +1,452 @@
+from meshagent.agents.agent import AgentChatContext
+from meshagent.api import RoomClient, RoomException
+from meshagent.tools.blob import Blob, BlobStorage
+from meshagent.tools import Toolkit, ToolContext
+from meshagent.api.messaging import (
+    Response,
+    LinkResponse,
+    FileResponse,
+    JsonResponse,
+    TextResponse,
+    EmptyResponse,
+)
+from meshagent.agents.adapter import ToolResponseAdapter, LLMAdapter
+import json
+from typing import List
+
+from openai import AsyncOpenAI, APIStatusError
+from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall
+
+import os
+from typing import Optional, Any
+
+import logging
+import re
+import asyncio
+
+from meshagent.openai.proxy import get_client
+
+logger = logging.getLogger("openai_agent")
+
+
+def _replace_non_matching(text: str, allowed_chars: str, replacement: str) -> str:
+    """
+    Replaces every character in `text` that does not match the given
+    `allowed_chars` regex set with `replacement`.
+
+    Parameters:
+    -----------
+    text : str
+        The input string on which the replacement is to be done.
+    allowed_chars : str
+        A string defining the set of allowed characters (part of a character set).
+        For example, "a-zA-Z0-9" will keep only letters and digits.
+    replacement : str
+        The string to replace non-matching characters with.
+
+    Returns:
+    --------
+    str
+        A new string where all characters not in `allowed_chars` are replaced.
+    """
+    # Build a regex that matches any character NOT in allowed_chars
+    pattern = rf"[^{allowed_chars}]"
+    return re.sub(pattern, replacement, text)
+
+
+def safe_tool_name(name: str):
+    return _replace_non_matching(name, "a-zA-Z0-9_-", "_")
+
+
+# Collects a group of tool proxies and manages execution of openai tool calls
+class CompletionsToolBundle:
+    def __init__(self, toolkits: List[Toolkit]):
+        self._toolkits = toolkits
+        self._executors = dict[str, Toolkit]()
+        self._safe_names = {}
+
+        open_ai_tools = []
+
+        for toolkit in toolkits:
+            for v in toolkit.tools:
+                k = v.name
+
+                name = safe_tool_name(k)
+
+                if k in self._executors:
+                    raise Exception(
+                        f"duplicate in bundle '{k}', tool names must be unique."
+                    )
+
+                self._executors[k] = toolkit
+
+                self._safe_names[name] = k
+
+                fn = {
+                    "name": name,
+                    "parameters": {
+                        **v.input_schema,
+                    },
+                    "strict": True,
+                }
+
+                if v.defs is not None:
+                    fn["parameters"]["$defs"] = v.defs
+
+                schema = {
+                    "type": "function",
+                    "function": fn,
+                }
+
+                open_ai_tools.append(schema)
+
+        if len(open_ai_tools) == 0:
+            open_ai_tools = None
+
+        self._open_ai_tools = open_ai_tools
+
+    async def execute(
+        self, *, context: ToolContext, tool_call: ChatCompletionMessageToolCall
+    ) -> Response:
+        try:
+            function = tool_call.function
+            name = function.name
+            arguments = json.loads(function.arguments)
+
+            if name not in self._safe_names:
+                raise RoomException(
+                    f"Invalid tool name {name}, check the name of the tool"
+                )
+
+            name = self._safe_names[name]
+
+            if name not in self._executors:
+                raise Exception(f"Unregistered tool name {name}")
+
+            logger.info("executing %s %s %s", tool_call.id, name, arguments)
+
+            proxy = self._executors[name]
+            result = await proxy.execute(
+                context=context, name=name, arguments=arguments
+            )
+            logger.info("success calling %s %s %s", tool_call.id, name, result)
+            return result
+
+        except Exception as e:
+            logger.error("failed calling %s %s", tool_call.id, name, exc_info=e)
+            raise
+
+    def contains(self, name: str) -> bool:
+        return name in self._open_ai_tools
+
+    def to_json(self) -> List[dict] | None:
+        if self._open_ai_tools is None:
+            return None
+        return self._open_ai_tools.copy()
+
+
+# Converts a tool response into a series of messages that can be inserted into the openai context
+class OpenAICompletionsToolResponseAdapter(ToolResponseAdapter):
+    def __init__(self, blob_storage: Optional[BlobStorage] = None):
+        self._blob_storage = blob_storage
+        pass
+
+    async def to_plain_text(self, *, room: RoomClient, response: Response) -> str:
+        if isinstance(response, LinkResponse):
+            return json.dumps(
+                {
+                    "name": response.name,
+                    "url": response.url,
+                }
+            )
+
+        elif isinstance(response, JsonResponse):
+            return json.dumps(response.json)
+
+        elif isinstance(response, TextResponse):
+            return response.text
+
+        elif isinstance(response, FileResponse):
+            blob = Blob(mime_type=response.mime_type, data=response.data)
+            uri = self._blob_storage.store(blob=blob)
+
+            return f"The results have been written to a blob with the uri {uri} with the mime type {blob.mime_type}."
+
+        elif isinstance(response, EmptyResponse):
+            return "ok"
+
+        # elif isinstance(response, ImageResponse):
+        #     context.messages.append({
+        #         "role" : "tool",
+        #         "content" : "the user will upload the image",
+        #         "tool_call_id" : tool_call.id,
+        #     })
+        #     context.messages.append({
+        #         "role" : "user",
+        #         "content" : [
+        #             { "type" : "text", "text": "this is the image from tool call id {tool_call.id}" },
+        #             { "type" : "image_url", "image_url": {"url": response.url, "detail": "auto"} }
+        #         ]
+        #     })
+
+        elif isinstance(response, dict):
+            return json.dumps(response)
+
+        elif isinstance(response, str):
+            return response
+
+        elif response is None:
+            return "ok"
+
+        else:
+            raise Exception(
+                "unexpected return type: {type}".format(type=type(response))
+            )
+
+    async def create_messages(
+        self,
+        *,
+        context: AgentChatContext,
+        tool_call: Any,
+        room: RoomClient,
+        response: Response,
+    ) -> list:
+        message = {
+            "role": "tool",
+            "content": await self.to_plain_text(room=room, response=response),
+            "tool_call_id": tool_call.id,
+        }
+
+        room.developer.log_nowait(
+            type="llm.message",
+            data={
+                "context": context.id,
+                "participant_id": room.local_participant.id,
+                "participant_name": room.local_participant.get_attribute("name"),
+                "message": message,
+            },
+        )
+
+        return [message]
+
+
+class OpenAICompletionsAdapter(LLMAdapter):
+    def __init__(
+        self,
+        model: str = os.getenv("OPENAI_MODEL"),
+        parallel_tool_calls: Optional[bool] = None,
+        client: Optional[AsyncOpenAI] = None,
+    ):
+        self._model = model
+        self._parallel_tool_calls = parallel_tool_calls
+        self._client = client
+
+    def create_chat_context(self):
+        system_role = "system"
+        if self._model.startswith("o1"):
+            system_role = "developer"
+        elif self._model.startswith("o3"):
+            system_role = "developer"
+        elif self._model.startswith("o4"):
+            system_role = "developer"
+
+        context = AgentChatContext(system_role=system_role)
+
+        return context
+
+    # Takes the current chat context, executes a completion request and processes the response.
+    # If a tool calls are requested, invokes the tools, processes the tool calls results, and appends the tool call results to the context
+    async def next(
+        self,
+        *,
+        context: AgentChatContext,
+        room: RoomClient,
+        toolkits: Toolkit,
+        tool_adapter: Optional[ToolResponseAdapter] = None,
+        output_schema: Optional[dict] = None,
+    ):
+        if tool_adapter is None:
+            tool_adapter = OpenAICompletionsToolResponseAdapter()
+
+        try:
+            openai = get_client(room=room)
+
+            tool_bundle = CompletionsToolBundle(
+                toolkits=[
+                    *toolkits,
+                ]
+            )
+            open_ai_tools = tool_bundle.to_json()
+
+            if open_ai_tools is not None:
+                logger.info("OpenAI Tools: %s", json.dumps(open_ai_tools))
+            else:
+                logger.info("OpenAI Tools: Empty")
+
+            response_schema = output_schema
+            response_name = "response"
+
+            while True:
+                logger.info(
+                    "model: %s, context: %s, output_schema: %s",
+                    self._model,
+                    context.messages,
+                    output_schema,
+                )
+                ptc = self._parallel_tool_calls
+                extra = {}
+                if ptc is not None and not self._model.startswith("o"):
+                    extra["parallel_tool_calls"] = ptc
+
+                if output_schema is not None:
+                    extra["response_format"] = {
+                        "type": "json_schema",
+                        "json_schema": {
+                            "name": response_name,
+                            "schema": response_schema,
+                            "strict": True,
+                        },
+                    }
+
+                response: ChatCompletion = await openai.chat.completions.create(
+                    n=1,
+                    model=self._model,
+                    messages=context.messages,
+                    tools=open_ai_tools,
+                    **extra,
+                )
+                message = response.choices[0].message
+                room.developer.log_nowait(
+                    type="llm.message",
+                    data={
+                        "context": context.id,
+                        "participant_id": room.local_participant.id,
+                        "participant_name": room.local_participant.get_attribute(
+                            "name"
+                        ),
+                        "message": message.to_dict(),
+                    },
+                )
+                context.messages.append(message)
+
+                if message.tool_calls is not None:
+                    tasks = []
+
+                    async def do_tool_call(tool_call: ChatCompletionMessageToolCall):
+                        try:
+                            tool_context = ToolContext(
+                                room=room,
+                                caller=room.local_participant,
+                                caller_context={"chat": context.to_json},
+                            )
+                            tool_response = await tool_bundle.execute(
+                                context=tool_context, tool_call=tool_call
+                            )
+                            logger.info(f"tool response {tool_response}")
+                            return await tool_adapter.create_messages(
+                                context=context,
+                                tool_call=tool_call,
+                                room=room,
+                                response=tool_response,
+                            )
+
+                        except Exception as e:
+                            logger.error(
+                                f"unable to complete tool call {tool_call}", exc_info=e
+                            )
+                            room.developer.log_nowait(
+                                type="llm.error",
+                                data={
+                                    "participant_id": room.local_participant.id,
+                                    "participant_name": room.local_participant.get_attribute(
+                                        "name"
+                                    ),
+                                    "error": f"{e}",
+                                },
+                            )
+
+                            return [
+                                {
+                                    "role": "tool",
+                                    "content": json.dumps(
+                                        {"error": f"unable to complete tool call: {e}"}
+                                    ),
+                                    "tool_call_id": tool_call.id,
+                                }
+                            ]
+
+                    for tool_call in message.tool_calls:
+                        tasks.append(asyncio.create_task(do_tool_call(tool_call)))
+
+                    results = await asyncio.gather(*tasks)
+
+                    for result in results:
+                        if result is not None:
+                            room.developer.log_nowait(
+                                type="llm.message",
+                                data={
+                                    "context": context.id,
+                                    "participant_id": room.local_participant.id,
+                                    "participant_name": room.local_participant.get_attribute(
+                                        "name"
+                                    ),
+                                    "message": result,
+                                },
+                            )
+                            context.messages.append(result)
+
+                elif message.content is not None:
+                    content = message.content
+
+                    logger.info("RESPONSE FROM OPENAI %s", content)
+                    if response_schema is None:
+                        return content
+
+                    # First try to parse the result
+                    try:
+                        full_response = json.loads(content)
+                    # sometimes open ai packs two JSON chunks seperated by newline, check if that's why we couldn't parse
+                    except json.decoder.JSONDecodeError:
+                        for part in content.splitlines():
+                            if len(part.strip()) > 0:
+                                full_response = json.loads(part)
+
+                                try:
+                                    self.validate(
+                                        response=full_response,
+                                        output_schema=response_schema,
+                                    )
+                                except Exception as e:
+                                    logger.error(
+                                        "recieved invalid response, retrying",
+                                        exc_info=e,
+                                    )
+                                    error = {
+                                        "role": "user",
+                                        "content": "encountered a validation error with the output: {error}".format(
+                                            error=e
+                                        ),
+                                    }
+                                    room.developer.log_nowait(
+                                        type="llm.message",
+                                        data={
+                                            "context": context.id,
+                                            "participant_id": room.local_participant.id,
+                                            "participant_name": room.local_participant.get_attribute(
+                                                "name"
+                                            ),
+                                            "message": error,
+                                        },
+                                    )
+                                    context.messages.append(error)
+                                    continue
+
+                    return full_response
+                else:
+                    raise RoomException(
+                        "Unexpected response from OpenAI {response}".format(
+                            response=message
+                        )
+                    )
+        except APIStatusError as e:
+            raise RoomException(f"Error from OpenAI: {e}")