swarmauri_live_model_openai 0.1.0.dev3__py3-none-any.whl

swarmauri_live_model_openai/OpenAIRealtimeCodec.py

@@ -0,0 +1,184 @@
+import base64
+import json
+from typing import Any
+
+from swarmauri_base.live_models.events.AudioDeltaEvent import AudioDeltaEvent
+from swarmauri_base.live_models.events.InputTranscriptDeltaEvent import (
+    InputTranscriptDeltaEvent,
+)
+from swarmauri_base.live_models.events.InterruptionEvent import (
+    InterruptionEvent,
+)
+from swarmauri_base.live_models.events.OutputTranscriptDeltaEvent import (
+    OutputTranscriptDeltaEvent,
+)
+from swarmauri_base.live_models.events.SessionErrorEvent import (
+    SessionErrorEvent,
+)
+from swarmauri_base.live_models.events.TextDeltaEvent import TextDeltaEvent
+from swarmauri_base.live_models.events.ToolCallEvent import ToolCallEvent
+from swarmauri_base.live_models.events.TurnCompleteEvent import (
+    TurnCompleteEvent,
+)
+from swarmauri_base.live_models.events.UsageEvent import UsageEvent
+from swarmauri_base.live_models.LiveEventBase import LiveEventBase
+from swarmauri_base.live_models.LiveInputBase import LiveInputBase
+
+
+class OpenAIRealtimeCodec:
+    def encode(self, input_data: LiveInputBase) -> list[str]:
+        if input_data.kind == "text":
+            return [
+                json.dumps(
+                    {
+                        "type": "conversation.item.create",
+                        "item": {
+                            "type": "message",
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "input_text",
+                                    "text": input_data.payload,
+                                }
+                            ],
+                        },
+                    }
+                ),
+                json.dumps({"type": "response.create"}),
+            ]
+        if input_data.kind == "audio":
+            return [
+                json.dumps(
+                    {
+                        "type": "input_audio_buffer.append",
+                        "audio": base64.b64encode(input_data.payload).decode(),
+                    }
+                )
+            ]
+        if input_data.kind == "image":
+            encoded = base64.b64encode(input_data.payload).decode()
+            image_url = f"data:{input_data.mime_type};base64,{encoded}"
+            return [
+                json.dumps(
+                    {
+                        "type": "conversation.item.create",
+                        "item": {
+                            "type": "message",
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "input_image",
+                                    "image_url": image_url,
+                                }
+                            ],
+                        },
+                    }
+                )
+            ]
+        if input_data.kind == "tool_result":
+            return [
+                json.dumps(
+                    {
+                        "type": "conversation.item.create",
+                        "item": {
+                            "type": "function_call_output",
+                            "call_id": input_data.call_id,
+                            "output": json.dumps(input_data.payload),
+                        },
+                    }
+                ),
+                json.dumps({"type": "response.create"}),
+            ]
+        if input_data.kind == "commit_turn":
+            return [
+                json.dumps({"type": "input_audio_buffer.commit"}),
+                json.dumps({"type": "response.create"}),
+            ]
+        if input_data.kind == "interrupt":
+            return [json.dumps({"type": "response.cancel"})]
+        raise ValueError(
+            f"Unsupported OpenAI realtime input: {input_data.kind}"
+        )
+
+    def decode(self, message: str | bytes) -> list[LiveEventBase]:
+        data: dict[str, Any] = json.loads(message)
+        kind = data.get("type", "")
+        correlation = {
+            "provider_event_id": data.get("event_id"),
+            "response_id": data.get("response_id")
+            or data.get("response", {}).get("id"),
+            "item_id": data.get("item_id"),
+            "call_id": data.get("call_id"),
+        }
+        if kind in {"response.output_text.delta", "response.text.delta"}:
+            return [
+                TextDeltaEvent(
+                    payload=data.get("delta", ""), correlation=correlation
+                )
+            ]
+        if kind in {"response.output_audio.delta", "response.audio.delta"}:
+            return [
+                AudioDeltaEvent(
+                    payload=base64.b64decode(data.get("delta", "")),
+                    correlation=correlation,
+                )
+            ]
+        if kind in {
+            "response.output_audio_transcript.delta",
+            "response.audio_transcript.delta",
+        }:
+            return [
+                OutputTranscriptDeltaEvent(
+                    payload=data.get("delta", ""), correlation=correlation
+                )
+            ]
+        if "input_audio_transcription" in kind:
+            text = data.get("delta") or data.get("transcript", "")
+            return [
+                InputTranscriptDeltaEvent(
+                    payload=text, correlation=correlation
+                )
+            ]
+        if kind == "response.function_call_arguments.done":
+            arguments = data.get("arguments", {})
+            if isinstance(arguments, str):
+                arguments = json.loads(arguments or "{}")
+            return [
+                ToolCallEvent(
+                    call_id=data.get("call_id", ""),
+                    tool_name=data.get("name", ""),
+                    arguments=arguments,
+                    correlation=correlation,
+                )
+            ]
+        if kind == "input_audio_buffer.speech_started":
+            return [
+                InterruptionEvent(
+                    reason="speech_started", correlation=correlation
+                )
+            ]
+        if kind == "response.done":
+            events: list[LiveEventBase] = [
+                TurnCompleteEvent(correlation=correlation)
+            ]
+            usage = data.get("response", {}).get("usage")
+            if usage:
+                events.append(
+                    UsageEvent(
+                        payload={
+                            key: int(value)
+                            for key, value in usage.items()
+                            if isinstance(value, int)
+                        },
+                        correlation=correlation,
+                    )
+                )
+            return events
+        if kind == "error":
+            return [
+                SessionErrorEvent(
+                    message=str(data.get("error", data)),
+                    correlation=correlation,
+                )
+            ]
+        return []

swarmauri_live_model_openai/OpenAIRealtimeConfigMapper.py

@@ -0,0 +1,19 @@
+from typing import Any
+
+from swarmauri_base.live_models.LiveConfigBase import LiveConfigBase
+from swarmauri_provider_openai import OpenAIToolSchemaConverter
+
+
+class OpenAIRealtimeConfigMapper:
+    def map(self, config: LiveConfigBase) -> dict[str, Any]:
+        session: dict[str, Any] = {
+            "type": "realtime",
+            "output_modalities": config.requested_modalities,
+        }
+        if config.tools:
+            converter = OpenAIToolSchemaConverter()
+            session["tools"] = [
+                converter.convert(tool) for tool in config.tools
+            ]
+        session.update(config.provider_options.get("session", {}))
+        return {"type": "session.update", "session": session}

swarmauri_live_model_openai/OpenAIRealtimeModel.py

@@ -0,0 +1,42 @@
+from typing import Any, Literal
+from pydantic import ConfigDict, Field
+from swarmauri_base.ComponentBase import ComponentBase
+from swarmauri_base.live_models.LiveCapabilitiesBase import (
+    LiveCapabilitiesBase,
+)
+from swarmauri_base.live_models.LiveConfigBase import LiveConfigBase
+from swarmauri_base.live_models.LiveModelBase import LiveModelBase
+from swarmauri_provider_openai import OpenAIProvider
+from swarmauri_transport_websocket import WebSocketClient
+
+from .OpenAIRealtimeConfigMapper import OpenAIRealtimeConfigMapper
+from .OpenAIRealtimeSession import OpenAIRealtimeSession
+
+
+@ComponentBase.register_type(LiveModelBase, "OpenAIRealtimeModel")
+class OpenAIRealtimeModel(LiveModelBase):
+    name: str = "gpt-realtime"
+    provider: OpenAIProvider = Field(default_factory=OpenAIProvider)
+    transport: Any = Field(default_factory=WebSocketClient, exclude=True)
+    capabilities: LiveCapabilitiesBase = Field(
+        default_factory=lambda: LiveCapabilitiesBase(
+            input_modalities=frozenset({"text", "audio", "image"}),
+            output_modalities=frozenset({"text", "audio"}),
+            features=frozenset({"tools", "transcription"}),
+            session_features=frozenset({"realtime", "interruption"}),
+        )
+    )
+    type: Literal["OpenAIRealtimeModel"] = "OpenAIRealtimeModel"
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
+
+    async def aconnect(self, config: LiveConfigBase) -> OpenAIRealtimeSession:
+        connection_config = self.provider.prepare_connection(
+            "realtime.connect", model=self.name
+        )
+        connection = await self.transport.aconnect(
+            connection_config.resolved_url(),
+            headers=connection_config.headers,
+        )
+        return await OpenAIRealtimeSession.create(
+            connection, OpenAIRealtimeConfigMapper().map(config)
+        )

swarmauri_live_model_openai/OpenAIRealtimeSession.py

@@ -0,0 +1,64 @@
+import asyncio
+import json
+from contextlib import suppress
+from typing import Any, Literal
+
+from pydantic import PrivateAttr
+from swarmauri_base.live_models.events.SessionErrorEvent import (
+    SessionErrorEvent,
+)
+from swarmauri_base.live_models.LiveInputBase import LiveInputBase
+from swarmauri_base.live_models.LiveSessionBase import LiveSessionBase
+from swarmauri_base.ComponentBase import ComponentBase
+from swarmauri_core.transports import IWebSocketConnection
+
+from .OpenAIRealtimeCodec import OpenAIRealtimeCodec
+
+
+@ComponentBase.register_type(LiveSessionBase, "OpenAIRealtimeSession")
+class OpenAIRealtimeSession(LiveSessionBase):
+    type: Literal["OpenAIRealtimeSession"] = "OpenAIRealtimeSession"
+    _connection: IWebSocketConnection = PrivateAttr()
+    _codec: OpenAIRealtimeCodec = PrivateAttr(
+        default_factory=OpenAIRealtimeCodec
+    )
+    _receiver: Any = PrivateAttr(default=None)
+
+    @classmethod
+    async def create(
+        cls, connection, setup: dict | None = None
+    ) -> "OpenAIRealtimeSession":
+        session = cls()
+        session._connection = connection
+        if setup:
+            await connection.asend(json.dumps(setup))
+        session.mark_open()
+        session._receiver = asyncio.create_task(session._receive_loop())
+        return session
+
+    async def _receive_loop(self) -> None:
+        try:
+            async for message in self._connection.aiter_messages():
+                for event in self._codec.decode(message):
+                    await self.emit(event)
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            await self.emit(SessionErrorEvent(message=str(exc)))
+            await self.fail(exc)
+        else:
+            await self.finish()
+
+    async def _asend_impl(self, input_data: LiveInputBase) -> None:
+        for message in self._codec.encode(input_data):
+            await self._connection.asend(message)
+
+    async def _aclose_impl(self) -> None:
+        if (
+            self._receiver is not None
+            and self._receiver is not asyncio.current_task()
+        ):
+            self._receiver.cancel()
+            with suppress(asyncio.CancelledError):
+                await self._receiver
+        await self._connection.aclose()

swarmauri_live_model_openai/OpenAIRealtimeToolAdapter.py

@@ -0,0 +1,10 @@
+from swarmauri_base.live_models.inputs.ToolResultInput import ToolResultInput
+
+
+class OpenAIRealtimeToolAdapter:
+    def result(
+        self, call_id: str, value: object, *, is_error: bool = False
+    ) -> ToolResultInput:
+        return ToolResultInput(
+            call_id=call_id, payload=value, is_error=is_error
+        )

swarmauri_live_model_openai/OpenAIResponsesWSModel.py

@@ -0,0 +1,23 @@
+from typing import Literal
+
+from swarmauri_base.ComponentBase import ComponentBase
+from swarmauri_base.live_models.LiveConfigBase import LiveConfigBase
+from swarmauri_base.live_models.LiveModelBase import LiveModelBase
+
+from .OpenAIRealtimeModel import OpenAIRealtimeModel
+from .OpenAIRealtimeSession import OpenAIRealtimeSession
+
+
+@ComponentBase.register_type(LiveModelBase, "OpenAIResponsesWSModel")
+class OpenAIResponsesWSModel(OpenAIRealtimeModel):
+    type: Literal["OpenAIResponsesWSModel"] = "OpenAIResponsesWSModel"
+
+    async def aconnect(self, config: LiveConfigBase) -> OpenAIRealtimeSession:
+        connection_config = self.provider.prepare_connection(
+            "responses.connect", model=self.name
+        )
+        connection = await self.transport.aconnect(
+            connection_config.resolved_url(),
+            headers=connection_config.headers,
+        )
+        return await OpenAIRealtimeSession.create(connection)

swarmauri_live_model_openai/__init__.py

@@ -0,0 +1,15 @@
+from .OpenAIRealtimeCodec import OpenAIRealtimeCodec
+from .OpenAIRealtimeConfigMapper import OpenAIRealtimeConfigMapper
+from .OpenAIRealtimeModel import OpenAIRealtimeModel
+from .OpenAIRealtimeSession import OpenAIRealtimeSession
+from .OpenAIRealtimeToolAdapter import OpenAIRealtimeToolAdapter
+from .OpenAIResponsesWSModel import OpenAIResponsesWSModel
+
+__all__ = [
+    "OpenAIRealtimeCodec",
+    "OpenAIRealtimeConfigMapper",
+    "OpenAIRealtimeModel",
+    "OpenAIRealtimeSession",
+    "OpenAIRealtimeToolAdapter",
+    "OpenAIResponsesWSModel",
+]

swarmauri_live_model_openai-0.1.0.dev3.dist-info/METADATA

@@ -0,0 +1,53 @@
+Metadata-Version: 2.4
+Name: swarmauri_live_model_openai
+Version: 0.1.0.dev3
+Summary: OpenAI Realtime and persistent Responses WebSocket model adapters for Swarmauri.
+License: Copyright 2024 Jacob Stewart
+         
+         Licensed under the Apache License, Version 2.0 (the "License");
+         you may not use this file except in compliance with the License.
+         You may obtain a copy of the License at
+         
+             http://www.apache.org/licenses/LICENSE-2.0
+         
+         Unless required by applicable law or agreed to in writing, software
+         distributed under the License is distributed on an "AS IS" BASIS,
+         WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         See the License for the specific language governing permissions and
+         limitations under the License.
+License-File: LICENSE
+Keywords: swarmauri,openai realtime,websocket,audio,tools
+Author: Jacob Stewart
+Author-email: jacob@swarmauri.com
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 1 - Planning
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: swarmauri_base (>=0.11.0.dev5)
+Requires-Dist: swarmauri_core (>=0.11.0.dev5)
+Requires-Dist: swarmauri_provider_openai (>=0.1.0.dev3)
+Requires-Dist: swarmauri_transport_websocket (>=0.1.0.dev4)
+Description-Content-Type: text/markdown
+
+![Swarmauri](https://github.com/swarmauri/swarmauri-sdk/blob/3d4d1cfa949399d7019ae9d8f296afba773dfb7f/assets/swarmauri.brand.theme.svg)
+
+[![PyPI](https://img.shields.io/pypi/v/swarmauri_live_model_openai.svg)](https://pypi.org/project/swarmauri_live_model_openai/) [![Python](https://img.shields.io/pypi/pyversions/swarmauri_live_model_openai.svg)](https://pypi.org/project/swarmauri_live_model_openai/) [![License](https://img.shields.io/badge/license-Apache--2.0-blue)](LICENSE) [![Downloads](https://static.pepy.tech/badge/swarmauri_live_model_openai)](https://pepy.tech/project/swarmauri_live_model_openai) [![Hits](https://hits.sh/github.com/swarmauri/swarmauri-sdk.svg)](https://hits.sh/github.com/swarmauri/swarmauri-sdk/)
+
+# swarmauri_live_model_openai
+
+## Features
+
+- OpenAI Realtime multimodal conversations over WebSocket.
+- Persistent Responses WebSocket adapter.
+- Typed text, audio, transcripts, tools, interruption, and usage events.
+
+## Installation
+
+`uv add swarmauri_live_model_openai` or `pip install swarmauri_live_model_openai`.
+
+## Usage
+
+Inject an `OpenAIProvider`, connect with `LiveConfigBase`, send typed inputs, and consume the session event stream.
+

swarmauri_live_model_openai-0.1.0.dev3.dist-info/RECORD

@@ -0,0 +1,12 @@
+swarmauri_live_model_openai/OpenAIRealtimeCodec.py,sha256=TUSbJfXf5Z7taBcoTxTknyMI-YzaAG0OZ_dNZkxwm7k,6869
+swarmauri_live_model_openai/OpenAIRealtimeConfigMapper.py,sha256=0OADxUzAEfim7B4S_YaDguhMB3bCYsNamqnsOKlABRw,712
+swarmauri_live_model_openai/OpenAIRealtimeModel.py,sha256=Y2fZR84knhMvOs0uZBz1cjPV8fZgEwnoF3nFYNJB8QE,1870
+swarmauri_live_model_openai/OpenAIRealtimeSession.py,sha256=QIx4YJKvqu3SUQsfAGm7dbEoO8GmEX-ogp8NRnxI-Ao,2259
+swarmauri_live_model_openai/OpenAIRealtimeToolAdapter.py,sha256=7VpKTMEye3-9f5DmD0JfGDYjvACUsFivNvtQN5Y47QU,328
+swarmauri_live_model_openai/OpenAIResponsesWSModel.py,sha256=rK8fJEnvzimXLNV2ynXAX8FGrqLpYpICClAgtr9U0qs,942
+swarmauri_live_model_openai/__init__.py,sha256=BxxaX-iN-PwA2YktnRWUS7YQ9f8AdSiZKqNWDIsUBZY,549
+swarmauri_live_model_openai-0.1.0.dev3.dist-info/METADATA,sha256=fmHA8G50OAfsyQaRDjlMrlDfSJ5EIBGrqDsS5QcJeTA,2693
+swarmauri_live_model_openai-0.1.0.dev3.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
+swarmauri_live_model_openai-0.1.0.dev3.dist-info/entry_points.txt,sha256=DYt_dq1gKCZExhHlWg5Z7MS-jPY2sdDVVawfC87HbSQ,167
+swarmauri_live_model_openai-0.1.0.dev3.dist-info/licenses/LICENSE,sha256=kkwAgrDtfQ7c-Ozql2xlztDDjHt0YwGmRG7b_oUU9Jg,554
+swarmauri_live_model_openai-0.1.0.dev3.dist-info/RECORD,,

swarmauri_live_model_openai-0.1.0.dev3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.5.0
+Root-Is-Purelib: true
+Tag: py3-none-any

swarmauri_live_model_openai-0.1.0.dev3.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[swarmauri.live_models]
+OpenAIRealtimeModel=swarmauri_live_model_openai:OpenAIRealtimeModel
+OpenAIResponsesWSModel=swarmauri_live_model_openai:OpenAIResponsesWSModel
+

swarmauri_live_model_openai-0.1.0.dev3.dist-info/licenses/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2024 Jacob Stewart
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.