swarmauri_live_model_gemini 0.1.0.dev3__py3-none-any.whl

swarmauri_live_model_gemini/GeminiLiveCodec.py

@@ -0,0 +1,150 @@
+import base64
+import json
+from typing import Any
+
+from swarmauri_base.live_models.events.AudioDeltaEvent import AudioDeltaEvent
+from swarmauri_base.live_models.events.InputTranscriptDeltaEvent import (
+    InputTranscriptDeltaEvent,
+)
+from swarmauri_base.live_models.events.OutputTranscriptDeltaEvent import (
+    OutputTranscriptDeltaEvent,
+)
+from swarmauri_base.live_models.events.SessionErrorEvent import (
+    SessionErrorEvent,
+)
+from swarmauri_base.live_models.events.SessionLifecycleEvent import (
+    SessionLifecycleEvent,
+)
+from swarmauri_base.live_models.events.SessionResumptionEvent import (
+    SessionResumptionEvent,
+)
+from swarmauri_base.live_models.events.TextDeltaEvent import TextDeltaEvent
+from swarmauri_base.live_models.events.ToolCallEvent import ToolCallEvent
+from swarmauri_base.live_models.events.TurnCompleteEvent import (
+    TurnCompleteEvent,
+)
+from swarmauri_base.live_models.events.UsageEvent import UsageEvent
+from swarmauri_base.live_models.LiveEventBase import LiveEventBase
+from swarmauri_base.live_models.LiveInputBase import LiveInputBase
+
+
+class GeminiLiveCodec:
+    def encode(self, input_data: LiveInputBase) -> str:
+        if input_data.kind == "text":
+            payload = {"realtimeInput": {"text": input_data.payload}}
+        elif input_data.kind in {"audio", "image", "video"}:
+            payload = {
+                "realtimeInput": {
+                    "mediaChunks": [
+                        {
+                            "mimeType": input_data.mime_type,
+                            "data": base64.b64encode(
+                                input_data.payload
+                            ).decode(),
+                        }
+                    ]
+                }
+            }
+        elif input_data.kind == "tool_result":
+            payload = {
+                "toolResponse": {
+                    "functionResponses": [
+                        {
+                            "id": input_data.call_id,
+                            "name": input_data.correlation_id or "tool",
+                            "response": {"output": input_data.payload},
+                        }
+                    ]
+                }
+            }
+        elif input_data.kind == "commit_turn":
+            payload = {"realtimeInput": {"activityEnd": {}}}
+        elif input_data.kind == "interrupt":
+            payload = {"realtimeInput": {"activityStart": {}}}
+        else:
+            raise ValueError(
+                f"Unsupported Gemini live input: {input_data.kind}"
+            )
+        return json.dumps(payload)
+
+    def decode(self, message: str | bytes) -> list[LiveEventBase]:
+        data: dict[str, Any] = json.loads(message)
+        events: list[LiveEventBase] = []
+        server = data.get("serverContent", {})
+        model_turn = server.get("modelTurn", {})
+        correlation = {"provider_event_id": data.get("id")}
+
+        for part in model_turn.get("parts", []):
+            if "text" in part:
+                events.append(
+                    TextDeltaEvent(
+                        payload=part["text"], correlation=correlation
+                    )
+                )
+            inline = part.get("inlineData") or part.get("inline_data")
+            if inline and str(inline.get("mimeType", "")).startswith("audio/"):
+                events.append(
+                    AudioDeltaEvent(
+                        payload=base64.b64decode(inline["data"]),
+                        mime_type=inline.get("mimeType", "audio/pcm"),
+                        correlation=correlation,
+                    )
+                )
+
+        input_tx = server.get("inputTranscription", {})
+        if input_tx.get("text"):
+            events.append(
+                InputTranscriptDeltaEvent(
+                    payload=input_tx["text"], correlation=correlation
+                )
+            )
+        output_tx = server.get("outputTranscription", {})
+        if output_tx.get("text"):
+            events.append(
+                OutputTranscriptDeltaEvent(
+                    payload=output_tx["text"], correlation=correlation
+                )
+            )
+        if server.get("turnComplete"):
+            events.append(TurnCompleteEvent(correlation=correlation))
+
+        for call in data.get("toolCall", {}).get("functionCalls", []):
+            events.append(
+                ToolCallEvent(
+                    call_id=call.get("id", call.get("name", "")),
+                    tool_name=call["name"],
+                    arguments=call.get("args", {}),
+                    correlation=correlation,
+                )
+            )
+        if data.get("usageMetadata"):
+            usage = {
+                key: int(value)
+                for key, value in data["usageMetadata"].items()
+                if isinstance(value, int)
+            }
+            events.append(UsageEvent(payload=usage, correlation=correlation))
+        if data.get("sessionResumptionUpdate"):
+            update = data["sessionResumptionUpdate"]
+            events.append(
+                SessionResumptionEvent(
+                    token=update.get("newHandle"),
+                    resumable=bool(update.get("resumable", True)),
+                    correlation=correlation,
+                )
+            )
+        if data.get("goAway"):
+            events.append(
+                SessionLifecycleEvent(
+                    state="closing",
+                    payload=data["goAway"],
+                    correlation=correlation,
+                )
+            )
+        if data.get("error"):
+            events.append(
+                SessionErrorEvent(
+                    message=str(data["error"]), correlation=correlation
+                )
+            )
+        return events

swarmauri_live_model_gemini/GeminiLiveConfigMapper.py

@@ -0,0 +1,30 @@
+from typing import Any
+
+from swarmauri_base.live_models.LiveConfigBase import LiveConfigBase
+from swarmauri_provider_gemini import GeminiToolSchemaConverter
+
+
+class GeminiLiveConfigMapper:
+    def __init__(self) -> None:
+        self._tools = GeminiToolSchemaConverter()
+
+    def map(self, model: str, config: LiveConfigBase) -> dict[str, Any]:
+        generation = {
+            "responseModalities": [
+                modality.upper() for modality in config.requested_modalities
+            ]
+        }
+        setup: dict[str, Any] = {
+            "model": f"models/{model}",
+            "generationConfig": generation,
+        }
+        if config.tools:
+            setup["tools"] = [
+                {
+                    "functionDeclarations": [
+                        self._tools.convert(tool) for tool in config.tools
+                    ]
+                }
+            ]
+        setup.update(config.provider_options.get("setup", {}))
+        return {"setup": setup}

swarmauri_live_model_gemini/GeminiLiveModel.py

@@ -0,0 +1,43 @@
+from typing import Any, Literal
+from pydantic import ConfigDict, Field
+from swarmauri_base.ComponentBase import ComponentBase
+from swarmauri_base.live_models.LiveCapabilitiesBase import (
+    LiveCapabilitiesBase,
+)
+from swarmauri_base.live_models.LiveConfigBase import LiveConfigBase
+from swarmauri_base.live_models.LiveModelBase import LiveModelBase
+from swarmauri_provider_gemini import GeminiProvider
+from swarmauri_transport_websocket import WebSocketClient
+
+from .GeminiLiveConfigMapper import GeminiLiveConfigMapper
+from .GeminiLiveSession import GeminiLiveSession
+
+
+@ComponentBase.register_type(LiveModelBase, "GeminiLiveModel")
+class GeminiLiveModel(LiveModelBase):
+    name: str = "gemini-3.1-flash-live-preview"
+    provider: GeminiProvider = Field(default_factory=GeminiProvider)
+    transport: Any = Field(default_factory=WebSocketClient, exclude=True)
+    capabilities: LiveCapabilitiesBase = Field(
+        default_factory=lambda: LiveCapabilitiesBase(
+            input_modalities=frozenset({"text", "audio", "image", "video"}),
+            output_modalities=frozenset({"text", "audio"}),
+            features=frozenset({"tools", "transcription"}),
+            session_features=frozenset(
+                {"realtime", "interruption", "session_resumption"}
+            ),
+        )
+    )
+    type: Literal["GeminiLiveModel"] = "GeminiLiveModel"
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
+
+    async def aconnect(self, config: LiveConfigBase) -> GeminiLiveSession:
+        connection_config = self.provider.prepare_connection(
+            "live.connect", model=self.name
+        )
+        connection = await self.transport.aconnect(
+            connection_config.resolved_url(),
+            headers=connection_config.headers,
+        )
+        setup = GeminiLiveConfigMapper().map(self.name, config)
+        return await GeminiLiveSession.create(connection, setup)

swarmauri_live_model_gemini/GeminiLiveSession.py

@@ -0,0 +1,58 @@
+import asyncio
+import json
+from contextlib import suppress
+from typing import Any, Literal
+
+from pydantic import PrivateAttr
+from swarmauri_base.live_models.events.SessionErrorEvent import (
+    SessionErrorEvent,
+)
+from swarmauri_base.live_models.LiveInputBase import LiveInputBase
+from swarmauri_base.live_models.LiveSessionBase import LiveSessionBase
+from swarmauri_base.ComponentBase import ComponentBase
+from swarmauri_core.transports import IWebSocketConnection
+
+from .GeminiLiveCodec import GeminiLiveCodec
+
+
+@ComponentBase.register_type(LiveSessionBase, "GeminiLiveSession")
+class GeminiLiveSession(LiveSessionBase):
+    type: Literal["GeminiLiveSession"] = "GeminiLiveSession"
+    _connection: IWebSocketConnection = PrivateAttr()
+    _codec: GeminiLiveCodec = PrivateAttr(default_factory=GeminiLiveCodec)
+    _receiver: Any = PrivateAttr(default=None)
+
+    @classmethod
+    async def create(cls, connection, setup: dict) -> "GeminiLiveSession":
+        session = cls()
+        session._connection = connection
+        await connection.asend(json.dumps(setup))
+        session.mark_open()
+        session._receiver = asyncio.create_task(session._receive_loop())
+        return session
+
+    async def _receive_loop(self) -> None:
+        try:
+            async for message in self._connection.aiter_messages():
+                for event in self._codec.decode(message):
+                    await self.emit(event)
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            await self.emit(SessionErrorEvent(message=str(exc)))
+            await self.fail(exc)
+        else:
+            await self.finish()
+
+    async def _asend_impl(self, input_data: LiveInputBase) -> None:
+        await self._connection.asend(self._codec.encode(input_data))
+
+    async def _aclose_impl(self) -> None:
+        if (
+            self._receiver is not None
+            and self._receiver is not asyncio.current_task()
+        ):
+            self._receiver.cancel()
+            with suppress(asyncio.CancelledError):
+                await self._receiver
+        await self._connection.aclose()

swarmauri_live_model_gemini/GeminiLiveToolAdapter.py

@@ -0,0 +1,12 @@
+from swarmauri_base.live_models.inputs.ToolResultInput import ToolResultInput
+
+
+class GeminiLiveToolAdapter:
+    def result(
+        self, call_id: str, name: str, value: object
+    ) -> ToolResultInput:
+        return ToolResultInput(
+            call_id=call_id,
+            correlation_id=name,
+            payload=value,
+        )

swarmauri_live_model_gemini/__init__.py

@@ -0,0 +1,13 @@
+from .GeminiLiveCodec import GeminiLiveCodec
+from .GeminiLiveConfigMapper import GeminiLiveConfigMapper
+from .GeminiLiveModel import GeminiLiveModel
+from .GeminiLiveSession import GeminiLiveSession
+from .GeminiLiveToolAdapter import GeminiLiveToolAdapter
+
+__all__ = [
+    "GeminiLiveCodec",
+    "GeminiLiveConfigMapper",
+    "GeminiLiveModel",
+    "GeminiLiveSession",
+    "GeminiLiveToolAdapter",
+]

swarmauri_live_model_gemini-0.1.0.dev3.dist-info/METADATA

@@ -0,0 +1,54 @@
+Metadata-Version: 2.4
+Name: swarmauri_live_model_gemini
+Version: 0.1.0.dev3
+Summary: Gemini Live bidirectional multimodal model adapter for Swarmauri.
+License: Copyright 2024 Jacob Stewart
+         
+         Licensed under the Apache License, Version 2.0 (the "License");
+         you may not use this file except in compliance with the License.
+         You may obtain a copy of the License at
+         
+             http://www.apache.org/licenses/LICENSE-2.0
+         
+         Unless required by applicable law or agreed to in writing, software
+         distributed under the License is distributed on an "AS IS" BASIS,
+         WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         See the License for the specific language governing permissions and
+         limitations under the License.
+License-File: LICENSE
+Keywords: swarmauri,gemini live,realtime,audio,multimodal,tools
+Author: Jacob Stewart
+Author-email: jacob@swarmauri.com
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 1 - Planning
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: swarmauri_base (>=0.11.0.dev5)
+Requires-Dist: swarmauri_core (>=0.11.0.dev5)
+Requires-Dist: swarmauri_provider_gemini (>=0.1.0.dev3)
+Requires-Dist: swarmauri_transport_websocket (>=0.1.0.dev4)
+Description-Content-Type: text/markdown
+
+![Swarmauri](https://github.com/swarmauri/swarmauri-sdk/blob/3d4d1cfa949399d7019ae9d8f296afba773dfb7f/assets/swarmauri.brand.theme.svg)
+
+[![PyPI](https://img.shields.io/pypi/v/swarmauri_live_model_gemini.svg)](https://pypi.org/project/swarmauri_live_model_gemini/) [![Python](https://img.shields.io/pypi/pyversions/swarmauri_live_model_gemini.svg)](https://pypi.org/project/swarmauri_live_model_gemini/) [![License](https://img.shields.io/badge/license-Apache--2.0-blue)](LICENSE) [![Downloads](https://static.pepy.tech/badge/swarmauri_live_model_gemini)](https://pepy.tech/project/swarmauri_live_model_gemini) [![Hits](https://hits.sh/github.com/swarmauri/swarmauri-sdk.svg)](https://hits.sh/github.com/swarmauri/swarmauri-sdk/)
+
+# swarmauri_live_model_gemini
+
+## Features
+
+- Text, audio, image, and video input.
+- Audio and transcript output as independent event streams.
+- Function calls, VAD/interruption events, usage, and session resumption.
+- Synchronous façade and canonical asynchronous WebSocket session.
+
+## Installation
+
+`uv add swarmauri_live_model_gemini` or `pip install swarmauri_live_model_gemini`.
+
+## Usage
+
+Create `GeminiLiveModel(provider=GeminiProvider(api_key=...))`, connect with `LiveConfigBase`, send typed inputs, and consume typed events.
+

swarmauri_live_model_gemini-0.1.0.dev3.dist-info/RECORD

@@ -0,0 +1,11 @@
+swarmauri_live_model_gemini/GeminiLiveCodec.py,sha256=Rxj7b3aKdzY3AgVVBHveI5d0ED2_MqeunZfd_SuL_Mw,5718
+swarmauri_live_model_gemini/GeminiLiveConfigMapper.py,sha256=R1XOHwpJabVU-eGY93PZo2hyRrnpGECjW34uo--veEo,977
+swarmauri_live_model_gemini/GeminiLiveModel.py,sha256=U_CadJBWvSR8MnIHMtYYyFvUfMBWoe_6lZvbH_kJuQg,1911
+swarmauri_live_model_gemini/GeminiLiveSession.py,sha256=qzN90HlqFHTz2LBcoO0Dp-q7JOcUthOUHREcHTIN8pU,2123
+swarmauri_live_model_gemini/GeminiLiveToolAdapter.py,sha256=vRSDWMoHh_QmmwVK3QprlkOBPCbbeBx6mys2nb77D_w,335
+swarmauri_live_model_gemini/__init__.py,sha256=4wmuUeEnAjBS9YjulYpObiRHA2EDCu9F9A7VWX061ng,400
+swarmauri_live_model_gemini-0.1.0.dev3.dist-info/METADATA,sha256=T3EINxqkaYwfjAa44rFXFxezJvEtBKWVJZ47_NNI0jA,2775
+swarmauri_live_model_gemini-0.1.0.dev3.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
+swarmauri_live_model_gemini-0.1.0.dev3.dist-info/entry_points.txt,sha256=XJEMZUDZtibvXvhkl2OnGKKvmt882zBkT8lwasSdK3A,85
+swarmauri_live_model_gemini-0.1.0.dev3.dist-info/licenses/LICENSE,sha256=kkwAgrDtfQ7c-Ozql2xlztDDjHt0YwGmRG7b_oUU9Jg,554
+swarmauri_live_model_gemini-0.1.0.dev3.dist-info/RECORD,,

swarmauri_live_model_gemini-0.1.0.dev3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.5.0
+Root-Is-Purelib: true
+Tag: py3-none-any

swarmauri_live_model_gemini-0.1.0.dev3.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[swarmauri.live_models]
+GeminiLiveModel=swarmauri_live_model_gemini:GeminiLiveModel
+

swarmauri_live_model_gemini-0.1.0.dev3.dist-info/licenses/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2024 Jacob Stewart
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.