livekit-plugins-spatialreal 1.3.12__py3-none-any.whl

livekit/plugins/spatialreal/__init__.py

@@ -0,0 +1,37 @@
+"""SpatialReal avatar plugin for LiveKit Agents.
+
+This plugin provides integration with SpatialReal's avatar service for
+lip-synced avatar rendering in LiveKit voice agents.
+
+See https://docs.spatialreal.com for more information.
+
+Usage:
+    from livekit.plugins.spatialreal import AvatarSession
+
+    avatar = AvatarSession()
+    await avatar.start(agent_session, room=ctx.room)
+"""
+
+from .avatar import AvatarSession, SpatialRealException
+from .version import __version__
+
+__all__ = [
+    "AvatarSession",
+    "SpatialRealException",
+    "__version__",
+]
+
+# Try to register plugin if Plugin class is available (livekit-agents >= 1.3)
+try:
+    from livekit.agents import Plugin
+
+    from .log import logger
+
+    class SpatialRealPlugin(Plugin):
+        def __init__(self) -> None:
+            super().__init__(__name__, __version__, __package__, logger)
+
+    Plugin.register_plugin(SpatialRealPlugin())
+except (ImportError, AttributeError):
+    # Plugin registration not available in older versions
+    pass

livekit/plugins/spatialreal/avatar.py

@@ -0,0 +1,278 @@
+"""
+SpatialReal Avatar integration for LiveKit Agents.
+
+This module provides AvatarSession which hooks into an AgentSession
+to route TTS audio to the SpatialReal avatar service.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from datetime import datetime, timedelta, timezone
+
+from avatarkit import (
+    AvatarSession as AvatarkitSession,
+)
+from avatarkit import (
+    LiveKitEgressConfig,
+    new_avatar_session,
+)
+from livekit.agents import AgentSession
+from livekit.agents.voice.avatar import AudioSegmentEnd, QueueAudioOutput
+
+from livekit import rtc
+
+from .log import logger
+
+__all__ = ["AvatarSession", "SpatialRealException"]
+
+DEFAULT_AVATAR_PARTICIPANT_IDENTITY = "spatialreal-avatar"
+DEFAULT_SAMPLE_RATE = 24000
+
+# Default endpoints (China)
+DEFAULT_CONSOLE_ENDPOINT = "https://console.us-west.spatialwalk.cloud/v1/console"
+DEFAULT_INGRESS_ENDPOINT = "wss://api.us-west.spatialwalk.cloud/v2/driveningress"
+
+
+class SpatialRealException(Exception):
+    """Exception raised for SpatialReal-related errors."""
+
+    pass
+
+
+class AvatarSession:
+    """
+    This connects to SpatialReal's avatar service and routes TTS audio
+    from the agent to the avatar for lip-synced rendering. The avatar
+    service joins the LiveKit room and publishes synchronized video + audio.
+
+    Args:
+        api_key: SpatialReal API key. Falls back to SPATIALREAL_API_KEY env var.
+        app_id: SpatialReal application ID. Falls back to SPATIALREAL_APP_ID env var.
+        avatar_id: Avatar ID to use. Falls back to SPATIALREAL_AVATAR_ID env var.
+        console_endpoint_url: Console endpoint URL. Falls back to
+            SPATIALREAL_CONSOLE_ENDPOINT env var or default.
+        ingress_endpoint_url: Ingress endpoint URL. Falls back to
+            SPATIALREAL_INGRESS_ENDPOINT env var or default.
+        avatar_participant_identity: LiveKit identity for the avatar participant.
+
+    Usage:
+        avatar = AvatarSession()
+        await avatar.start(session, room=ctx.room)
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        app_id: str | None = None,
+        avatar_id: str | None = None,
+        console_endpoint_url: str | None = None,
+        ingress_endpoint_url: str | None = None,
+        avatar_participant_identity: str | None = None,
+    ) -> None:
+        # Resolve API key
+        self._api_key = api_key or os.getenv("SPATIALREAL_API_KEY")
+        if not self._api_key:
+            raise SpatialRealException(
+                "api_key must be provided or SPATIALREAL_API_KEY environment variable must be set"
+            )
+
+        # Resolve app ID
+        self._app_id = app_id or os.getenv("SPATIALREAL_APP_ID")
+        if not self._app_id:
+            raise SpatialRealException("app_id must be provided or SPATIALREAL_APP_ID environment variable must be set")
+
+        # Resolve avatar ID
+        self._avatar_id = avatar_id or os.getenv("SPATIALREAL_AVATAR_ID")
+        if not self._avatar_id:
+            raise SpatialRealException(
+                "avatar_id must be provided or SPATIALREAL_AVATAR_ID environment variable must be set"
+            )
+
+        # Resolve endpoints
+        self._console_endpoint_url = (
+            console_endpoint_url or os.getenv("SPATIALREAL_CONSOLE_ENDPOINT") or DEFAULT_CONSOLE_ENDPOINT
+        )
+        self._ingress_endpoint_url = (
+            ingress_endpoint_url or os.getenv("SPATIALREAL_INGRESS_ENDPOINT") or DEFAULT_INGRESS_ENDPOINT
+        )
+
+        # Avatar participant configuration
+        self._avatar_participant_identity = avatar_participant_identity or DEFAULT_AVATAR_PARTICIPANT_IDENTITY
+
+        # Internal state
+        self._avatarkit_session: AvatarkitSession | None = None
+        self._agent_session: AgentSession | None = None
+        self._audio_buffer: QueueAudioOutput | None = None
+        self._main_task: asyncio.Task | None = None
+        self._initialized = False
+
+    async def start(
+        self,
+        agent_session: AgentSession,
+        room: rtc.Room,
+        *,
+        livekit_url: str | None = None,
+        livekit_api_key: str | None = None,
+        livekit_api_secret: str | None = None,
+    ) -> None:
+        """
+        Start the avatar session and hook into the agent session.
+
+        Args:
+            agent_session: The AgentSession to hook into for TTS audio.
+            room: The LiveKit room for egress configuration.
+            livekit_url: LiveKit server URL. Falls back to LIVEKIT_URL env var.
+            livekit_api_key: LiveKit API key. Falls back to LIVEKIT_API_KEY env var.
+            livekit_api_secret: LiveKit API secret. Falls back to LIVEKIT_API_SECRET env var.
+        """
+        if self._initialized:
+            logger.warning("Avatar session already initialized")
+            return
+
+        self._agent_session = agent_session
+
+        # Resolve LiveKit credentials
+        lk_url = livekit_url or os.getenv("LIVEKIT_URL")
+        lk_api_key = livekit_api_key or os.getenv("LIVEKIT_API_KEY")
+        lk_api_secret = livekit_api_secret or os.getenv("LIVEKIT_API_SECRET")
+
+        if not lk_url or not lk_api_key or not lk_api_secret:
+            raise SpatialRealException(
+                "livekit_url, livekit_api_key, and livekit_api_secret must be provided "
+                "or LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET environment variables must be set"
+            )
+
+        room_name = room.name
+        logger.info(f"Initializing SpatialReal avatar session for room: {room_name}")
+        logger.debug(f"Console endpoint: {self._console_endpoint_url}")
+        logger.debug(f"Ingress endpoint: {self._ingress_endpoint_url}")
+
+        # Create LiveKit egress configuration for the avatar to join the room
+        livekit_egress = LiveKitEgressConfig(
+            url=lk_url,
+            api_key=lk_api_key,
+            api_secret=lk_api_secret,
+            room_name=room_name,
+            publisher_id=self._avatar_participant_identity,
+        )
+
+        # Create avatar session with LiveKit egress mode
+        self._avatarkit_session = new_avatar_session(
+            api_key=self._api_key,
+            app_id=self._app_id,
+            avatar_id=self._avatar_id,
+            console_endpoint_url=self._console_endpoint_url,
+            ingress_endpoint_url=self._ingress_endpoint_url,
+            expire_at=datetime.now(timezone.utc) + timedelta(hours=1),
+            livekit_egress=livekit_egress,
+        )
+
+        # Initialize and start the avatar session
+        await self._avatarkit_session.init()
+        await self._avatarkit_session.start()
+        logger.info("SpatialReal avatar session connected")
+
+        # Create audio buffer using livekit-agents' QueueAudioOutput
+        sample_rate = agent_session.tts.sample_rate if agent_session.tts else DEFAULT_SAMPLE_RATE
+        self._audio_buffer = QueueAudioOutput(sample_rate=sample_rate)
+
+        # Hook into agent session's audio output
+        agent_session.output.audio = self._audio_buffer
+
+        # Start the audio buffer
+        await self._audio_buffer.start()
+
+        # Register for clear_buffer events (interruptions)
+        @self._audio_buffer.on("clear_buffer")
+        def on_clear_buffer() -> None:
+            asyncio.create_task(self._handle_interrupt())
+
+        # Start the main task that forwards audio to avatar
+        self._main_task = asyncio.create_task(self._run_main_task())
+
+        self._initialized = True
+        logger.info("Avatar audio output attached to agent session")
+
+        # Register cleanup on session close
+        @agent_session.on("close")
+        def on_session_close() -> None:
+            asyncio.create_task(self.aclose())
+
+    async def _run_main_task(self) -> None:
+        """Main task that forwards audio from the buffer to the avatar service."""
+        if not self._audio_buffer or not self._avatarkit_session:
+            return
+
+        try:
+            frame_count = 0
+            async for item in self._audio_buffer:
+                if isinstance(item, rtc.AudioFrame):
+                    # Convert AudioFrame to bytes and send to avatar
+                    audio_bytes = bytes(item.data)
+                    frame_count += 1
+
+                    if frame_count == 1:
+                        logger.debug("Avatar: First audio frame received")
+
+                    await self._avatarkit_session.send_audio(
+                        audio=audio_bytes,
+                        end=False,
+                    )
+
+                elif isinstance(item, AudioSegmentEnd):
+                    # End of audio segment - signal completion to avatar
+                    logger.debug(f"Avatar: Segment end, sent {frame_count} frames")
+                    await self._avatarkit_session.send_audio(
+                        audio=b"",
+                        end=True,
+                    )
+
+                    # Notify the buffer that playback is finished
+                    self._audio_buffer.notify_playback_finished(
+                        playback_position=0.0,
+                        interrupted=False,
+                    )
+                    frame_count = 0
+
+        except asyncio.CancelledError:
+            logger.debug("Avatar main task cancelled")
+        except Exception as e:
+            logger.error(f"Error in avatar main task: {e}")
+
+    async def _handle_interrupt(self) -> None:
+        """Handle interruption - stop avatar's current audio processing."""
+        if not self._avatarkit_session:
+            return
+
+        try:
+            interrupted_id = await self._avatarkit_session.interrupt()
+            logger.debug(f"Avatar interrupted, request_id={interrupted_id}")
+        except Exception as e:
+            logger.warning(f"Failed to interrupt avatar: {e}")
+
+    async def aclose(self) -> None:
+        """Clean up avatar session resources."""
+        if self._main_task:
+            self._main_task.cancel()
+            try:
+                await self._main_task
+            except asyncio.CancelledError:
+                pass
+            self._main_task = None
+
+        if self._audio_buffer:
+            await self._audio_buffer.aclose()
+            self._audio_buffer = None
+
+        if self._avatarkit_session:
+            try:
+                await self._avatarkit_session.close()
+                logger.info("Avatar session closed")
+            except Exception as e:
+                logger.warning(f"Error closing avatar session: {e}")
+            finally:
+                self._avatarkit_session = None
+                self._initialized = False

livekit/plugins/spatialreal/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.spatialreal")

livekit/plugins/spatialreal/version.py

@@ -0,0 +1 @@
+__version__ = "1.3.12"

livekit_plugins_spatialreal-1.3.12.dist-info/METADATA

@@ -0,0 +1,135 @@
+Metadata-Version: 2.4
+Name: livekit-plugins-spatialreal
+Version: 1.3.12
+Summary: Agent Framework plugin for SpatialReal Avatar
+Project-URL: Documentation, https://docs.spatialreal.com
+Project-URL: Website, https://spatialreal.com/
+Project-URL: Source, https://github.com/spatialreal/livekit-plugins-spatialreal
+Author-email: 3DRX <3drxkjy@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,audio,avatar,livekit,realtime,spatialreal,video,voice,webrtc
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9.0
+Requires-Dist: avatarkit>=0.1.3
+Requires-Dist: livekit-agents>=1.2.9
+Description-Content-Type: text/markdown
+
+# LiveKit Agents Plugin for SpatialReal Avatar
+
+This plugin provides integration with [SpatialReal](https://spatialreal.com)'s avatar service for lip-synced avatar rendering in LiveKit voice agents.
+
+## Installation
+
+```bash
+pip install livekit-plugins-spatialreal
+```
+
+Or install from source:
+
+```bash
+pip install -e .
+```
+
+## Configuration
+
+Set the following environment variables:
+
+```bash
+# Required
+SPATIALREAL_API_KEY=your-api-key
+SPATIALREAL_APP_ID=your-app-id
+SPATIALREAL_AVATAR_ID=your-avatar-id
+
+# Optional
+SPATIALREAL_CONSOLE_ENDPOINT=
+SPATIALREAL_INGRESS_ENDPOINT=
+
+# LiveKit credentials
+LIVEKIT_URL=
+LIVEKIT_API_KEY=
+LIVEKIT_API_SECRET=
+```
+
+## Usage
+
+```python
+from livekit.agents import Agent, AgentSession, JobContext, cli, WorkerOptions
+from livekit.plugins import spatialreal
+
+class VoiceAssistant(Agent):
+    def __init__(self):
+        super().__init__(
+            instructions="You are a helpful voice assistant."
+        )
+
+async def entrypoint(ctx: JobContext):
+    await ctx.connect()
+
+    # Configure your pipeline components (VAD, STT, LLM, TTS)
+    session = AgentSession(
+        vad=vad,
+        stt=stt,
+        llm=llm,
+        tts=tts,
+    )
+
+    # Initialize and start the avatar session
+    avatar = spatialreal.AvatarSession()
+    await avatar.start(session, room=ctx.room)
+
+    # Start the agent session
+    await session.start(
+        agent=VoiceAssistant(),
+        room=ctx.room,
+    )
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))
+```
+
+## API Reference
+
+### `AvatarSession`
+
+Main class for integrating SpatialReal avatars with LiveKit agents.
+
+#### Constructor Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `api_key` | `str` | SpatialReal API key (or set `SPATIALREAL_API_KEY`) |
+| `app_id` | `str` | SpatialReal application ID (or set `SPATIALREAL_APP_ID`) |
+| `avatar_id` | `str` | Avatar ID to use (or set `SPATIALREAL_AVATAR_ID`) |
+| `console_endpoint_url` | `str` | Custom console endpoint URL |
+| `ingress_endpoint_url` | `str` | Custom ingress endpoint URL |
+| `avatar_participant_identity` | `str` | LiveKit identity for avatar participant |
+
+#### Methods
+
+- `start(agent_session, room, *, livekit_url, livekit_api_key, livekit_api_secret)`: Start the avatar session and hook into the agent's audio output.
+- `aclose()`: Clean up avatar session resources.
+
+### `SpatialRealException`
+
+Exception raised for SpatialReal-related errors.
+
+## How It Works
+
+1. The plugin intercepts TTS audio output from the agent session
+2. Audio frames are forwarded to SpatialReal's avatar service
+3. SpatialReal generates lip-synced video and audio
+4. The avatar joins the LiveKit room and publishes the synchronized streams
+
+## License
+
+MIT

livekit_plugins_spatialreal-1.3.12.dist-info/RECORD

@@ -0,0 +1,9 @@
+livekit/plugins/spatialreal/__init__.py,sha256=wp1ICd36yzIT7zK1EVy7NJUfCNC-BsLWmcNK3Ww8Ds0,1008
+livekit/plugins/spatialreal/avatar.py,sha256=GStcE-40MJi1pLRmQhom-cPp-_a0p_8sUtVLWnmTCiM,10510
+livekit/plugins/spatialreal/log.py,sha256=Gi7neqj1wBakfvLRZY98c8D9FL8UJn-BDExJx0hTwwc,74
+livekit/plugins/spatialreal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/spatialreal/version.py,sha256=2Rgd7p7gALBiXufv4uSeaC9anR_muEfi4m5NMAgfMHI,23
+livekit_plugins_spatialreal-1.3.12.dist-info/METADATA,sha256=PJs0t02iuBJJni6wW6kfUTFf4SsVsM9fcqbM9erPXMM,3867
+livekit_plugins_spatialreal-1.3.12.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+livekit_plugins_spatialreal-1.3.12.dist-info/licenses/LICENSE,sha256=J0AyxvqWXwDXalRKg9CQzVxw0-xT8tu2h-316a-9dY8,1068
+livekit_plugins_spatialreal-1.3.12.dist-info/RECORD,,

livekit_plugins_spatialreal-1.3.12.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

livekit_plugins_spatialreal-1.3.12.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 spatialwalk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.