livekit-plugins-spatius 1.4.5__tar.gz

livekit_plugins_spatius-1.4.5/.gitignore

@@ -0,0 +1,216 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

livekit_plugins_spatius-1.4.5/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Spatius AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

livekit_plugins_spatius-1.4.5/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: livekit-plugins-spatius
+Version: 1.4.5
+Summary: Agent Framework plugin for Spatius Avatar
+Project-URL: Documentation, https://docs.spatius.ai
+Project-URL: Website, https://www.spatius.ai/
+Project-URL: Source, https://github.com/spatius-ai/livekit-plugins-spatius
+Author-email: 3DRX <3drxkjy@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,audio,avatar,livekit,realtime,spatius,video,voice,webrtc
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10.0
+Requires-Dist: livekit-agents>=1.2.9
+Requires-Dist: spatius>=1.0.0
+Description-Content-Type: text/markdown
+
+# LiveKit Agents Plugin for Spatius Avatar
+
+LiveKit Agents plugin for [Spatius](https://www.spatius.ai) avatar sessions. It forwards TTS audio from a LiveKit agent session to Spatius and lets the avatar publish synchronized audio/video back into the same room.
+
+## Installation
+
+```bash
+pip install livekit-plugins-spatius
+```
+
+## Quick Start
+
+Set credentials:
+
+```bash
+export SPATIUS_API_KEY=your-api-key
+export SPATIUS_APP_ID=your-app-id
+export SPATIUS_AVATAR_ID=your-avatar-id
+
+export LIVEKIT_URL=wss://your-livekit-host
+export LIVEKIT_API_KEY=your-livekit-api-key
+export LIVEKIT_API_SECRET=your-livekit-api-secret
+```
+
+Use plugin in your LiveKit agent:
+
+```python
+from livekit.agents import Agent, AgentSession, JobContext, WorkerOptions, cli
+from livekit.plugins import spatius
+
+
+class VoiceAssistant(Agent):
+    def __init__(self) -> None:
+        super().__init__(instructions="You are a helpful voice assistant.")
+
+
+async def entrypoint(ctx: JobContext) -> None:
+    await ctx.connect()
+
+    session = AgentSession(
+        vad=vad,
+        stt=stt,
+        llm=llm,
+        tts=tts,
+    )
+
+    avatar = spatius.AvatarSession()
+    await avatar.start(session, room=ctx.room)
+
+    await session.start(agent=VoiceAssistant(), room=ctx.room)
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))
+```
+
+`AvatarSession` defaults to `region="us-west"` and composes Spatius endpoints from that region. To use another region:
+
+```python
+avatar = spatius.AvatarSession(region="us-east")
+```
+
+Explicit endpoint URLs still override region:
+
+```python
+avatar = spatius.AvatarSession(
+    console_endpoint_url="https://console.example.com/v1/console",
+    ingress_endpoint_url="wss://api.example.com/v2/driveningress",
+)
+```
+
+For detailed usage, see [Spatius docs](https://docs.spatius.ai).
+
+## License
+
+MIT

livekit_plugins_spatius-1.4.5/README.md

@@ -0,0 +1,76 @@
+# LiveKit Agents Plugin for Spatius Avatar
+
+LiveKit Agents plugin for [Spatius](https://www.spatius.ai) avatar sessions. It forwards TTS audio from a LiveKit agent session to Spatius and lets the avatar publish synchronized audio/video back into the same room.
+
+## Installation
+
+```bash
+pip install livekit-plugins-spatius
+```
+
+## Quick Start
+
+Set credentials:
+
+```bash
+export SPATIUS_API_KEY=your-api-key
+export SPATIUS_APP_ID=your-app-id
+export SPATIUS_AVATAR_ID=your-avatar-id
+
+export LIVEKIT_URL=wss://your-livekit-host
+export LIVEKIT_API_KEY=your-livekit-api-key
+export LIVEKIT_API_SECRET=your-livekit-api-secret
+```
+
+Use plugin in your LiveKit agent:
+
+```python
+from livekit.agents import Agent, AgentSession, JobContext, WorkerOptions, cli
+from livekit.plugins import spatius
+
+
+class VoiceAssistant(Agent):
+    def __init__(self) -> None:
+        super().__init__(instructions="You are a helpful voice assistant.")
+
+
+async def entrypoint(ctx: JobContext) -> None:
+    await ctx.connect()
+
+    session = AgentSession(
+        vad=vad,
+        stt=stt,
+        llm=llm,
+        tts=tts,
+    )
+
+    avatar = spatius.AvatarSession()
+    await avatar.start(session, room=ctx.room)
+
+    await session.start(agent=VoiceAssistant(), room=ctx.room)
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))
+```
+
+`AvatarSession` defaults to `region="us-west"` and composes Spatius endpoints from that region. To use another region:
+
+```python
+avatar = spatius.AvatarSession(region="us-east")
+```
+
+Explicit endpoint URLs still override region:
+
+```python
+avatar = spatius.AvatarSession(
+    console_endpoint_url="https://console.example.com/v1/console",
+    ingress_endpoint_url="wss://api.example.com/v2/driveningress",
+)
+```
+
+For detailed usage, see [Spatius docs](https://docs.spatius.ai).
+
+## License
+
+MIT

livekit_plugins_spatius-1.4.5/livekit/plugins/spatius/__init__.py

@@ -0,0 +1,45 @@
+"""Spatius avatar plugin for LiveKit Agents.
+
+This plugin provides integration with Spatius's avatar service for
+lip-synced avatar rendering in LiveKit voice agents.
+
+See https://docs.spatius.ai for more information.
+
+Usage:
+    from livekit.plugins.spatius import AvatarSession
+
+    avatar = AvatarSession()
+    await avatar.start(agent_session, room=ctx.room)
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .avatar import AvatarSession, SpatiusException
+
+try:
+    __version__ = version("livekit-plugins-spatius")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+
+__all__ = [
+    "AvatarSession",
+    "SpatiusException",
+    "__version__",
+]
+
+# Try to register plugin if Plugin class is available (livekit-agents >= 1.3)
+try:
+    from livekit.agents import Plugin
+
+    from .log import logger
+
+    class SpatiusPlugin(Plugin):
+        """LiveKit plugin registration shim for Spatius avatar support."""
+
+        def __init__(self) -> None:
+            super().__init__(__name__, __version__, __package__, logger)
+
+    Plugin.register_plugin(SpatiusPlugin())
+except (ImportError, AttributeError):
+    # Plugin registration not available in older versions
+    pass

livekit_plugins_spatius-1.4.5/livekit/plugins/spatius/avatar.py

@@ -0,0 +1,700 @@
+"""
+Spatius Avatar integration for LiveKit Agents.
+
+This module provides AvatarSession which hooks into an AgentSession
+to route TTS audio to the Spatius avatar service.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from collections import deque
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from livekit.agents import AgentSession, UserStateChangedEvent, get_job_context
+from livekit.agents.voice.avatar import AudioSegmentEnd, QueueAudioOutput
+from livekit.agents.voice.room_io import ATTRIBUTE_PUBLISH_ON_BEHALF
+
+from livekit import rtc
+from spatius import AvatarSession as SpatiusSession
+from spatius import LiveKitEgressConfig, new_avatar_session
+from spatius.proto.generated import message_pb2 as _message_pb2
+
+from .log import logger
+
+message_pb2: Any = _message_pb2
+
+__all__ = ["AvatarSession", "SpatiusException"]
+
+DEFAULT_REGION = "us-west"
+DEFAULT_AVATAR_PARTICIPANT_IDENTITY = "spatius-avatar"
+DEFAULT_SAMPLE_RATE = 24000
+MIN_COMPLETION_TIMEOUT_SECONDS = 3.0
+COMPLETION_TIMEOUT_BUFFER_SECONDS = 2.0
+ACTIVE_SEGMENT_IDLE_END_SECONDS = 1.0
+DEFAULT_SESSION_TTL = timedelta(hours=1)
+LIVEKIT_AVATAR_PUBLISH_SOURCES = ["camera", "microphone"]
+
+
+class SpatiusException(Exception):
+    """Exception raised for Spatius-related errors."""
+
+    pass
+
+
+@dataclass
+class _SegmentState:
+    req_id: str
+    pushed_duration: float = 0.0
+    first_frame_at: float | None = None
+    completion_timeout_task: asyncio.Task[None] | None = None
+
+
+class AvatarSession:
+    """
+    LiveKit Agents adapter for Spatius avatars.
+
+    This connects to Spatius's avatar service and routes TTS audio
+    from the agent to the avatar for lip-synced rendering. The avatar
+    service joins the LiveKit room and publishes synchronized video + audio.
+
+    Args:
+        api_key: Spatius API key. Falls back to ``SPATIUS_API_KEY``.
+        app_id: Spatius application ID. Falls back to ``SPATIUS_APP_ID``.
+        avatar_id: Avatar ID to use. Falls back to ``SPATIUS_AVATAR_ID``.
+        region: Spatius region used to compose endpoint URLs when explicit
+            endpoints are omitted. Falls back to ``SPATIUS_REGION`` or
+            ``"us-west"``.
+        console_endpoint_url: Optional explicit Console API URL. Falls back to
+            ``SPATIUS_CONSOLE_ENDPOINT`` and overrides ``region``.
+        ingress_endpoint_url: Optional explicit ingress WebSocket URL. Falls back
+            to ``SPATIUS_INGRESS_ENDPOINT`` and overrides ``region``.
+        avatar_participant_identity: LiveKit identity for the avatar participant.
+        idle_timeout_seconds: Idle timeout in seconds for the egress connection.
+            A value of 0 uses server defaults.
+        sample_rate: Optional audio sample rate override for avatar audio.
+            Falls back to agent_session.tts.sample_rate or a default value.
+
+    Usage:
+        avatar = AvatarSession()
+        await avatar.start(session, room=ctx.room)
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        app_id: str | None = None,
+        avatar_id: str | None = None,
+        region: str | None = None,
+        console_endpoint_url: str | None = None,
+        ingress_endpoint_url: str | None = None,
+        avatar_participant_identity: str | None = None,
+        idle_timeout_seconds: int = 0,
+        sample_rate: int | None = None,
+    ) -> None:
+        # Resolve Spatius credentials and routing configuration.
+        self._api_key = api_key or os.getenv("SPATIUS_API_KEY")
+        if not self._api_key:
+            raise SpatiusException("api_key must be provided or SPATIUS_API_KEY environment variable must be set")
+
+        self._app_id = app_id or os.getenv("SPATIUS_APP_ID")
+        if not self._app_id:
+            raise SpatiusException("app_id must be provided or SPATIUS_APP_ID environment variable must be set")
+
+        self._avatar_id = avatar_id or os.getenv("SPATIUS_AVATAR_ID")
+        if not self._avatar_id:
+            raise SpatiusException("avatar_id must be provided or SPATIUS_AVATAR_ID environment variable must be set")
+
+        self._region = region or os.getenv("SPATIUS_REGION") or DEFAULT_REGION
+        self._console_endpoint_url = console_endpoint_url or os.getenv("SPATIUS_CONSOLE_ENDPOINT") or ""
+        self._ingress_endpoint_url = ingress_endpoint_url or os.getenv("SPATIUS_INGRESS_ENDPOINT") or ""
+
+        # Avatar participant configuration
+        self._avatar_participant_identity = avatar_participant_identity or DEFAULT_AVATAR_PARTICIPANT_IDENTITY
+
+        if idle_timeout_seconds < 0:
+            raise SpatiusException("idle_timeout_seconds must be greater than or equal to 0")
+        self._idle_timeout_seconds = idle_timeout_seconds
+
+        if sample_rate is not None and sample_rate <= 0:
+            raise SpatiusException("sample_rate must be greater than 0")
+        self._sample_rate = sample_rate
+
+        # Internal state
+        self._spatius_session: SpatiusSession | None = None
+        self._agent_session: AgentSession | None = None
+        self._audio_buffer: QueueAudioOutput | None = None
+        self._original_audio_output: Any | None = None
+        self._audio_output_attached = False
+        self._main_task: asyncio.Task | None = None
+        self._initialized = False
+        self._segments: dict[str, _SegmentState] = {}
+        self._pending_segment_ids: deque[str] = deque()
+        self._active_req_id: str | None = None
+        self._active_segment_idle_end_task: asyncio.Task[None] | None = None
+        self._segment_finalize_lock = asyncio.Lock()
+
+    async def start(
+        self,
+        agent_session: AgentSession,
+        room: rtc.Room,
+        *,
+        livekit_url: str | None = None,
+        livekit_api_key: str | None = None,
+        livekit_api_secret: str | None = None,
+    ) -> None:
+        """
+        Start the avatar session and hook into the agent session.
+
+        Args:
+            agent_session: The AgentSession to hook into for TTS audio.
+            room: The LiveKit room for egress configuration.
+            livekit_url: LiveKit server URL. Falls back to LIVEKIT_URL env var.
+            livekit_api_key: LiveKit API key. Falls back to LIVEKIT_API_KEY env var.
+            livekit_api_secret: LiveKit API secret. Falls back to LIVEKIT_API_SECRET env var.
+        """
+        if self._initialized:
+            logger.warning("Avatar session already initialized")
+            return
+
+        # Resolve LiveKit credentials
+        lk_url = livekit_url or os.getenv("LIVEKIT_URL")
+        lk_api_key = livekit_api_key or os.getenv("LIVEKIT_API_KEY")
+        lk_api_secret = livekit_api_secret or os.getenv("LIVEKIT_API_SECRET")
+
+        if not lk_url:
+            raise SpatiusException("livekit_url must be provided or LIVEKIT_URL environment variable must be set")
+
+        if not lk_api_key or not lk_api_secret:
+            raise SpatiusException(
+                "livekit_api_key and livekit_api_secret must be provided "
+                "or LIVEKIT_API_KEY and LIVEKIT_API_SECRET environment variables must be set"
+            )
+
+        room_name = room.name
+        agent_participant_identity = self._resolve_local_participant_identity(room)
+        logger.info(f"Initializing Spatius avatar session for room: {room_name}")
+        logger.debug(f"Region: {self._region}")
+        if self._console_endpoint_url:
+            logger.debug(f"Console endpoint override: {self._console_endpoint_url}")
+        if self._ingress_endpoint_url:
+            logger.debug(f"Ingress endpoint override: {self._ingress_endpoint_url}")
+
+        egress_attributes = {ATTRIBUTE_PUBLISH_ON_BEHALF: agent_participant_identity}
+        session_expire_at = datetime.now(timezone.utc) + DEFAULT_SESSION_TTL
+        resolved_lk_api_token = self._generate_livekit_api_token(
+            room_name=room_name,
+            livekit_api_key=lk_api_key,
+            livekit_api_secret=lk_api_secret,
+            publisher_identity=self._avatar_participant_identity,
+            publisher_attributes=egress_attributes,
+            ttl=DEFAULT_SESSION_TTL,
+        )
+
+        # Create LiveKit egress configuration for the avatar to join the room
+        livekit_egress_kwargs: dict[str, Any] = {
+            "url": lk_url,
+            "api_token": resolved_lk_api_token,
+            "room_name": room_name,
+            "publisher_id": self._avatar_participant_identity,
+            "extra_attributes": egress_attributes,
+            "idle_timeout": self._idle_timeout_seconds,
+        }
+        livekit_egress = LiveKitEgressConfig(**livekit_egress_kwargs)
+
+        resolved_sample_rate = self._sample_rate
+        if resolved_sample_rate is None:
+            resolved_sample_rate = agent_session.tts.sample_rate if agent_session.tts else DEFAULT_SAMPLE_RATE
+        if resolved_sample_rate <= 0:
+            raise SpatiusException("sample_rate must be greater than 0")
+
+        self._agent_session = agent_session
+        self._original_audio_output = agent_session.output.audio
+
+        try:
+            # Create avatar session with LiveKit egress mode
+            self._spatius_session = new_avatar_session(
+                api_key=self._api_key,
+                app_id=self._app_id,
+                avatar_id=self._avatar_id,
+                region=self._region,
+                console_endpoint_url=self._console_endpoint_url,
+                ingress_endpoint_url=self._ingress_endpoint_url,
+                expire_at=session_expire_at,
+                livekit_egress=livekit_egress,
+                sample_rate=resolved_sample_rate,
+                transport_frames=self._on_transport_frame,
+            )
+
+            # Initialize and start the avatar session
+            await self._spatius_session.init()
+            await self._spatius_session.start()
+            logger.info("Spatius avatar session connected")
+
+            # Create audio buffer using livekit-agents' QueueAudioOutput
+            self._audio_buffer = QueueAudioOutput(sample_rate=resolved_sample_rate)
+
+            # Hook into agent session's audio output
+            agent_session.output.audio = self._audio_buffer
+            self._audio_output_attached = True
+
+            # Start the audio buffer
+            await self._audio_buffer.start()
+
+            # Register for clear_buffer events (interruptions)
+            self._audio_buffer.on("clear_buffer", self._on_clear_buffer)  # type: ignore[arg-type]
+
+            # Register for user_state_changed events (interrupt on user speaking)
+            @agent_session.on("user_state_changed")
+            def on_user_state_changed(ev: UserStateChangedEvent) -> None:
+                if ev.new_state == "speaking":
+                    asyncio.create_task(self._handle_interrupt())
+
+            # Start the main task that forwards audio to avatar
+            self._main_task = asyncio.create_task(self._run_main_task())
+
+            self._initialized = True
+            logger.info("Avatar audio output attached to agent session")
+
+            # Register cleanup on session close
+            @agent_session.on("close")
+            def on_session_close() -> None:
+                asyncio.create_task(self.aclose())
+
+        except asyncio.CancelledError:
+            await self.aclose()
+            raise
+        except Exception as e:
+            logger.debug("Spatius avatar session startup failed", exc_info=True)
+            await self.aclose()
+            raise SpatiusException(
+                self._build_start_error_message(
+                    error=e,
+                    room_name=room_name,
+                    sample_rate=resolved_sample_rate,
+                )
+            ) from None
+
+    def _build_start_error_message(
+        self,
+        *,
+        error: Exception,
+        room_name: str,
+        sample_rate: int,
+    ) -> str:
+        reason = self._format_error_reason(error)
+        return (
+            "Failed to start Spatius avatar session. "
+            "Check Spatius credentials, LiveKit room auth/token configuration, "
+            "region/endpoint URLs, and outbound network access. "
+            f"room={room_name}, avatar_id={self._avatar_id}, region={self._region}, "
+            f"sample_rate={sample_rate}. Reason: {reason}"
+        )
+
+    @staticmethod
+    def _generate_livekit_api_token(
+        *,
+        room_name: str,
+        livekit_api_key: str,
+        livekit_api_secret: str,
+        publisher_identity: str,
+        publisher_attributes: dict[str, str],
+        ttl: timedelta,
+    ) -> str:
+        try:
+            from livekit import api as livekit_api
+        except ImportError as exc:
+            raise SpatiusException(
+                "livekit-api must be installed to generate LiveKit access tokens for avatar egress"
+            ) from exc
+
+        try:
+            return (
+                livekit_api.AccessToken(livekit_api_key, livekit_api_secret)
+                .with_kind("agent")
+                .with_identity(publisher_identity)
+                .with_name(publisher_identity)
+                .with_ttl(ttl)
+                .with_attributes(publisher_attributes)
+                .with_grants(
+                    livekit_api.VideoGrants(
+                        room_join=True,
+                        room=room_name,
+                        can_subscribe=False,
+                        can_publish_data=False,
+                        can_publish_sources=LIVEKIT_AVATAR_PUBLISH_SOURCES,
+                    )
+                )
+                .to_jwt()
+            )
+        except Exception as exc:
+            raise SpatiusException(
+                "Failed to generate LiveKit access token for avatar worker. "
+                f"room={room_name}, publisher_identity={publisher_identity}. "
+                f"Reason: {AvatarSession._format_error_reason(exc)}"
+            ) from exc
+
+    @staticmethod
+    def _resolve_local_participant_identity(room: rtc.Room) -> str:
+        try:
+            return get_job_context().token_claims().identity
+        except RuntimeError as exc:
+            if not room.isconnected():
+                raise SpatiusException("failed to get local participant identity") from exc
+            return room.local_participant.identity
+
+    @staticmethod
+    def _format_error_reason(error: BaseException) -> str:
+        root_error = error
+        seen_errors: set[int] = set()
+
+        while id(root_error) not in seen_errors:
+            seen_errors.add(id(root_error))
+            next_error = root_error.__cause__ or (None if root_error.__suppress_context__ else root_error.__context__)
+            if next_error is None:
+                break
+            root_error = next_error
+
+        message = str(root_error) or str(error)
+        if message:
+            return f"{type(root_error).__name__}: {message}"
+
+        return type(root_error).__name__
+
+    async def _run_main_task(self) -> None:
+        """Main task that forwards audio from the buffer to the avatar service."""
+        if not self._audio_buffer or not self._spatius_session:
+            return
+
+        try:
+            async for item in self._audio_buffer:
+                if isinstance(item, rtc.AudioFrame):
+                    # Convert AudioFrame to bytes and send to avatar
+                    audio_bytes = bytes(item.data)
+
+                    previous_req_id = self._active_req_id
+
+                    req_id = await self._spatius_session.send_audio(
+                        audio=audio_bytes,
+                        end=False,
+                    )
+
+                    if previous_req_id and previous_req_id != req_id:
+                        logger.warning(
+                            "Avatar: request ID changed while streaming audio "
+                            f"(previous={previous_req_id}, current={req_id})"
+                        )
+                        previous_segment = self._segments.get(previous_req_id)
+                        if previous_segment is not None:
+                            self._mark_segment_waiting_for_completion(previous_segment)
+
+                    segment = self._segments.get(req_id)
+                    if segment is None:
+                        segment = _SegmentState(req_id=req_id)
+                        self._segments[req_id] = segment
+
+                    if segment.first_frame_at is None:
+                        segment.first_frame_at = time.time()
+                        logger.debug(f"Avatar: First audio frame received (request_id={req_id})")
+
+                    segment.pushed_duration += item.duration
+                    self._active_req_id = req_id
+                    self._schedule_active_segment_idle_end()
+
+                elif isinstance(item, AudioSegmentEnd):
+                    # End of audio segment - signal completion to avatar
+                    if not await self._finalize_active_segment(source="segment_end"):
+                        logger.debug("Avatar: Segment end received without an active request")
+
+        except asyncio.CancelledError:
+            logger.debug("Avatar main task cancelled")
+        except Exception as e:
+            logger.error(f"Error in avatar main task: {e}")
+
+    def _cancel_active_segment_idle_end(self) -> None:
+        if self._active_segment_idle_end_task and not self._active_segment_idle_end_task.done():
+            self._active_segment_idle_end_task.cancel()
+        self._active_segment_idle_end_task = None
+
+    def _schedule_active_segment_idle_end(self) -> None:
+        active_req_id = self._active_req_id
+        if active_req_id is None:
+            return
+
+        self._cancel_active_segment_idle_end()
+        self._active_segment_idle_end_task = asyncio.create_task(
+            self._wait_for_active_segment_idle_end(active_req_id, ACTIVE_SEGMENT_IDLE_END_SECONDS),
+            name=f"spatius_idle_segment_end_{active_req_id}",
+        )
+
+    async def _wait_for_active_segment_idle_end(self, req_id: str, timeout: float) -> None:
+        try:
+            await asyncio.sleep(timeout)
+        except asyncio.CancelledError:
+            return
+
+        if self._active_req_id != req_id:
+            return
+
+        if req_id in self._pending_segment_ids:
+            return
+
+        if req_id not in self._segments:
+            return
+
+        if await self._finalize_active_segment(source="idle_timeout"):
+            logger.warning(
+                "Avatar: Segment end marker missing, forcing segment finalization "
+                f"(request_id={req_id}, idle_timeout={timeout:.2f}s)"
+            )
+
+    async def _finalize_active_segment(self, *, source: str) -> bool:
+        if self._active_req_id is None or not self._spatius_session:
+            return False
+
+        async with self._segment_finalize_lock:
+            active_req_id = self._active_req_id
+            if active_req_id is None:
+                return False
+
+            self._cancel_active_segment_idle_end()
+
+            req_id = await self._spatius_session.send_audio(
+                audio=b"",
+                end=True,
+            )
+
+            if req_id != active_req_id:
+                logger.warning(
+                    "Avatar: Request ID changed while finalizing segment "
+                    f"(expected={active_req_id}, actual={req_id}, source={source})"
+                )
+
+            self._active_req_id = None
+
+            active_segment = self._segments.pop(active_req_id, None)
+            segment = self._segments.get(req_id)
+
+            if active_segment is None and segment is None:
+                logger.debug(
+                    "Avatar: Segment completed before finalization finished "
+                    f"(request_id={active_req_id}, finalize_request_id={req_id}, source={source})"
+                )
+                return True
+
+            if segment is None:
+                if active_segment is None:
+                    return True
+                active_segment.req_id = req_id
+                segment = active_segment
+                self._segments[req_id] = segment
+            elif active_segment is not None and segment is not active_segment:
+                segment.pushed_duration = max(segment.pushed_duration, active_segment.pushed_duration)
+                if segment.first_frame_at is None:
+                    segment.first_frame_at = active_segment.first_frame_at
+
+            logger.debug(
+                "Avatar: Segment input completed "
+                f"(request_id={req_id}, duration={segment.pushed_duration:.3f}s, source={source})"
+            )
+            self._mark_segment_waiting_for_completion(segment)
+            return True
+
+    def _mark_segment_waiting_for_completion(self, segment: _SegmentState) -> None:
+        if segment.req_id not in self._pending_segment_ids:
+            self._pending_segment_ids.append(segment.req_id)
+
+        if segment.completion_timeout_task and not segment.completion_timeout_task.done():
+            segment.completion_timeout_task.cancel()
+
+        timeout = self._compute_completion_timeout(segment)
+        segment.completion_timeout_task = asyncio.create_task(
+            self._wait_for_segment_completion_timeout(segment.req_id, timeout),
+            name=f"spatius_segment_timeout_{segment.req_id}",
+        )
+
+    @staticmethod
+    def _compute_completion_timeout(segment: _SegmentState) -> float:
+        if segment.first_frame_at is None:
+            return MIN_COMPLETION_TIMEOUT_SECONDS
+
+        expected_playback_end = segment.first_frame_at + segment.pushed_duration
+        remaining_playback = max(0.0, expected_playback_end - time.time())
+        return max(
+            MIN_COMPLETION_TIMEOUT_SECONDS,
+            remaining_playback + COMPLETION_TIMEOUT_BUFFER_SECONDS,
+        )
+
+    async def _wait_for_segment_completion_timeout(self, req_id: str, timeout: float) -> None:
+        try:
+            await asyncio.sleep(timeout)
+        except asyncio.CancelledError:
+            return
+
+        if self._complete_segment(req_id=req_id, interrupted=False, reason="timeout"):
+            logger.warning(
+                "Avatar segment completion timed out, assuming playback finished "
+                f"(request_id={req_id}, timeout={timeout:.2f}s)"
+            )
+
+    def _on_transport_frame(self, frame: bytes, is_last: bool) -> None:
+        if not is_last:
+            return
+
+        req_id = self._extract_req_id_from_transport_frame(frame)
+        if req_id is not None:
+            if req_id not in self._pending_segment_ids:
+                logger.debug(
+                    f"Avatar: ignoring provider completion before local segment finalization (request_id={req_id})"
+                )
+                return
+
+            if not self._complete_segment(req_id=req_id, interrupted=False, reason="provider_end"):
+                logger.debug(f"Avatar: completion event for unknown request_id={req_id}")
+            return
+
+        if self._pending_segment_ids:
+            fallback_req_id = self._pending_segment_ids[0]
+            if self._complete_segment(req_id=fallback_req_id, interrupted=False, reason="provider_end_fallback"):
+                logger.warning(
+                    "Avatar: completion event missing request ID, matched oldest pending segment "
+                    f"(request_id={fallback_req_id})"
+                )
+
+    def _on_clear_buffer(self) -> None:
+        asyncio.create_task(self._handle_interrupt())
+
+    @staticmethod
+    def _extract_req_id_from_transport_frame(frame: bytes) -> str | None:
+        try:
+            envelope = message_pb2.Message()
+            envelope.ParseFromString(frame)
+        except Exception:
+            return None
+
+        if envelope.type != message_pb2.MESSAGE_SERVER_RESPONSE_ANIMATION:
+            return None
+
+        req_id = envelope.server_response_animation.req_id
+        return req_id or None
+
+    def _complete_segment(self, *, req_id: str, interrupted: bool, reason: str) -> bool:
+        segment = self._segments.pop(req_id, None)
+        if segment is None:
+            return False
+
+        self._pending_segment_ids = deque(
+            pending_req_id for pending_req_id in self._pending_segment_ids if pending_req_id != req_id
+        )
+
+        if segment.completion_timeout_task and not segment.completion_timeout_task.done():
+            segment.completion_timeout_task.cancel()
+
+        if self._active_req_id == req_id:
+            self._active_req_id = None
+            self._cancel_active_segment_idle_end()
+
+        playback_position = (
+            self._estimate_interrupted_playback_position(segment) if interrupted else segment.pushed_duration
+        )
+
+        if self._audio_buffer:
+            self._audio_buffer.notify_playback_finished(
+                playback_position=playback_position,
+                interrupted=interrupted,
+            )
+
+        logger.debug(
+            "Avatar: Segment playback completed "
+            f"(request_id={req_id}, reason={reason}, interrupted={interrupted}, "
+            f"playback_position={playback_position:.3f}s, pushed_duration={segment.pushed_duration:.3f}s)"
+        )
+        return True
+
+    @staticmethod
+    def _estimate_interrupted_playback_position(segment: _SegmentState) -> float:
+        if segment.first_frame_at is None:
+            return 0.0
+
+        elapsed = max(0.0, time.time() - segment.first_frame_at)
+        return min(segment.pushed_duration, elapsed)
+
+    def _complete_all_segments(self, *, interrupted: bool, reason: str) -> None:
+        for req_id in list(self._segments.keys()):
+            self._complete_segment(req_id=req_id, interrupted=interrupted, reason=reason)
+
+        self._active_req_id = None
+        self._cancel_active_segment_idle_end()
+        self._pending_segment_ids.clear()
+
+    async def _handle_interrupt(self) -> None:
+        """Handle interruption - stop avatar's current audio processing."""
+        if not self._spatius_session:
+            return
+
+        try:
+            interrupted_id = await self._spatius_session.interrupt()
+
+            async with self._segment_finalize_lock:
+                if not self._complete_segment(req_id=interrupted_id, interrupted=True, reason="interrupt"):
+                    # Fallback: a race can leave the request id unmatched.
+                    if self._active_req_id is not None:
+                        self._complete_segment(
+                            req_id=self._active_req_id,
+                            interrupted=True,
+                            reason="interrupt_fallback",
+                        )
+                # Complete any remaining pending segments that were also interrupted
+                for req_id in list(self._segments.keys()):
+                    self._complete_segment(req_id=req_id, interrupted=True, reason="interrupt_remaining")
+
+            logger.debug(f"Avatar interrupted, request_id={interrupted_id}")
+        except Exception as e:
+            logger.warning(f"Failed to interrupt avatar: {e}")
+
+    async def aclose(self) -> None:
+        """Clean up avatar session resources."""
+        if self._main_task:
+            self._main_task.cancel()
+            try:
+                await self._main_task
+            except asyncio.CancelledError:
+                pass
+            self._main_task = None
+
+        self._cancel_active_segment_idle_end()
+
+        self._complete_all_segments(interrupted=True, reason="session_close")
+
+        if (
+            self._agent_session
+            and self._audio_buffer
+            and self._audio_output_attached
+            and self._agent_session.output.audio is self._audio_buffer
+        ):
+            self._agent_session.output.audio = self._original_audio_output
+            self._audio_output_attached = False
+            self._original_audio_output = None
+
+        if self._audio_buffer:
+            await self._audio_buffer.aclose()
+            self._audio_buffer = None
+
+        if self._spatius_session:
+            try:
+                await self._spatius_session.close()
+                logger.info("Avatar session closed")
+            except Exception as e:
+                logger.warning(f"Error closing avatar session: {e}")
+            finally:
+                self._spatius_session = None
+
+        self._initialized = False
+        self._agent_session = None
+        self._audio_output_attached = False
+        self._original_audio_output = None

livekit_plugins_spatius-1.4.5/livekit/plugins/spatius/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.spatius")

livekit_plugins_spatius-1.4.5/pyproject.toml

@@ -0,0 +1,67 @@
+[build-system]
+requires = ["hatchling>=1.28.0", "hatch-vcs>=0.5.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "livekit-plugins-spatius"
+dynamic = ["version"]
+description = "Agent Framework plugin for Spatius Avatar"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10.0"
+authors = [{ name = "3DRX", email = "3drxkjy@gmail.com" }]
+keywords = ["voice", "ai", "realtime", "audio", "video", "livekit", "webrtc", "avatar", "spatius"]
+classifiers = [
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Multimedia :: Sound/Audio",
+    "Topic :: Multimedia :: Video",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3 :: Only",
+]
+dependencies = [
+    "livekit-agents>=1.2.9",
+    "spatius>=1.0.0",
+]
+
+[project.urls]
+Documentation = "https://docs.spatius.ai"
+Website = "https://www.spatius.ai/"
+Source = "https://github.com/spatius-ai/livekit-plugins-spatius"
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build]
+exclude = [
+    "/.uv-cache",
+    "/.venv",
+    "/.pytest_cache",
+    "/dist",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["livekit"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/livekit"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "UP",  # pyupgrade
+    "B",   # flake8-bugbear
+]