vision-agents-plugins-liveavatar 0.6.0__tar.gz

vision_agents_plugins_liveavatar-0.6.0/.gitignore

@@ -0,0 +1,102 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.cursor/*
+# Distribution / packaging
+.Python
+build/
+dist/
+downloads/
+develop-eggs/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+pip-wheel-metadata/
+MANIFEST
+*.egg-info/
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+coverage.xml
+nosetests.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Type checker / lint caches
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pytype/
+.pyre/
+.ruff_cache/
+
+# Environments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.env
+.env.local
+.env.*.local
+.env.bak
+pyvenv.cfg
+.python-version
+
+# Editors / IDEs
+.vscode/
+.idea/
+
+# Jupyter Notebook
+.ipynb_checkpoints/
+
+# OS / Misc
+.DS_Store
+*.log
+
+# Tooling & repo-specific
+pyrightconfig.json
+shell.nix
+bin/*
+lib/*
+stream-py/
+
+# Example lock files (regenerated by uv sync)
+examples/*/uv.lock
+plugins/*/example/uv.lock
+
+# Artifacts / assets
+*.pt
+*.kef
+*.onnx
+profile.html
+
+/opencode.json
+.ralph-tui/
+.claude/*
+!.claude/skills/
+
+.uv-cache/
+
+# pytest json report
+.report.json

vision_agents_plugins_liveavatar-0.6.0/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: vision-agents-plugins-liveavatar
+Version: 0.6.0
+Summary: LiveAvatar plugin for Vision Agents
+Project-URL: Documentation, https://visionagents.ai/
+Project-URL: Website, https://visionagents.ai/
+Project-URL: Source, https://github.com/GetStream/Vision-Agents
+License-Expression: MIT
+Keywords: AI,agents,avatars,heygen,liveavatar,voice agents
+Requires-Python: >=3.10
+Requires-Dist: httpx<1,>=0.28.1
+Requires-Dist: livekit<2,>=1.1.2
+Requires-Dist: vision-agents
+Requires-Dist: websockets<16,>=13.0

vision_agents_plugins_liveavatar-0.6.0/pyproject.toml

@@ -0,0 +1,41 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vision-agents-plugins-liveavatar"
+dynamic = ["version"]
+description = "LiveAvatar plugin for Vision Agents"
+keywords = ["heygen", "liveavatar", "avatars", "AI", "voice agents", "agents"]
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+    "vision-agents",
+    "livekit>=1.1.2,<2",
+    "httpx>=0.28.1,<1",
+    "websockets>=13.0,<16",
+]
+
+[project.urls]
+Documentation = "https://visionagents.ai/"
+Website = "https://visionagents.ai/"
+Source = "https://github.com/GetStream/Vision-Agents"
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "..", search_parent_directories = true, fallback_version = "0.0.0" }
+
+[tool.hatch.build.targets.wheel]
+packages = [".", "vision_agents"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/vision_agents"]
+
+[tool.uv.sources]
+vision-agents = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "pytest-asyncio>=1.0.0",
+]

vision_agents_plugins_liveavatar-0.6.0/vision_agents/plugins/liveavatar/__init__.py

@@ -0,0 +1,3 @@
+from .liveavatar_avatar import LiveAvatar as Avatar
+
+__all__ = ["Avatar"]

vision_agents_plugins_liveavatar-0.6.0/vision_agents/plugins/liveavatar/exceptions.py

@@ -0,0 +1,16 @@
+class LiveAvatarError(Exception):
+    """Base exception for LiveAvatar API errors."""
+
+
+class LiveAvatarAPIError(LiveAvatarError):
+    """Raised when an HTTP request to the LiveAvatar API fails."""
+
+    def __init__(
+        self,
+        message: str,
+        status_code: int | None = None,
+        body: str | None = None,
+    ) -> None:
+        self.status_code = status_code
+        self.body = body
+        super().__init__(message)

vision_agents_plugins_liveavatar-0.6.0/vision_agents/plugins/liveavatar/liveavatar_avatar.py

@@ -0,0 +1,216 @@
+import asyncio
+import logging
+import os
+
+import av
+from getstream.video.rtc.track_util import PcmData
+from vision_agents.core.agents.inference import (
+    AudioOutputChunk,
+    AudioOutputFlush,
+    AudioOutputStream,
+)
+from vision_agents.core.avatars import Avatar, AVSynchronizer
+from vision_agents.core.utils.utils import cancel_and_wait
+from vision_agents.core.utils.video_track import QueuedVideoTrack
+
+from .liveavatar_client import LiveAvatarClient, Session
+from .liveavatar_rtc_manager import LiveAvatarRTCManager
+from .liveavatar_websocket import LiveAvatarWebSocket
+
+logger = logging.getLogger(__name__)
+
+
+def _task_done_callback(task: asyncio.Task[None]) -> None:
+    if not task.cancelled() and task.exception() is not None:
+        logger.exception(
+            "LiveAvatar background task %s failed",
+            task.get_name(),
+            exc_info=task.exception(),
+        )
+
+
+class LiveAvatar(Avatar):
+    """LiveAvatar plugin (LITE mode, custom-agent integration path).
+
+    References:
+    - https://docs.liveavatar.com
+    - https://docs.liveavatar.com/docs/lite-mode/integration-paths
+
+    Sends TTS audio via the LiveAvatar media-server WebSocket and
+    receives synchronized lip-synced video and audio from the room
+    LiveAvatar provisions for the session.
+    """
+
+    provider_name = "liveavatar"
+
+    def __init__(
+        self,
+        avatar_id: str | None = None,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        is_sandbox: bool = True,
+        max_session_duration: int | None = None,
+        video_quality: str = "high",
+        video_encoding: str = "H264",
+        width: int = 1280,
+        height: int = 720,
+        fps: int = 30,
+        buffer_seconds: float = 1.0,
+    ) -> None:
+        """Initialize the LiveAvatar plugin.
+
+        Args:
+            avatar_id: LiveAvatar avatar UUID. Falls back to LIVEAVATAR_AVATAR_ID.
+            api_key: LiveAvatar API key. Falls back to LIVEAVATAR_API_KEY.
+            base_url: Override the LiveAvatar API base URL.
+            is_sandbox: Sandbox sessions don't burn credits but are duration-capped.
+            max_session_duration: Session length cap in seconds; None for the API default.
+            video_quality: One of "low", "medium", "high", "very_high".
+                Default - `"high"`.
+            video_encoding: One of "H264", "VP8".
+            width: Output video width in pixels.
+            height: Output video height in pixels.
+            fps: Output video frame rate. Must be > 0.
+            buffer_seconds: Max video buffer depth in seconds. Caps how many frames
+                can be queued ahead of audio playback. Must be > 0.
+        """
+        super().__init__()
+        if buffer_seconds <= 0:
+            raise ValueError("buffer_seconds must be > 0")
+        if fps <= 0:
+            raise ValueError("fps must be > 0")
+
+        api_key = api_key or os.getenv("LIVEAVATAR_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "LiveAvatar API key required. Set LIVEAVATAR_API_KEY or pass api_key."
+            )
+        avatar_id = avatar_id or os.getenv("LIVEAVATAR_AVATAR_ID")
+        if not avatar_id:
+            raise ValueError(
+                "Avatar ID required. Set LIVEAVATAR_AVATAR_ID or pass avatar_id."
+            )
+
+        self._client = LiveAvatarClient(api_key=api_key, base_url=base_url)
+        self._rtc_manager = LiveAvatarRTCManager(
+            on_video=self._on_video_frame,
+            on_audio=self._on_audio_frame,
+            on_disconnect=self._on_disconnect,
+        )
+        self._sync = AVSynchronizer(
+            width=width,
+            height=height,
+            fps=fps,
+            max_queue_size=int(fps * buffer_seconds),
+        )
+
+        self._avatar_id = avatar_id
+        self._is_sandbox = is_sandbox
+        self._max_session_duration = max_session_duration
+        self._video_quality = video_quality
+        self._video_encoding = video_encoding
+
+        self._session: Session | None = None
+        self._websocket: LiveAvatarWebSocket | None = None
+        self._audio_input_task: asyncio.Task[None] | None = None
+        self._connected = False
+
+        logger.debug("LiveAvatar initialized (%dx%d)", width, height)
+
+    def video_output(self) -> QueuedVideoTrack:
+        """Return the video track that receives avatar video frames."""
+        return self._sync.video_output
+
+    def audio_output(self) -> AudioOutputStream:
+        """Return the audio stream that receives avatar audio frames."""
+        return self._sync.audio_output
+
+    async def start(self) -> None:
+        """Connect to LiveAvatar. Called by the Agent during ``join()``."""
+        await self._connect()
+
+    async def close(self) -> None:
+        """Stop the session and release all resources."""
+        if self._audio_input_task is not None:
+            await cancel_and_wait(self._audio_input_task)
+
+        if self._websocket is not None:
+            try:
+                await self._websocket.close()
+            except Exception as exc:
+                logger.warning("Failed to close LiveAvatar websocket: %s", exc)
+
+        try:
+            await self._rtc_manager.close()
+        except Exception as exc:
+            logger.warning("Failed to close LiveAvatar RTC manager: %s", exc)
+
+        # Close sync AFTER rtc_manager so its receive tasks can't write to a
+        # closed stream during teardown.
+        self._sync.close()
+
+        if self._session is not None:
+            try:
+                await self._client.stop_session(session_id=self._session.session_id)
+            except Exception as exc:
+                logger.warning("Failed to stop LiveAvatar session: %s", exc)
+
+        try:
+            await self._client.close()
+        finally:
+            self._connected = False
+            logger.debug("LiveAvatar closed")
+
+    async def _process_audio_input(self) -> None:
+        async for item in self.input_audio_stream:
+            if isinstance(item, AudioOutputChunk):
+                if item.data is not None and self._websocket is not None:
+                    await self._websocket.send_audio_frame(item.data)
+                if item.final and self._websocket is not None:
+                    await self._websocket.end_turn()
+            elif isinstance(item, AudioOutputFlush):
+                if self._websocket is not None:
+                    await self._websocket.interrupt()
+                await self._sync.flush()
+
+    async def _connect(self) -> None:
+        token = await self._client.create_session_token(
+            self._avatar_id,
+            is_sandbox=self._is_sandbox,
+            max_session_duration=self._max_session_duration,
+            video_quality=self._video_quality,
+            video_encoding=self._video_encoding,
+        )
+        self._session = await self._client.start_session(token.session_token)
+
+        try:
+            await self._rtc_manager.connect(
+                self._session.livekit_url, self._session.livekit_agent_token
+            )
+            self._websocket = LiveAvatarWebSocket(self._session.ws_url)
+            await self._websocket.connect()
+        except Exception:
+            await self._rtc_manager.close()
+            if self._websocket is not None:
+                await self._websocket.close()
+            await self._client.stop_session(session_id=self._session.session_id)
+            raise
+
+        self._connected = True
+        logger.info(
+            "LiveAvatar connection established session_id=%s", self._session.session_id
+        )
+
+        if self._audio_input_task is None:
+            self._audio_input_task = asyncio.create_task(self._process_audio_input())
+            self._audio_input_task.add_done_callback(_task_done_callback)
+
+    async def _on_video_frame(self, frame: av.VideoFrame) -> None:
+        await self._sync.write_video(frame)
+
+    async def _on_audio_frame(self, pcm: PcmData) -> None:
+        await self._sync.write_audio(pcm)
+
+    async def _on_disconnect(self) -> None:
+        logger.info("LiveAvatar disconnected")
+        self._connected = False

vision_agents_plugins_liveavatar-0.6.0/vision_agents/plugins/liveavatar/liveavatar_client.py

@@ -0,0 +1,123 @@
+import logging
+from dataclasses import dataclass
+from typing import Self
+
+import httpx
+
+from .exceptions import LiveAvatarAPIError
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_BASE_URL = "https://api.liveavatar.com"
+
+
+@dataclass
+class SessionToken:
+    session_id: str
+    session_token: str
+
+
+@dataclass
+class Session:
+    session_id: str
+    livekit_url: str
+    livekit_agent_token: str
+    livekit_client_token: str
+    ws_url: str
+    max_session_duration: int | None = None
+
+
+class LiveAvatarClient:
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str | None = None,
+        timeout: float = 30.0,
+    ) -> None:
+        if not api_key:
+            raise ValueError("LiveAvatar API key required")
+        self._api_key = api_key
+        self._base_url = (base_url or DEFAULT_BASE_URL).rstrip("/")
+        self._http = httpx.AsyncClient(
+            base_url=self._base_url,
+            timeout=timeout,
+            headers={"X-API-KEY": api_key},
+        )
+
+    async def close(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.close()
+
+    async def create_session_token(
+        self,
+        avatar_id: str,
+        *,
+        is_sandbox: bool = True,
+        max_session_duration: int | None = None,
+        video_quality: str = "high",
+        video_encoding: str = "H264",
+    ) -> SessionToken:
+        body: dict[str, object] = {
+            "mode": "LITE",
+            "avatar_id": avatar_id,
+            "is_sandbox": is_sandbox,
+            "video_settings": {
+                "quality": video_quality,
+                "encoding": video_encoding,
+            },
+        }
+        if max_session_duration is not None:
+            body["max_session_duration"] = max_session_duration
+
+        resp = await self._http.post("/v1/sessions/token", json=body)
+        self._raise_for_status(resp)
+        data = resp.json()["data"]
+        return SessionToken(
+            session_id=data["session_id"],
+            session_token=data["session_token"],
+        )
+
+    async def start_session(self, session_token: str) -> Session:
+        resp = await self._http.post(
+            "/v1/sessions/start",
+            headers={"Authorization": f"Bearer {session_token}"},
+            json={},
+        )
+        self._raise_for_status(resp)
+        data = resp.json()["data"]
+        return Session(
+            session_id=data["session_id"],
+            livekit_url=data["livekit_url"],
+            livekit_agent_token=data["livekit_agent_token"],
+            livekit_client_token=data["livekit_client_token"],
+            ws_url=data["ws_url"],
+            max_session_duration=data.get("max_session_duration"),
+        )
+
+    async def stop_session(
+        self,
+        *,
+        session_id: str,
+        reason: str = "USER_CLOSED",
+    ) -> None:
+        resp = await self._http.post(
+            "/v1/sessions/stop",
+            json={"session_id": session_id, "reason": reason},
+        )
+        self._raise_for_status(resp)
+
+    @staticmethod
+    def _raise_for_status(resp: httpx.Response) -> None:
+        try:
+            resp.raise_for_status()
+        except httpx.HTTPStatusError as e:
+            raise LiveAvatarAPIError(
+                f"{e.request.method} {e.request.url} -> {resp.status_code}: {resp.text}",
+                status_code=resp.status_code,
+                body=resp.text,
+            ) from e

vision_agents_plugins_liveavatar-0.6.0/vision_agents/plugins/liveavatar/liveavatar_rtc_manager.py

@@ -0,0 +1,132 @@
+import asyncio
+import logging
+from typing import Callable, Coroutine
+
+import av
+import numpy as np
+from getstream.video.rtc.track_util import AudioFormat, PcmData
+from livekit import rtc
+from vision_agents.core.utils.utils import cancel_and_wait
+
+logger = logging.getLogger(__name__)
+
+
+VideoCallback = Callable[[av.VideoFrame], Coroutine[None, None, None]]
+AudioCallback = Callable[[PcmData], Coroutine[None, None, None]]
+DisconnectCallback = Callable[[], Coroutine[None, None, None]]
+
+
+def _task_done_callback(task: asyncio.Task[None]) -> None:
+    if not task.cancelled() and task.exception() is not None:
+        logger.error(
+            "Background task %s failed", task.get_name(), exc_info=task.exception()
+        )
+
+
+class LiveAvatarRTCManager:
+    """Joins HeyGen's avatar room and forwards remote video/audio to callbacks."""
+
+    def __init__(
+        self,
+        on_video: VideoCallback,
+        on_audio: AudioCallback,
+        on_disconnect: DisconnectCallback,
+    ) -> None:
+        self._on_video = on_video
+        self._on_audio = on_audio
+        self._on_disconnect = on_disconnect
+
+        self._room: rtc.Room | None = None
+        self._connected = False
+        self._tasks: set[asyncio.Task[None]] = set()
+
+    @property
+    def is_connected(self) -> bool:
+        return self._connected
+
+    async def connect(self, room_url: str, token: str) -> None:
+        room = rtc.Room()
+
+        @room.on("participant_connected")
+        def on_participant_connected(participant: rtc.RemoteParticipant) -> None:
+            logger.info("LiveAvatar participant joined: %s", participant.identity)
+
+        @room.on("track_subscribed")
+        def on_track_subscribed(
+            track: rtc.Track,
+            publication: rtc.RemoteTrackPublication,
+            participant: rtc.RemoteParticipant,
+        ) -> None:
+            if track.kind == rtc.TrackKind.KIND_VIDEO:
+                logger.info("Subscribed video track from %s", participant.identity)
+                video_stream = rtc.VideoStream(track)
+                self._create_task(self._consume_video(video_stream))
+            elif track.kind == rtc.TrackKind.KIND_AUDIO:
+                logger.info("Subscribed audio track from %s", participant.identity)
+                audio_stream = rtc.AudioStream(track, sample_rate=48000, num_channels=2)
+                self._create_task(self._consume_audio(audio_stream))
+
+        @room.on("participant_disconnected")
+        def on_participant_disconnected(participant: rtc.RemoteParticipant) -> None:
+            logger.info(
+                "LiveAvatar participant disconnected: %s; reason: %s",
+                participant.identity,
+                participant.disconnect_reason,
+            )
+            self._connected = False
+            self._create_task(self._on_disconnect())
+            if self._room is not None:
+                self._create_task(self._room.disconnect())
+
+        @room.on("disconnected")
+        def on_disconnected(reason: str) -> None:
+            # The "disconnected" callback may fire multiple times because we
+            # also disconnect ourselves when the avatar leaves.
+            if self._connected:
+                logger.info("LiveAvatar room disconnected; reason: %s", reason)
+                self._connected = False
+                self._create_task(self._on_disconnect())
+
+        logger.info("Connecting to LiveAvatar room url=%s", room_url)
+        await room.connect(room_url, token)
+        logger.info("Connected to LiveAvatar room")
+
+        self._room = room
+        self._connected = True
+
+    async def close(self) -> None:
+        try:
+            if self._room is not None:
+                await self._room.disconnect()
+            while self._tasks:
+                tasks = tuple(self._tasks)
+                self._tasks.clear()
+                await cancel_and_wait(*tasks)
+        finally:
+            self._room = None
+            self._connected = False
+
+    async def _consume_video(self, video_stream: rtc.VideoStream) -> None:
+        async for event in video_stream:
+            lk_frame = event.frame.convert(rtc.VideoBufferType.RGBA)
+            arr = np.frombuffer(lk_frame.data, dtype=np.uint8).reshape(
+                lk_frame.height, lk_frame.width, 4
+            )
+            await self._on_video(av.VideoFrame.from_ndarray(arr, format="rgba"))
+
+    async def _consume_audio(self, audio_stream: rtc.AudioStream) -> None:
+        async for event in audio_stream:
+            frame = event.frame
+            pcm = PcmData.from_bytes(
+                frame.data,  # type: ignore[arg-type]
+                sample_rate=frame.sample_rate,
+                format=AudioFormat.S16,
+                channels=frame.num_channels,
+            )
+            await self._on_audio(pcm)
+
+    def _create_task(self, coro: Coroutine[None, None, None]) -> None:
+        task: asyncio.Task[None] = asyncio.create_task(coro)
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+        task.add_done_callback(_task_done_callback)

vision_agents_plugins_liveavatar-0.6.0/vision_agents/plugins/liveavatar/liveavatar_websocket.py

@@ -0,0 +1,102 @@
+import asyncio
+import base64
+import json
+import logging
+import uuid
+
+import websockets
+from getstream.video.rtc.track_util import PcmData
+from websockets.asyncio.client import ClientConnection
+from websockets.exceptions import ConnectionClosed
+
+logger = logging.getLogger(__name__)
+
+
+class LiveAvatarWebSocket:
+    """Audio bridge to the LiveAvatar media server (LITE-mode events)."""
+
+    def __init__(
+        self,
+        ws_url: str,
+        sample_rate: int = 24000,
+        num_channels: int = 1,
+    ) -> None:
+        self._ws_url = ws_url
+        self._sample_rate = sample_rate
+        self._num_channels = num_channels
+        self._ws: ClientConnection | None = None
+        self._closed = False
+        self._reconnect_lock = asyncio.Lock()
+
+    @property
+    def connected(self) -> bool:
+        return self._ws is not None
+
+    async def connect(self) -> None:
+        if self.connected:
+            return
+        # ping_interval=None: continuous audio frames are themselves a
+        # liveness signal; the LiveAvatar media server stalls pong responses
+        # under load and the client tears down the conn (1011) otherwise.
+        self._ws = await websockets.connect(self._ws_url, ping_interval=None)
+        try:
+            await self._ws.send(
+                json.dumps(
+                    {
+                        "type": "start",
+                        "encoding": "pcm_s16le",
+                        "sample_rate": self._sample_rate,
+                        "channels": self._num_channels,
+                    }
+                )
+            )
+        except ConnectionClosed:
+            self._ws = None
+            raise
+        logger.info("liveavatar_ws connected url=%s", self._ws_url)
+
+    async def close(self) -> None:
+        self._closed = True
+        if self._ws is not None:
+            try:
+                await self._ws.close()
+            except ConnectionClosed:
+                pass
+            self._ws = None
+
+    async def send_audio_frame(self, pcm: PcmData) -> None:
+        pcm = pcm.resample(
+            target_sample_rate=self._sample_rate,
+            target_channels=self._num_channels,
+        )
+        b64 = base64.b64encode(pcm.to_bytes()).decode("ascii")
+        await self._send_json({"type": "agent.speak", "audio": b64})
+
+    async def end_turn(self) -> None:
+        await self._send_json({"type": "agent.speak_end"})
+
+    async def interrupt(self) -> None:
+        await self._send_json(
+            {"type": "agent.interrupt", "event_id": str(uuid.uuid4())}
+        )
+
+    async def _send_json(self, msg: dict[str, object]) -> None:
+        if self._closed:
+            raise RuntimeError("liveavatar_ws is closed")
+        if not self.connected:
+            await self.connect()
+        assert self._ws is not None
+        try:
+            await self._ws.send(json.dumps(msg))
+        except ConnectionClosed:
+            logger.warning("liveavatar_ws connection closed during send; reconnecting")
+            self._ws = None
+            await self._reconnect()
+            assert self._ws is not None
+            await self._ws.send(json.dumps(msg))
+
+    async def _reconnect(self) -> None:
+        async with self._reconnect_lock:
+            if self.connected or self._closed:
+                return
+            await self.connect()