openspline 0.1.0__tar.gz

openspline-0.1.0/.gitignore

@@ -0,0 +1,21 @@
+.venv/
+__pycache__/
+*.py[cod]
+*.egg-info/
+.env
+.env.*
+!.env.example
+node_modules/
+dist/
+.pytest_cache/
+.ruff_cache/
+/models/
+/runtime/
+artifacts/private/
+*.log
+.gstack/
+
+.venvs/
+.cache/
+.tools/
+experiments/

openspline-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 openspline contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

openspline-0.1.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.5
+Name: openspline
+Version: 0.1.0
+Summary: Add a live face to any voice agent
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: <3.13,>=3.10
+Requires-Dist: httpx<1,>=0.27
+Requires-Dist: websockets<16,>=12
+Provides-Extra: media
+Requires-Dist: aiortc<2,>=1.14; extra == 'media'
+Requires-Dist: av<17,>=14; extra == 'media'

openspline-0.1.0/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+[project]
+name = "openspline"
+version = "0.1.0"
+description = "Add a live face to any voice agent"
+requires-python = ">=3.10,<3.13"
+license = "MIT"
+dependencies = ["httpx>=0.27,<1", "websockets>=12,<16"]
+[project.optional-dependencies]
+media = ["aiortc>=1.14,<2", "av>=14,<17"]
+[tool.hatch.build.targets.wheel]
+packages = ["src/openspline"]

openspline-0.1.0/src/openspline/__init__.py

@@ -0,0 +1,20 @@
+from .client import AvatarSession, Openspline
+from .config import OpensplineConfig
+from .errors import (
+    CapacityError,
+    ConfigurationError,
+    ConnectionError,
+    InferenceError,
+    OpensplineError,
+)
+
+__all__ = [
+    "AvatarSession",
+    "CapacityError",
+    "ConfigurationError",
+    "ConnectionError",
+    "InferenceError",
+    "Openspline",
+    "OpensplineConfig",
+    "OpensplineError",
+]

openspline-0.1.0/src/openspline/client.py

@@ -0,0 +1,301 @@
+"""Lightweight async client. No CUDA, NumPy, or framework imports."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import contextlib
+import inspect
+import json
+import uuid
+from collections.abc import Mapping
+from dataclasses import fields
+from pathlib import Path
+
+import httpx
+from websockets.legacy.client import connect
+
+from .config import OpensplineConfig, Quality
+from .errors import ConnectionError, error_from
+
+
+class Openspline:
+    def __init__(
+        self,
+        url: str | None = None,
+        timeout: float | None = None,
+        *,
+        quality: Quality | None = None,
+        viewer_timeout: float | None = None,
+        config: OpensplineConfig | Mapping | None = None,
+    ):
+        if config is None:
+            values = {}
+        elif isinstance(config, OpensplineConfig):
+            values = {field.name: getattr(config, field.name) for field in fields(config)}
+        elif isinstance(config, Mapping):
+            values = dict(config)
+        else:
+            raise TypeError("config must be an OpensplineConfig or mapping")
+        overrides = {
+            key: value
+            for key, value in {
+                "url": url,
+                "timeout": timeout,
+                "quality": quality,
+                "viewer_timeout": viewer_timeout,
+            }.items()
+            if value is not None
+        }
+        self.config = OpensplineConfig(**(values | overrides))
+        self.url = self.config.url
+        self.timeout = self.config.timeout
+
+    def avatar(self, portrait, quality: Quality | None = None):
+        quality = self.config.quality if quality is None else quality
+        if quality not in {"low", "high"}:
+            raise ValueError("quality must be low or high")
+        return AvatarSession(self, portrait, quality)
+
+
+class AvatarSession:
+    def __init__(self, client, portrait, quality):
+        self.client, self.portrait, self.quality = client, portrait, quality
+        self.id = None
+        self.viewer_url = None
+        self.session = None
+        self.ws = None
+        self.http = None
+        self.pending = {}
+        self.callbacks = []
+        self.epoch = 0
+        self.viewer_ready = asyncio.Event()
+        self.reader = None
+        self.heartbeat = None
+        self._dirty = False
+        self._closed = False
+        self._send_lock = asyncio.Lock()
+        self._media = None
+
+    async def __aenter__(self):
+        self.http = httpx.AsyncClient(base_url=self.client.url, timeout=self.client.timeout)
+        try:
+            if isinstance(self.portrait, (str, Path)):
+                raw = Path(self.portrait).read_bytes()
+            else:
+                raw = bytes(self.portrait)
+            response = await self.http.post(
+                "/v1/sessions",
+                files={"portrait": ("portrait", raw)},
+                data={"quality": self.quality},
+            )
+            self._check(response)
+            body = response.json()
+            self.id = body["id"]
+            self.token = body["publisher_token"]
+            self.session = {k: v for k, v in body.items() if k != "publisher_token"}
+            self.viewer_url = body["viewer_url"]
+            url = (
+                self.client.url.replace("https://", "wss://").replace("http://", "ws://")
+                + f"/v1/sessions/{self.id}/audio"
+            )
+            self.ws = await connect(url, max_size=1024 * 1024, open_timeout=30)
+            await self.ws.send(json.dumps({"token": self.token}))
+            self.reader = asyncio.create_task(self._receive())
+            self.heartbeat = asyncio.create_task(self._heartbeat())
+            return self
+        except BaseException:
+            await self.close()
+            raise
+
+    async def __aexit__(self, typ, exc, tb):
+        try:
+            if typ is None and self._dirty:
+                await self.end_turn()
+        finally:
+            await self.close()
+
+    @staticmethod
+    def _check(response):
+        if response.is_error:
+            try:
+                error = response.json().get("error", {})
+                message = error.get("message", response.text)
+                code = error.get("code", "request")
+            except (ValueError, AttributeError):
+                message = response.text
+                code = "request"
+            raise error_from(code, message)
+
+    def on(self, callback):
+        self.callbacks.append(callback)
+        return lambda: self.callbacks.remove(callback)
+
+    async def _receive(self):
+        failure = ConnectionError("Session connection closed", "connection")
+        try:
+            async for raw in self.ws:
+                event = json.loads(raw)
+                self.epoch = max(self.epoch, event.get("epoch", self.epoch))
+                if event.get("type") == "viewer_ready" or event.get("viewer_ready"):
+                    self.viewer_ready.set()
+                if event.get("type") == "viewer_disconnected":
+                    self.viewer_ready.clear()
+                id = event.get("id")
+                future = self.pending.get(id)
+                if event.get("type") == "error":
+                    error = error_from(
+                        event.get("code", "error"), event.get("message", "Session error")
+                    )
+                    if future and not future.done():
+                        future.set_exception(error)
+                    elif not id:
+                        failure = error
+                        for f in self.pending.values():
+                            if not f.done():
+                                f.set_exception(error)
+                elif future and not future.done():
+                    future.set_result(event)
+                for cb in list(self.callbacks):
+                    try:
+                        value = cb(event)
+                        if inspect.isawaitable(value):
+                            asyncio.create_task(value)
+                    except Exception:
+                        import logging
+
+                        logging.getLogger(__name__).exception("Session event callback failed")
+        except Exception as exc:
+            failure = ConnectionError(str(exc), "connection")
+        finally:
+            for future in self.pending.values():
+                if not future.done():
+                    future.set_exception(failure)
+
+    async def _heartbeat(self):
+        try:
+            while True:
+                await asyncio.sleep(15)
+                await self._request("ping")
+        except (asyncio.CancelledError, Exception):
+            return
+
+    async def _request(self, type, **data):
+        if self._closed or not self.ws:
+            raise ConnectionError("Session is not connected", "connection")
+        id = uuid.uuid4().hex
+        future = asyncio.get_running_loop().create_future()
+        self.pending[id] = future
+        try:
+            async with self._send_lock:
+                await self.ws.send(json.dumps({"type": type, "id": id, **data}))
+            return await asyncio.wait_for(future, self.client.timeout)
+        finally:
+            self.pending.pop(id, None)
+
+    async def wait_for_viewer(self, timeout=None):
+        await asyncio.wait_for(
+            self.viewer_ready.wait(),
+            self.client.config.viewer_timeout if timeout is None else timeout,
+        )
+
+    async def send_audio(self, data, *, sample_rate=24000, channels=1, encoding="pcm_s16le"):
+        if (
+            sample_rate not in {8000, 16000, 22050, 24000, 32000, 44100, 48000}
+            or channels not in {1, 2}
+            or encoding not in {"pcm_s16le", "pcm_f32le", "mp3"}
+        ):
+            raise ValueError("Invalid audio format")
+        raw = bytes(data)
+        size = (
+            16384
+            if encoding == "mp3"
+            else sample_rate * channels * (4 if encoding == "pcm_f32le" else 2) // 10
+        )
+        epoch = self.epoch
+        for i in range(0, len(raw), size):
+            if epoch != self.epoch:
+                return
+            await self._request(
+                "audio",
+                epoch=epoch,
+                data=base64.b64encode(raw[i : i + size]).decode(),
+                format={"sample_rate": sample_rate, "channels": channels, "encoding": encoding},
+            )
+            self._dirty = True
+
+    async def stream(self, source, **format):
+        await self.wait_for_viewer()
+        try:
+            async for packet in source:
+                await self.send_audio(packet, **format)
+            await self.end_turn()
+        except BaseException:
+            with contextlib.suppress(Exception):
+                await self.interrupt()
+            raise
+
+    async def end_turn(self, drain=True):
+        result = await self._request("end_turn", drain=drain)
+        self._dirty = False
+        return result
+
+    async def interrupt(self):
+        # A separate request can overtake a backpressured audio WebSocket.
+        response = await self.http.post(
+            f"/v1/sessions/{self.id}/interrupt", headers=self._headers()
+        )
+        self._check(response)
+        self.epoch = response.json()["epoch"]
+        self._dirty = False
+
+    def _headers(self):
+        return {"Authorization": "Bearer " + self.token}
+
+    async def viewer(self):
+        response = await self.http.post(f"/v1/sessions/{self.id}/viewer", headers=self._headers())
+        self._check(response)
+        self.session = response.json()
+        self.viewer_url = self.session["viewer_url"]
+        return self.session
+
+    async def play_file(self, path):
+        await self.wait_for_viewer()
+        with Path(path).open("rb") as f:
+            response = await self.http.post(
+                f"/v1/sessions/{self.id}/file",
+                headers=self._headers(),
+                files={"audio": (Path(path).name, f)},
+            )
+        self._check(response)
+        self._dirty = True
+        await self.end_turn()
+
+    async def media(self):
+        from .media import MediaReceiver
+
+        if self._media is None:
+            self._media = MediaReceiver(self)
+            await self._media.start()
+        return self._media
+
+    async def close(self):
+        if self._closed:
+            return
+        self._closed = True
+        if self._media:
+            await self._media.close()
+        if self.heartbeat:
+            self.heartbeat.cancel()
+        # Explicit delete is idempotent and releases the worker even if WebSocket setup failed.
+        if self.id and self.http:
+            with contextlib.suppress(Exception):
+                await self.http.delete(f"/v1/sessions/{self.id}", headers=self._headers())
+        if self.ws:
+            await self.ws.close()
+        if self.reader:
+            self.reader.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self.reader
+        if self.http:
+            await self.http.aclose()

openspline-0.1.0/src/openspline/config.py

@@ -0,0 +1,49 @@
+"""Validated, dependency-free client configuration."""
+
+import math
+import os
+from dataclasses import dataclass, field
+from typing import Literal
+from urllib.parse import urlsplit
+
+Quality = Literal["low", "high"]
+
+
+@dataclass(frozen=True)
+class OpensplineConfig:
+    url: str = field(default_factory=lambda: os.getenv("OPENSPLINE_URL", "http://localhost:7860"))
+    quality: Quality = "low"
+    timeout: float = 180
+    viewer_timeout: float = 60
+
+    def __post_init__(self):
+        if not isinstance(self.url, str):
+            raise ValueError("url must be an HTTP or HTTPS URL")
+        try:
+            parsed = urlsplit(self.url)
+            valid = parsed.scheme in {"http", "https"} and parsed.hostname and parsed.port != 0
+        except ValueError:
+            valid = False
+        if (
+            not valid
+            or parsed.username is not None
+            or parsed.password is not None
+            or parsed.query
+            or parsed.fragment
+            or any(c.isspace() for c in self.url)
+        ):
+            raise ValueError(
+                "url must be an HTTP or HTTPS URL without credentials, query, or fragment"
+            )
+        if self.quality not in ("low", "high"):
+            raise ValueError("quality must be low or high")
+        for name in ("timeout", "viewer_timeout"):
+            value = getattr(self, name)
+            if (
+                isinstance(value, bool)
+                or not isinstance(value, (int, float))
+                or not math.isfinite(value)
+                or value <= 0
+            ):
+                raise ValueError(f"{name} must be a finite positive number of seconds")
+        object.__setattr__(self, "url", self.url.rstrip("/"))

openspline-0.1.0/src/openspline/errors.py

@@ -0,0 +1,30 @@
+class OpensplineError(Exception):
+    def __init__(self, message, code="error"):
+        super().__init__(message)
+        self.code = code
+
+
+class CapacityError(OpensplineError):
+    pass
+
+
+class ConfigurationError(OpensplineError):
+    pass
+
+
+class ConnectionError(OpensplineError):
+    pass
+
+
+class InferenceError(OpensplineError):
+    pass
+
+
+def error_from(code, message):
+    cls = {
+        "capacity": CapacityError,
+        "configuration": ConfigurationError,
+        "connection": ConnectionError,
+        "inference": InferenceError,
+    }.get(code, OpensplineError)
+    return cls(message, code)

openspline-0.1.0/src/openspline/integrations/__init__.py

@@ -0,0 +1,6 @@
+"""Optional integrations. Import only the adapter your application uses."""
+
+from .elevenlabs import ElevenLabsAgents
+from .events import GeminiLive, GoogleADK, OpenAIAgents, OpenAIRealtime
+
+__all__ = ["ElevenLabsAgents", "GeminiLive", "GoogleADK", "OpenAIAgents", "OpenAIRealtime"]

openspline-0.1.0/src/openspline/integrations/elevenlabs.py

@@ -0,0 +1,72 @@
+"""ElevenLabs Agents WebSocket events, without a provider SDK dependency."""
+
+import base64
+import struct
+
+from .events import EventAdapter, value
+
+
+def audio_rate(audio_format):
+    if audio_format == "ulaw_8000":
+        return 8000
+    if audio_format not in {f"pcm_{rate}" for rate in (8000, 16000, 22050, 24000, 44100, 48000)}:
+        raise ValueError("Unsupported ElevenLabs audio format; use PCM or ulaw_8000.")
+    return int(audio_format[4:])
+
+
+def decode_audio(encoded, audio_format):
+    raw = base64.b64decode(encoded, validate=True)
+    if audio_format == "ulaw_8000":
+        samples = []
+        for byte in raw:
+            byte = ~byte & 255
+            sample = (((byte & 15) << 3) + 132) << ((byte >> 4) & 7)
+            samples.append(132 - sample if byte & 128 else sample - 132)
+        return struct.pack(f"<{len(samples)}h", *samples)
+    return raw
+
+
+class ElevenLabsAgents(EventAdapter):
+    """Forward decoded WebSocket events; the application owns pings, tools, and input.
+
+    Include the initiation metadata event to select the negotiated audio format.
+    If attaching after initiation, pass that format explicitly.
+    """
+
+    def __init__(self, avatar, *, audio_format="pcm_16000"):
+        super().__init__(avatar)
+        self.audio_format = audio_format
+        self.sample_rate = audio_rate(audio_format)
+        self.interrupted_id = -1
+        self.pending = False
+
+    async def handle(self, event):
+        kind = value(event, "type")
+        if kind == "conversation_initiation_metadata":
+            metadata = value(event, "conversation_initiation_metadata_event")
+            self.audio_format = value(metadata, "agent_output_audio_format")
+            self.sample_rate = audio_rate(self.audio_format)
+        elif kind == "interruption":
+            interruption = value(event, "interruption_event")
+            self.interrupted_id = max(self.interrupted_id, value(interruption, "event_id", -1))
+            self.pending = False
+            await self.avatar.interrupt()
+        elif kind == "audio":
+            audio = value(event, "audio_event")
+            if value(audio, "event_id", 0) <= self.interrupted_id:
+                return
+            raw = decode_audio(value(audio, "audio_base_64"), self.audio_format)
+            if raw:
+                await self.avatar.send_audio(raw, sample_rate=self.sample_rate)
+                self.pending = True
+            if value(audio, "is_final", False):
+                await self._finish()
+        elif kind == "agent_response_complete":
+            complete = value(event, "agent_response_complete_event")
+            if value(complete, "event_id", 0) > self.interrupted_id:
+                await self._finish()
+
+    async def _finish(self):
+        if self.pending:
+            await self.avatar.end_turn(drain=False)
+            self.pending = False

openspline-0.1.0/src/openspline/integrations/events.py

@@ -0,0 +1,153 @@
+import base64
+import re
+
+
+def value(obj, key, default=None):
+    return obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default)
+
+
+class EventAdapter:
+    def __init__(self, avatar):
+        self.avatar = avatar
+
+    async def wrap(self, events):
+        """Preserve every provider event for the application's existing event loop."""
+        async for event in events:
+            await self.handle(event)
+            yield event
+
+
+class OpenAIRealtime(EventAdapter):
+    def __init__(self, avatar, connection=None):
+        super().__init__(avatar)
+        self.connection = connection
+        self.item = None
+        self.content_index = 0
+        self.played = 0
+        self.generated = 0
+        self.base = 0
+        self.unsubscribe = avatar.on(self._playback)
+
+    def _playback(self, event):
+        if event.get("type") == "playback":
+            self.played = event["samples"] / 48000
+
+    async def handle(self, event):
+        kind = value(event, "type")
+        if kind == "response.output_audio.delta":
+            item = value(event, "item_id")
+            if item != self.item:
+                self.item = item
+                self.base = self.generated
+            self.content_index = value(event, "content_index", 0)
+            raw = base64.b64decode(value(event, "delta"))
+            self.generated += len(raw) / 48000
+            await self.avatar.send_audio(raw, sample_rate=24000)
+        elif kind == "response.output_audio.done":
+            await self.avatar.end_turn(drain=False)
+        elif kind == "input_audio_buffer.speech_started":
+            heard = max(0, self.played - self.base)
+            await self.avatar.interrupt()
+            if self.connection and self.item:
+                await self.connection.send(
+                    {
+                        "type": "conversation.item.truncate",
+                        "item_id": self.item,
+                        "content_index": self.content_index,
+                        "audio_end_ms": int(heard * 1000),
+                    }
+                )
+            self.item = None
+            self.generated = self.played = self.base = 0
+
+    def close(self):
+        self.unsubscribe()
+
+
+class OpenAIAgents(EventAdapter):
+    """Use .tracker in RealtimeRunner.run(model_config={"playback_tracker": ...})."""
+
+    def __init__(self, avatar, playback_tracker=None):
+        from agents.realtime import RealtimePlaybackTracker
+
+        super().__init__(avatar)
+        self.tracker = playback_tracker or RealtimePlaybackTracker()
+        self.spans = []
+        self.generated = 0
+        self.played = 0
+        self.unsubscribe = avatar.on(self._playback)
+
+    def _playback(self, event):
+        if event.get("type") != "playback":
+            return
+        new = event["samples"] / 48
+        for start, end, item, index in self.spans:
+            duration = max(0, min(new, end) - max(self.played, start))
+            if duration:
+                self.tracker.on_play_ms(item, index, duration)
+        self.played = max(self.played, new)
+        self.spans = [span for span in self.spans if span[1] > self.played]
+
+    async def handle(self, event):
+        kind = value(event, "type")
+        if kind == "audio":
+            audio = value(event, "audio")
+            raw = value(audio, "data", audio)
+            duration = len(raw) / 48
+            self.spans.append(
+                (
+                    self.generated,
+                    self.generated + duration,
+                    value(event, "item_id"),
+                    value(event, "content_index", 0),
+                )
+            )
+            self.generated += duration
+            await self.avatar.send_audio(raw, sample_rate=24000)
+        elif kind == "audio_end":
+            await self.avatar.end_turn(drain=False)
+        elif kind == "audio_interrupted":
+            await self.avatar.interrupt()
+            self.spans = []
+            self.generated = self.played = 0
+
+    def close(self):
+        self.unsubscribe()
+
+
+class GeminiLive(EventAdapter):
+    async def handle(self, event):
+        content = value(event, "server_content", value(event, "serverContent"))
+        if not content:
+            return
+        if value(content, "interrupted", False):
+            await self.avatar.interrupt()
+            return
+        turn = value(content, "model_turn", value(content, "modelTurn"))
+        for part in value(turn, "parts", []) or []:
+            blob = value(part, "inline_data", value(part, "inlineData"))
+            mime = value(blob, "mime_type", value(blob, "mimeType", ""))
+            if blob and mime.startswith("audio/"):
+                raw = value(blob, "data")
+                raw = base64.b64decode(raw) if isinstance(raw, str) else raw
+                match = re.search(r"rate=(\d+)", mime)
+                await self.avatar.send_audio(raw, sample_rate=int(match[1]) if match else 24000)
+        if value(content, "turn_complete", value(content, "turnComplete", False)):
+            await self.avatar.end_turn(drain=False)
+
+
+class GoogleADK(EventAdapter):
+    async def handle(self, event):
+        if value(event, "interrupted", False):
+            await self.avatar.interrupt()
+            return
+        for part in value(value(event, "content"), "parts", []) or []:
+            blob = value(part, "inline_data")
+            mime = value(blob, "mime_type", "")
+            if blob and mime.startswith("audio/"):
+                match = re.search(r"rate=(\d+)", mime)
+                await self.avatar.send_audio(
+                    value(blob, "data"), sample_rate=int(match[1]) if match else 24000
+                )
+        if value(event, "turn_complete", False):
+            await self.avatar.end_turn(drain=False)

openspline-0.1.0/src/openspline/integrations/livekit.py

@@ -0,0 +1,68 @@
+"""LiveKit native avatar plugin. Install the server's LiveKit extra as well."""
+
+import os
+import secrets
+
+from livekit import api
+from livekit.agents.voice.avatar import AvatarSession as BaseAvatarSession
+from livekit.agents.voice.avatar import DataStreamAudioOutput
+
+from openspline import Openspline
+
+
+class AvatarSession(BaseAvatarSession):
+    def __init__(
+        self, portrait, quality=None, client=None, url=None, api_key=None, api_secret=None
+    ):
+        super().__init__()
+        self.client = client or Openspline()
+        self.portrait = portrait
+        self.quality = quality
+        self.url = url or os.environ.get("LIVEKIT_URL")
+        self.key = api_key or os.environ.get("LIVEKIT_API_KEY")
+        self.secret = api_secret or os.environ.get("LIVEKIT_API_SECRET")
+        self.identity = "openspline-" + secrets.token_hex(6)
+        self.avatar = None
+
+    @property
+    def avatar_identity(self):
+        return self.identity
+
+    @property
+    def provider(self):
+        return "openspline"
+
+    async def start(self, agent_session, room):
+        await super().start(agent_session, room)
+        self.avatar = await self.client.avatar(self.portrait, self.quality).__aenter__()
+        token = (
+            api.AccessToken(self.key, self.secret)
+            .with_identity(self.identity)
+            .with_kind("agent")
+            .with_attributes({"lk.publish_on_behalf": room.local_participant.identity})
+            .with_grants(api.VideoGrants(room_join=True, room=room.name))
+            .to_jwt()
+        )
+        try:
+            response = await self.avatar.http.post(
+                f"/v1/sessions/{self.avatar.id}/livekit",
+                headers=self.avatar._headers(),
+                json={
+                    "url": self.url,
+                    "token": token,
+                    "sender_identity": room.local_participant.identity,
+                },
+            )
+            self.avatar._check(response)
+            agent_session.output.audio = DataStreamAudioOutput(
+                room, destination_identity=self.identity, wait_playback_start=True
+            )
+        except BaseException:
+            await self.aclose()
+            raise
+
+    async def aclose(self):
+        if self.avatar:
+            await self.avatar.close()
+            self.avatar = None
+        await super().aclose()

openspline-0.1.0/src/openspline/integrations/pipecat.py

@@ -0,0 +1,87 @@
+"""Place after TTS/realtime output and before transport.output()."""
+
+import asyncio
+
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    Frame,
+    InterruptionFrame,
+    OutputImageRawFrame,
+    StartFrame,
+    TTSAudioRawFrame,
+    TTSStoppedFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+from openspline import Openspline
+
+
+class AvatarProcessor(FrameProcessor):
+    def __init__(self, portrait, quality=None, client=None, **kwargs):
+        super().__init__(**kwargs)
+        self.client = client or Openspline()
+        self.portrait = portrait
+        self.quality = quality
+        self.avatar = None
+        self.task = None
+
+    async def _output(self):
+        import av
+
+        resampler = av.AudioResampler(format="s16", layout="mono", rate=48000)
+        media = await self.avatar.media()
+        async for kind, frame in media:
+            if kind == "audio":
+                for out in resampler.resample(frame):
+                    await self.push_frame(
+                        TTSAudioRawFrame(
+                            audio=out.to_ndarray().tobytes(), sample_rate=48000, num_channels=1
+                        )
+                    )
+                media.acknowledge()
+            else:
+                out = OutputImageRawFrame(
+                    image=frame.to_ndarray(format="rgb24").tobytes(),
+                    size=(frame.width, frame.height),
+                    format="RGB",
+                )
+                if frame.pts is not None:
+                    out.pts = int(frame.pts * frame.time_base * 1e9)
+                await self.push_frame(out)
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+        if direction != FrameDirection.DOWNSTREAM:
+            await self.push_frame(frame, direction)
+            return
+        if isinstance(frame, StartFrame):
+            self.avatar = await self.client.avatar(self.portrait, self.quality).__aenter__()
+            await self.push_frame(frame, direction)
+            self.task = asyncio.create_task(self._output())
+            return
+        if isinstance(frame, TTSAudioRawFrame):
+            await self.avatar.send_audio(
+                frame.audio, sample_rate=frame.sample_rate, channels=frame.num_channels
+            )
+            return
+        if isinstance(frame, TTSStoppedFrame):
+            await self.avatar.end_turn()
+        if isinstance(frame, InterruptionFrame):
+            await self.avatar.interrupt()
+        if isinstance(frame, (EndFrame, CancelFrame)):
+            await self._close()
+        await self.push_frame(frame, direction)
+
+    async def _close(self):
+        if self.task:
+            self.task.cancel()
+            await asyncio.gather(self.task, return_exceptions=True)
+            self.task = None
+        if self.avatar:
+            await self.avatar.close()
+            self.avatar = None
+
+    async def cleanup(self):
+        await self._close()
+        await super().cleanup()

openspline-0.1.0/src/openspline/integrations/ten.py

@@ -0,0 +1,45 @@
+"""TEN extension using its public AsyncExtension and AudioFrame interfaces."""
+
+import json
+
+from ten_runtime import AsyncExtension, CmdResult, Data, StatusCode
+
+from openspline import Openspline
+
+
+class AvatarExtension(AsyncExtension):
+    async def on_start(self, env):
+        raw, error = await env.get_property_to_json("")
+        if error:
+            raise ValueError("Cannot read openspline properties")
+        config = json.loads(raw)
+        self.avatar = (
+            await Openspline().avatar(config["portrait"], config.get("quality", "low")).__aenter__()
+        )
+        descriptor = Data.create("avatar_session")
+        descriptor.set_property_from_json("session", json.dumps(self.avatar.session))
+        await env.send_data(descriptor)
+
+    async def on_audio_frame(self, env, frame):
+        await self.avatar.send_audio(
+            bytes(frame.get_buf()),
+            sample_rate=frame.get_sample_rate(),
+            channels=frame.get_number_of_channels(),
+        )
+
+    async def on_data(self, env, data):
+        if data.get_name() == "tts_audio_end":
+            await self.avatar.end_turn(drain=False)
+        elif data.get_name() == "flush":
+            await self.avatar.interrupt()
+
+    async def on_cmd(self, env, cmd):
+        if cmd.get_name() == "flush":
+            await self.avatar.interrupt()
+        elif cmd.get_name() == "end_turn":
+            await self.avatar.end_turn()
+        await env.return_result(CmdResult.create(StatusCode.OK, cmd))
+
+    async def on_stop(self, env):
+        if getattr(self, "avatar", None):
+            await self.avatar.close()

openspline-0.1.0/src/openspline/integrations/vocode.py

@@ -0,0 +1,83 @@
+"""Vocode output device for the pinned current AudioChunk worker API."""
+
+import asyncio
+from collections import deque
+
+from vocode.streaming.models.audio import AudioEncoding
+from vocode.streaming.output_device.abstract_output_device import AbstractOutputDevice
+from vocode.streaming.output_device.audio_chunk import ChunkState
+
+
+class AvatarOutputDevice(AbstractOutputDevice):
+    """Use inside an open avatar context; its viewer owns output playback."""
+
+    def __init__(self, avatar, sampling_rate=24000, max_pending_seconds=10):
+        super().__init__(sampling_rate, AudioEncoding.LINEAR16)
+        self._input_queue = asyncio.Queue(maxsize=64)
+        self.avatar = avatar
+        self.pending = deque()
+        self.samples = 0
+        self.played = 0
+        self.limit = int(max_pending_seconds * 48000)
+        self.changed = asyncio.Event()
+        self.unsubscribe = avatar.on(self._feedback)
+        self.tasks = set()
+
+    def _feedback(self, event):
+        if event.get("type") == "playback":
+            self.played = event["samples"]
+            while self.pending and self.pending[0][0] <= self.played:
+                _, event = self.pending.popleft()
+                chunk = event.payload
+                if event.is_interrupted():
+                    chunk.state = ChunkState.INTERRUPTED
+                    chunk.on_interrupt()
+                else:
+                    chunk.state = ChunkState.PLAYED
+                    chunk.on_play()
+            self.changed.set()
+        elif event.get("type") == "interrupted":
+            self._clear()
+
+    def _clear(self):
+        while self.pending:
+            _, event = self.pending.popleft()
+            event.payload.state = ChunkState.INTERRUPTED
+            event.payload.on_interrupt()
+        self.samples = self.played = 0
+        self.changed.set()
+
+    async def _run_loop(self):
+        while True:
+            event = await self._input_queue.get()
+            if event.is_interrupted():
+                event.payload.state = ChunkState.INTERRUPTED
+                event.payload.on_interrupt()
+                continue
+            while self.samples - self.played > self.limit:
+                self.changed.clear()
+                await self.changed.wait()
+            self.samples += round(len(event.payload.data) / 2 / self.sampling_rate * 48000)
+            self.pending.append((self.samples, event))
+            await self.avatar.send_audio(event.payload.data, sample_rate=self.sampling_rate)
+            # Vocode output chunks have no turn boundary. Flush each chunk, retaining
+            # model history. This favors correctness for short utterances over throughput.
+            await self.avatar.end_turn(drain=False)
+
+    def interrupt(self):
+        self._clear()
+        while not self._input_queue.empty():
+            event = self._input_queue.get_nowait()
+            event.payload.state = ChunkState.INTERRUPTED
+            event.payload.on_interrupt()
+        task = asyncio.create_task(self.avatar.interrupt())
+        self.tasks.add(task)
+        task.add_done_callback(self.tasks.discard)
+
+    async def terminate(self):
+        self.unsubscribe()
+        self._clear()
+        for task in self.tasks:
+            task.cancel()
+        await asyncio.gather(*self.tasks, return_exceptions=True)
+        await super().terminate()

openspline-0.1.0/src/openspline/media.py

@@ -0,0 +1,88 @@
+"""Optional native media receiver for framework output adapters."""
+
+import asyncio
+import json
+
+from aiortc import RTCConfiguration, RTCIceServer, RTCPeerConnection, RTCSessionDescription
+
+
+class MediaReceiver:
+    def __init__(self, avatar):
+        self.avatar = avatar
+        self.pc = RTCPeerConnection()
+        self.frames = asyncio.Queue(100)
+        self.tasks = set()
+        self.clock = None
+
+    async def start(self):
+        response = await self.avatar.http.get(
+            f"/v1/sessions/{self.avatar.id}/ice",
+            headers={"Authorization": "Bearer " + self.avatar.session["token"]},
+        )
+        self.avatar._check(response)
+        await self.pc.close()
+        self.pc = RTCPeerConnection(
+            RTCConfiguration(iceServers=[RTCIceServer(**s) for s in response.json()["iceServers"]])
+        )
+        self.pc.addTransceiver("audio", direction="recvonly")
+        self.pc.addTransceiver("video", direction="recvonly")
+        self.channel = self.pc.createDataChannel("openspline")
+
+        @self.channel.on("open")
+        def opened():
+            self.channel.send(json.dumps({"type": "ready"}))
+
+        @self.channel.on("message")
+        def message(raw):
+            event = json.loads(raw)
+            if event["type"] == "clock":
+                self.clock = event
+            if event["type"] == "interrupted":
+                self.clock = None
+                while not self.frames.empty():
+                    self.frames.get_nowait()
+
+        @self.pc.on("track")
+        def track(track):
+            async def receive():
+                while True:
+                    frame = await track.recv()
+                    await self.frames.put((track.kind, frame))
+
+            task = asyncio.create_task(receive())
+            self.tasks.add(task)
+            task.add_done_callback(self.tasks.discard)
+
+        await self.pc.setLocalDescription(await self.pc.createOffer())
+        s = self.avatar.session
+        response = await self.avatar.http.post(
+            f"/v1/sessions/{self.avatar.id}/offer",
+            headers={"Authorization": "Bearer " + s["token"]},
+            json={"type": self.pc.localDescription.type, "sdp": self.pc.localDescription.sdp},
+        )
+        self.avatar._check(response)
+        await self.pc.setRemoteDescription(RTCSessionDescription(**response.json()))
+
+    def acknowledge(self):
+        if self.clock and self.channel.readyState == "open":
+            self.channel.send(
+                json.dumps(
+                    {
+                        "type": "played",
+                        "epoch": self.clock["epoch"],
+                        "samples": self.clock["samples"],
+                    }
+                )
+            )
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        return await self.frames.get()
+
+    async def close(self):
+        for task in self.tasks:
+            task.cancel()
+        await asyncio.gather(*self.tasks, return_exceptions=True)
+        await self.pc.close()