livekit-plugins-vakyam 1.8.0__py3-none-any.whl

livekit/plugins/vakyam/__init__.py

@@ -0,0 +1,45 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Vakyam AI plugin for LiveKit Agents
+
+Support for text-to-speech with [Vakyam AI](https://vakyam.ai/) Raaga 1.
+
+See https://docs.vakyam.ai/guides/realtime-websocket for protocol details.
+"""
+
+from .tts import TTS, ChunkedStream, SynthesizeStream
+from .version import __version__
+
+__all__ = ["TTS", "ChunkedStream", "SynthesizeStream", "__version__"]
+
+from livekit.agents import Plugin
+
+from .log import logger
+
+
+class VakyamPlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(VakyamPlugin())
+
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit/plugins/vakyam/_utils.py

@@ -0,0 +1,214 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+from typing import Any
+from urllib.parse import urlparse
+
+from livekit.agents import APIStatusError
+
+from .models import (
+    CUSTOM_VOICE_PREFIX,
+    MAX_SPEED,
+    MAX_TEXT_CHARACTERS,
+    MIN_SPEED,
+    SUPPORTED_LANGUAGES,
+    SUPPORTED_MODELS,
+    SUPPORTED_SAMPLE_RATES,
+    TTS_STREAM_PATH,
+    TTS_WEBSOCKET_PATH,
+)
+
+_RETRYABLE_WS_CODES = {
+    "rate_limit_exceeded",
+    "concurrency_limit_exceeded",
+    "tts_workers_busy",
+    "tts_workers_unconfigured",
+    "tts_workers_unavailable",
+    "internal_error",
+}
+
+
+def normalize_voice(voice: str) -> str:
+    """Normalize a preset voice name or custom ``vc_`` voice ID."""
+    if not isinstance(voice, str) or not voice.strip():
+        raise ValueError("voice must be a non-empty string")
+    normalized = voice.strip()
+    if normalized == CUSTOM_VOICE_PREFIX:
+        raise ValueError("custom voice IDs must include a value after 'vc_'")
+    return normalized
+
+
+def normalize_base_url(base_url: str, *, allow_insecure_base_url: bool = False) -> str:
+    """Normalize and validate a public API base URL."""
+    normalized = base_url.rstrip("/")
+    parsed = urlparse(normalized)
+    if parsed.scheme != "http":
+        return normalized
+
+    host = parsed.hostname or ""
+    if allow_insecure_base_url or host in {"localhost", "127.0.0.1", "::1"}:
+        return normalized
+
+    raise ValueError(
+        "base_url must use HTTPS unless it points to localhost. "
+        "Pass allow_insecure_base_url=True only for trusted development networks."
+    )
+
+
+def websocket_url(base_url: str) -> str:
+    """Build the Vakyam TTS WebSocket URL from an HTTP(S) or WS(S) base URL."""
+    root = base_url.rstrip("/")
+    if root.startswith("https://"):
+        root = "wss://" + root[len("https://") :]
+    elif root.startswith("http://"):
+        root = "ws://" + root[len("http://") :]
+    elif not root.startswith(("ws://", "wss://")):
+        root = "wss://" + root
+    if root.endswith(TTS_WEBSOCKET_PATH):
+        return root
+    return root + TTS_WEBSOCKET_PATH
+
+
+def http_stream_url(base_url: str) -> str:
+    """Build the Vakyam HTTP streaming TTS URL."""
+    return base_url.rstrip("/") + TTS_STREAM_PATH
+
+
+def validate_tts_options(
+    *,
+    model: str,
+    language: str,
+    sample_rate: int,
+    speed: float,
+    voice: str,
+) -> None:
+    """Validate constructor / update_options values."""
+    if model not in SUPPORTED_MODELS:
+        valid = ", ".join(sorted(SUPPORTED_MODELS))
+        raise ValueError(f"model '{model}' is not supported. Valid values are: {valid}.")
+    if language not in SUPPORTED_LANGUAGES:
+        valid = ", ".join(sorted(SUPPORTED_LANGUAGES))
+        raise ValueError(f"language '{language}' is not supported. Valid values are: {valid}.")
+    if sample_rate not in SUPPORTED_SAMPLE_RATES:
+        valid = ", ".join(str(v) for v in sorted(SUPPORTED_SAMPLE_RATES))
+        raise ValueError(
+            f"sample_rate '{sample_rate}' is not supported. Valid values are: {valid}."
+        )
+    if not MIN_SPEED <= speed <= MAX_SPEED:
+        raise ValueError(f"speed must be between {MIN_SPEED} and {MAX_SPEED}")
+    normalize_voice(voice)
+
+
+def validate_text(text: str) -> None:
+    """Validate utterance text for a single synthesis request."""
+    if not isinstance(text, str) or not text:
+        raise ValueError("text is required")
+    character_count = len(text)
+    if character_count > MAX_TEXT_CHARACTERS:
+        raise ValueError(
+            f"Input text is {character_count} characters. Maximum allowed is "
+            f"{MAX_TEXT_CHARACTERS} Unicode characters."
+        )
+
+
+def split_text(text: str, *, max_characters: int = MAX_TEXT_CHARACTERS) -> list[str]:
+    """Split an oversized utterance at whitespace, falling back to a hard boundary."""
+    if max_characters <= 0:
+        raise ValueError("max_characters must be greater than zero")
+
+    remaining = text.strip()
+    chunks: list[str] = []
+    while len(remaining) > max_characters:
+        split_at = remaining.rfind(" ", 0, max_characters + 1)
+        if split_at <= 0:
+            split_at = max_characters
+        chunk = remaining[:split_at].strip()
+        if chunk:
+            chunks.append(chunk)
+        remaining = remaining[split_at:].lstrip()
+    if remaining:
+        chunks.append(remaining)
+    return chunks
+
+
+def speech_payload(
+    *,
+    text: str,
+    model: str,
+    voice: str,
+    language: str,
+    sample_rate: int,
+    speed: float,
+    output_format: str = "pcm",
+) -> dict[str, Any]:
+    """JSON body for HTTP generate/stream requests."""
+    validate_text(text)
+    validate_tts_options(
+        model=model, language=language, sample_rate=sample_rate, speed=speed, voice=voice
+    )
+    return {
+        "text": text,
+        "model_id": model,
+        "voice": normalize_voice(voice),
+        "language": language,
+        "output_format": output_format,
+        "sample_rate": sample_rate,
+        "speed": speed,
+    }
+
+
+def raise_http_error(status: int, body: str) -> None:
+    """Raise ``APIStatusError`` from a Vakyam HTTP error envelope."""
+    parsed: object | None = None
+    try:
+        parsed = json.loads(body) if body else None
+    except json.JSONDecodeError:
+        parsed = None
+
+    error_code: str | None = None
+    if isinstance(parsed, dict) and isinstance(parsed.get("error"), dict):
+        error = parsed["error"]
+        code = error.get("code")
+        if isinstance(code, (str, int)):
+            error_code = str(code)
+
+    message = f"Vakyam TTS request failed with status {status}"
+    safe_body: dict[str, object] = {"status_code": status}
+    if error_code is not None:
+        message += f" (error code: {error_code})"
+        safe_body["error_code"] = error_code
+
+    raise APIStatusError(message, status_code=status, body=safe_body)
+
+
+def raise_ws_error(data: dict[str, Any]) -> None:
+    """Raise ``APIStatusError`` from a Vakyam WebSocket ``error`` frame."""
+    error = data.get("error") if isinstance(data.get("error"), dict) else {}
+    code = error.get("code") if isinstance(error, dict) else None
+    error_code = str(code) if isinstance(code, (str, int)) else None
+    retryable = error_code in _RETRYABLE_WS_CODES
+    message = "Vakyam TTS WebSocket request failed"
+    safe_body: dict[str, str] = {"type": "error"}
+    if error_code is not None:
+        message += f" (error code: {error_code})"
+        safe_body["code"] = error_code
+    raise APIStatusError(
+        message,
+        status_code=-1,
+        body=safe_body,
+        retryable=retryable,
+    )

livekit/plugins/vakyam/_websocket.py

@@ -0,0 +1,384 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Vakyam realtime WebSocket protocol for sentence-wise TTS.
+
+Copied from the Vakyam Python SDK protocol (config → text → binary* →
+end_of_utterance, with cancel + drain on barge-in). Implemented here directly
+so this plugin does not wrap ``vakyamai``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from collections.abc import AsyncIterator
+from contextlib import suppress
+from dataclasses import dataclass
+from typing import Any
+
+from livekit.agents import (
+    APIConnectionError,
+    APIStatusError,
+    APITimeoutError,
+    __version__ as livekit_version,
+)
+
+from ._utils import (
+    normalize_base_url,
+    normalize_voice,
+    raise_ws_error,
+    validate_text,
+    validate_tts_options,
+    websocket_url,
+)
+from .models import DEFAULT_BASE_URL, DEFAULT_LANGUAGE, DEFAULT_MODEL, DEFAULT_SAMPLE_RATE
+from .version import __version__
+
+USER_AGENT = f"LiveKit-Agents-Vakyam/{__version__} livekit-agents/{livekit_version}"
+
+
+@dataclass(frozen=True)
+class TTSSessionConfig:
+    """Immutable synthesis settings applied once per WebSocket session."""
+
+    model: str = DEFAULT_MODEL
+    voice: str = "Archana"
+    language: str = DEFAULT_LANGUAGE
+    sample_rate: int = DEFAULT_SAMPLE_RATE
+    speed: float = 1.0
+    output_format: str = "pcm"
+
+    def to_wire_message(self) -> dict[str, Any]:
+        return {
+            "type": "config",
+            "model_id": self.model,
+            "voice": normalize_voice(self.voice),
+            "language": self.language,
+            "output_format": self.output_format,
+            "sample_rate": self.sample_rate,
+            "speed": self.speed,
+        }
+
+
+@dataclass(frozen=True)
+class WebSocketSpeechResult:
+    """Terminal result for one utterance (``end_of_utterance`` or ``cancellation``)."""
+
+    characters_used: int
+    duration_seconds: float
+    truncated: bool = False
+    truncation_reason: str | None = None
+    cancelled: bool = False
+
+
+def text_message(text: str) -> dict[str, str]:
+    validate_text(text)
+    return {"type": "text", "text": text}
+
+
+def cancel_message() -> dict[str, str]:
+    return {"type": "cancel"}
+
+
+def ping_message() -> dict[str, str]:
+    return {"type": "ping"}
+
+
+def disconnect_message() -> dict[str, str]:
+    return {"type": "disconnect"}
+
+
+def speech_result_from_terminal(data: dict[str, Any]) -> WebSocketSpeechResult:
+    msg_type = data.get("type")
+    if msg_type == "cancellation":
+        return WebSocketSpeechResult(
+            characters_used=int(data.get("characters_used", 0)),
+            duration_seconds=float(data.get("duration_seconds", 0.0)),
+            cancelled=True,
+        )
+    if msg_type == "end_of_utterance":
+        return WebSocketSpeechResult(
+            characters_used=int(data["characters_used"]),
+            duration_seconds=float(data["duration_seconds"]),
+            truncated=bool(data.get("truncated", False)),
+            truncation_reason=(str(data["reason"]) if data.get("reason") is not None else None),
+        )
+    raise APIConnectionError(f"Not a terminal WebSocket message type: {msg_type}")
+
+
+class AsyncStreamingTTSSession:
+    """Low-level async WebSocket session that streams PCM audio per utterance.
+
+    Protocol:
+      connected → config → configured →
+      (text → binary* → end_of_utterance | cancel → binary* → cancellation)* →
+      disconnect
+
+    After ``cancel``, keep receiving until ``cancellation`` (drain trailing
+    audio). Do not send the next ``text`` until then. The socket stays open.
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        config: TTSSessionConfig,
+        base_url: str | None = None,
+        allow_insecure_base_url: bool = False,
+    ) -> None:
+        if not api_key:
+            raise ValueError("api_key is required")
+        validate_tts_options(
+            model=config.model,
+            language=config.language,
+            sample_rate=config.sample_rate,
+            speed=config.speed,
+            voice=config.voice,
+        )
+        resolved_base = normalize_base_url(
+            base_url or DEFAULT_BASE_URL, allow_insecure_base_url=allow_insecure_base_url
+        )
+        self._api_key = api_key
+        self._config = config
+        self._url = websocket_url(resolved_base)
+        self._connection: Any = None
+        self._utterance_active = False
+        self.last_result: WebSocketSpeechResult | None = None
+
+    @property
+    def sample_rate(self) -> int:
+        return int(self._config.sample_rate)
+
+    @property
+    def connected(self) -> bool:
+        return self._connection is not None
+
+    @property
+    def utterance_active(self) -> bool:
+        """True while waiting for ``end_of_utterance`` or ``cancellation``."""
+        return self._utterance_active
+
+    async def __aenter__(self) -> AsyncStreamingTTSSession:
+        await self.connect()
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: object
+    ) -> None:
+        await self.close()
+
+    async def connect(self, *, timeout: float = 10.0) -> None:
+        if self._connection is not None:
+            return
+        try:
+            from websockets.asyncio.client import connect
+        except ImportError as exc:
+            raise APIConnectionError("websockets is required for Vakyam TTS streaming") from exc
+
+        async def _connect_and_configure() -> None:
+            self._connection = await connect(
+                self._url,
+                additional_headers={
+                    "Authorization": f"Bearer {self._api_key}",
+                    "User-Agent": USER_AGENT,
+                },
+                open_timeout=timeout,
+                close_timeout=timeout,
+            )
+            connected = json.loads(await self._connection.recv())
+            if connected.get("type") != "connected":
+                raise APIConnectionError("Unexpected WebSocket handshake response")
+
+            await self._connection.send(json.dumps(self._config.to_wire_message()))
+            configured = json.loads(await self._connection.recv())
+            if configured.get("type") == "error":
+                raise_ws_error(configured)
+            if configured.get("type") != "configured":
+                raise APIConnectionError("Unexpected WebSocket config response")
+
+        try:
+            await asyncio.wait_for(_connect_and_configure(), timeout=timeout)
+        except asyncio.TimeoutError as exc:
+            await self.close()
+            raise APITimeoutError("Vakyam TTS WebSocket connection timed out") from exc
+        except (APIStatusError, APIConnectionError):
+            await self.close()
+            raise
+        except Exception as exc:
+            await self.close()
+            raise _websocket_connection_error(
+                exc, message="Vakyam TTS WebSocket connection failed"
+            ) from exc
+
+    async def synthesize_stream(self, text: str, *, timeout: float = 10.0) -> AsyncIterator[bytes]:
+        """Send one utterance and yield raw audio until EOU or cancellation.
+
+        If this coroutine is cancelled mid-stream (LiveKit barge-in), it sends
+        ``cancel``, drains until ``cancellation``, then re-raises so the
+        WebSocket can be reused.
+        """
+        if self._connection is None:
+            raise APIConnectionError("WebSocket session is not connected")
+
+        payload = json.dumps(text_message(text))
+        self.last_result = None
+        self._utterance_active = True
+
+        try:
+            await asyncio.wait_for(self._connection.send(payload), timeout=timeout)
+            while True:
+                message = await asyncio.wait_for(self._connection.recv(), timeout=timeout)
+                if isinstance(message, bytes):
+                    if message:
+                        yield message
+                    continue
+
+                data = json.loads(message)
+                msg_type = data.get("type")
+                if msg_type == "error":
+                    raise_ws_error(data)
+                if msg_type == "pong":
+                    continue
+                if msg_type in {"end_of_utterance", "cancellation"}:
+                    self.last_result = speech_result_from_terminal(data)
+                    return
+                raise APIConnectionError(f"Unexpected WebSocket message type: {msg_type}")
+        except asyncio.CancelledError:
+            await asyncio.shield(self._abort_and_drain(timeout=timeout))
+            raise
+        except asyncio.TimeoutError as exc:
+            raise APITimeoutError("Vakyam TTS WebSocket receive timed out") from exc
+        except (APIStatusError, APIConnectionError, APITimeoutError):
+            raise
+        except Exception as exc:
+            raise _websocket_connection_error(
+                exc, message="Vakyam TTS WebSocket synthesis failed"
+            ) from exc
+        finally:
+            self._utterance_active = False
+
+    async def cancel(
+        self, *, drain: bool = True, timeout: float = 10.0
+    ) -> WebSocketSpeechResult | None:
+        """Send barge-in ``cancel`` and optionally drain until ``cancellation``."""
+        if self._connection is None:
+            raise APIConnectionError("WebSocket session is not connected")
+
+        try:
+            await asyncio.wait_for(
+                self._connection.send(json.dumps(cancel_message())), timeout=timeout
+            )
+        except asyncio.TimeoutError as exc:
+            raise APITimeoutError("Vakyam TTS WebSocket cancellation timed out") from exc
+        if not drain:
+            return None
+        return await self._drain_until_terminal(timeout=timeout)
+
+    async def ping(self, *, timeout: float = 10.0) -> bool:
+        if self._connection is None:
+            raise APIConnectionError("WebSocket session is not connected")
+        try:
+            await asyncio.wait_for(
+                self._connection.send(json.dumps(ping_message())), timeout=timeout
+            )
+            response = json.loads(await asyncio.wait_for(self._connection.recv(), timeout=timeout))
+        except asyncio.TimeoutError as exc:
+            raise APITimeoutError("Vakyam TTS WebSocket ping timed out") from exc
+        except Exception as exc:
+            raise _websocket_connection_error(
+                exc, message="Vakyam TTS WebSocket ping failed"
+            ) from exc
+
+        if response.get("type") == "error":
+            raise_ws_error(response)
+        if response.get("type") != "pong":
+            raise APIConnectionError(
+                f"Unexpected Vakyam TTS WebSocket ping response: {response.get('type')}"
+            )
+        return True
+
+    async def close(self, *, timeout: float = 10.0) -> None:
+        if self._connection is None:
+            return
+        try:
+            with suppress(Exception):
+                await asyncio.wait_for(
+                    self._connection.send(json.dumps(disconnect_message())), timeout=timeout
+                )
+            with suppress(Exception):
+                await asyncio.wait_for(self._connection.close(), timeout=timeout)
+        finally:
+            self._connection = None
+            self._utterance_active = False
+
+    async def _abort_and_drain(self, *, timeout: float) -> None:
+        """Best-effort cancel + drain after the consumer task was cancelled."""
+        if self._connection is None:
+            return
+        with suppress(Exception):
+            await asyncio.wait_for(
+                self._connection.send(json.dumps(cancel_message())), timeout=timeout
+            )
+        with suppress(Exception):
+            self.last_result = await self._drain_until_terminal(timeout=timeout)
+
+    async def _drain_until_terminal(self, *, timeout: float) -> WebSocketSpeechResult:
+        if self._connection is None:
+            raise APIConnectionError("WebSocket session is not connected")
+
+        while True:
+            try:
+                message = await asyncio.wait_for(self._connection.recv(), timeout=timeout)
+            except asyncio.TimeoutError as exc:
+                raise APITimeoutError("Vakyam TTS WebSocket cancellation timed out") from exc
+            if isinstance(message, bytes):
+                continue
+            data = json.loads(message)
+            msg_type = data.get("type")
+            if msg_type == "error":
+                raise_ws_error(data)
+            if msg_type == "pong":
+                continue
+            if msg_type in {"end_of_utterance", "cancellation"}:
+                result = speech_result_from_terminal(data)
+                self.last_result = result
+                return result
+            raise APIConnectionError(
+                f"Unexpected WebSocket message type while draining: {msg_type}"
+            )
+
+
+def _websocket_connection_error(
+    exc: Exception, *, message: str
+) -> APIConnectionError | APIStatusError:
+    """Translate WebSocket close codes without exposing request headers."""
+    received = getattr(exc, "rcvd", None)
+    code = getattr(received, "code", None)
+    if code is None:
+        code = getattr(exc, "code", None)
+
+    if code == 4001:
+        return APIStatusError(
+            "Vakyam TTS authentication failed",
+            status_code=401,
+            retryable=False,
+        )
+    if isinstance(code, int):
+        return APIStatusError(
+            f"{message} (WebSocket closed with code {code})",
+            status_code=code,
+            retryable=code in {1011, 4002},
+        )
+    return APIConnectionError(f"{message}: {type(exc).__name__}")

livekit/plugins/vakyam/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.vakyam")

livekit/plugins/vakyam/models.py

@@ -0,0 +1,42 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Literal
+
+TTSModels = Literal["raaga-v1"]
+TTSLanguages = Literal["bn-IN", "en-IN", "gu-IN", "hi-IN", "kn-IN", "mr-IN", "ta-IN", "te-IN"]
+TTSSampleRates = Literal[8000, 16000, 24000, 48000]
+
+DEFAULT_MODEL: TTSModels = "raaga-v1"
+DEFAULT_VOICE = "Archana"
+DEFAULT_LANGUAGE: TTSLanguages = "ta-IN"
+DEFAULT_SAMPLE_RATE: TTSSampleRates = 24000
+DEFAULT_SPEED = 1.0
+DEFAULT_BASE_URL = "https://api.vakyam.ai"
+
+TTS_STREAM_PATH = "/v1/tts/stream"
+TTS_WEBSOCKET_PATH = "/v1/tts/websocket"
+MAX_TEXT_CHARACTERS = 3000
+CUSTOM_VOICE_PREFIX = "vc_"
+MIN_SPEED = 0.5
+MAX_SPEED = 2.0
+
+# Application-level keepalive. The server closes idle sockets after ~60s.
+KEEPALIVE_INTERVAL_SECONDS = 20.0
+
+SUPPORTED_MODELS: frozenset[str] = frozenset({"raaga-v1"})
+SUPPORTED_LANGUAGES: frozenset[str] = frozenset(
+    {"bn-IN", "en-IN", "gu-IN", "hi-IN", "kn-IN", "mr-IN", "ta-IN", "te-IN"}
+)
+SUPPORTED_SAMPLE_RATES: frozenset[int] = frozenset({8000, 16000, 24000, 48000})

livekit/plugins/vakyam/tts.py

@@ -0,0 +1,512 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Text-to-Speech implementation for Vakyam AI (Raaga 1).
+
+Streaming uses the realtime WebSocket API with sentence tokenization because
+Vakyam expects one complete utterance per ``text`` message. One-shot
+``synthesize()`` uses HTTP ``POST /v1/tts/stream`` for PCM bytes.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import weakref
+from contextlib import suppress
+from dataclasses import dataclass, replace
+
+import aiohttp
+
+from livekit.agents import (
+    APIConnectionError,
+    APIConnectOptions,
+    APIStatusError,
+    APITimeoutError,
+    tokenize,
+    tts,
+    utils,
+)
+from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS, NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import is_given
+
+from ._utils import (
+    http_stream_url,
+    normalize_base_url,
+    raise_http_error,
+    speech_payload,
+    split_text,
+    validate_tts_options,
+)
+from ._websocket import AsyncStreamingTTSSession, TTSSessionConfig
+from .log import logger
+from .models import (
+    DEFAULT_BASE_URL,
+    DEFAULT_LANGUAGE,
+    DEFAULT_MODEL,
+    DEFAULT_SAMPLE_RATE,
+    DEFAULT_SPEED,
+    DEFAULT_VOICE,
+    KEEPALIVE_INTERVAL_SECONDS,
+    TTSLanguages,
+    TTSModels,
+    TTSSampleRates,
+)
+from .version import __version__
+
+NUM_CHANNELS = 1
+
+
+@dataclass
+class _TTSOptions:
+    model: TTSModels | str
+    voice: str
+    language: TTSLanguages | str
+    sample_rate: TTSSampleRates | int
+    speed: float
+    api_key: str
+    base_url: str
+    allow_insecure_base_url: bool
+
+
+class TTS(tts.TTS):
+    """Vakyam AI text-to-speech for LiveKit Agents.
+
+    Uses the realtime WebSocket API (``WS /v1/tts/websocket``) for low-latency
+    streaming. Text is sentence-tokenized because Vakyam expects one complete
+    utterance per synthesis request.
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        model: TTSModels | str = DEFAULT_MODEL,
+        voice: str = DEFAULT_VOICE,
+        language: TTSLanguages | str = DEFAULT_LANGUAGE,
+        sample_rate: TTSSampleRates | int = DEFAULT_SAMPLE_RATE,
+        speed: float = DEFAULT_SPEED,
+        base_url: str | None = None,
+        allow_insecure_base_url: bool = False,
+        tokenizer: NotGivenOr[tokenize.SentenceTokenizer] = NOT_GIVEN,
+        http_session: aiohttp.ClientSession | None = None,
+    ) -> None:
+        """Create a Vakyam TTS instance.
+
+        Args:
+            api_key: Vakyam API key. Defaults to ``VAKYAM_API_KEY``.
+            model: TTS model id. Currently ``raaga-v1``.
+            voice: Voice selector — a preset name from ``GET /v1/voices``, or a
+                custom voice ID beginning with ``vc_``.
+            language: BCP-47 language code (for example ``ta-IN``).
+            sample_rate: Output PCM sample rate in Hz (8000, 16000, 24000, 48000).
+            speed: Speech rate multiplier (``0.5``–``2.0``).
+            base_url: API base URL. Defaults to ``https://api.vakyam.ai``.
+            allow_insecure_base_url: Allow non-localhost ``http://`` base URLs.
+            tokenizer: Sentence tokenizer used for streaming synthesis.
+            http_session: Optional aiohttp session for HTTP ``synthesize()``.
+        """
+        resolved_key = api_key or os.environ.get("VAKYAM_API_KEY")
+        if not resolved_key:
+            raise ValueError("Vakyam API key is required, either as api_key= or VAKYAM_API_KEY")
+
+        validate_tts_options(
+            model=str(model),
+            language=str(language),
+            sample_rate=int(sample_rate),
+            speed=speed,
+            voice=voice,
+        )
+
+        super().__init__(
+            capabilities=tts.TTSCapabilities(streaming=True, aligned_transcript=False),
+            sample_rate=int(sample_rate),
+            num_channels=NUM_CHANNELS,
+        )
+
+        self._opts = _TTSOptions(
+            model=model,
+            voice=voice,
+            language=language,
+            sample_rate=sample_rate,
+            speed=speed,
+            api_key=resolved_key,
+            base_url=normalize_base_url(
+                base_url or DEFAULT_BASE_URL, allow_insecure_base_url=allow_insecure_base_url
+            ),
+            allow_insecure_base_url=allow_insecure_base_url,
+        )
+        if is_given(tokenizer):
+            self._sentence_tokenizer = tokenizer
+        else:
+            try:
+                self._sentence_tokenizer = tokenize.blingfire.SentenceTokenizer()
+            except Exception:
+                self._sentence_tokenizer = tokenize.basic.SentenceTokenizer()
+        self._session = http_session
+        self._streams = weakref.WeakSet[SynthesizeStream]()
+        self._pools: dict[TTSSessionConfig, utils.ConnectionPool[AsyncStreamingTTSSession]] = {}
+        self._ws_keepalive_tasks: dict[AsyncStreamingTTSSession, asyncio.Task[None]] = {}
+
+    @property
+    def model(self) -> str:
+        return str(self._opts.model)
+
+    @property
+    def provider(self) -> str:
+        return "Vakyam"
+
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if not self._session:
+            self._session = utils.http_context.http_session()
+        return self._session
+
+    def update_options(
+        self,
+        *,
+        model: TTSModels | str | None = None,
+        voice: str | None = None,
+        language: TTSLanguages | str | None = None,
+        sample_rate: TTSSampleRates | int | None = None,
+        speed: float | None = None,
+    ) -> None:
+        """Update synthesis options for streams created after this call."""
+        next_opts = replace(self._opts)
+        next_sample_rate = self._sample_rate
+        if model is not None:
+            next_opts.model = model
+        if voice is not None:
+            next_opts.voice = voice
+        if language is not None:
+            next_opts.language = language
+        if sample_rate is not None:
+            next_opts.sample_rate = sample_rate
+            next_sample_rate = int(sample_rate)
+        if speed is not None:
+            next_opts.speed = speed
+
+        validate_tts_options(
+            model=str(next_opts.model),
+            language=str(next_opts.language),
+            sample_rate=int(next_opts.sample_rate),
+            speed=next_opts.speed,
+            voice=next_opts.voice,
+        )
+        self._opts = next_opts
+        self._sample_rate = next_sample_rate
+
+    def synthesize(
+        self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
+    ) -> ChunkedStream:
+        return ChunkedStream(tts=self, input_text=text, conn_options=conn_options)
+
+    def stream(
+        self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
+    ) -> SynthesizeStream:
+        stream = SynthesizeStream(tts=self, conn_options=conn_options)
+        self._streams.add(stream)
+        return stream
+
+    def prewarm(self) -> None:
+        self._pool_for(self._opts).prewarm()
+
+    async def aclose(self) -> None:
+        for stream in list(self._streams):
+            await stream.aclose()
+        self._streams.clear()
+        for pool in self._pools.values():
+            await pool.aclose()
+        self._pools.clear()
+
+    def _session_config(self, opts: _TTSOptions) -> TTSSessionConfig:
+        return TTSSessionConfig(
+            model=str(opts.model),
+            voice=opts.voice,
+            language=str(opts.language),
+            sample_rate=int(opts.sample_rate),
+            speed=opts.speed,
+            output_format="pcm",
+        )
+
+    def _pool_for(self, opts: _TTSOptions) -> utils.ConnectionPool[AsyncStreamingTTSSession]:
+        config = self._session_config(opts)
+        existing_pool = self._pools.get(config)
+        if existing_pool is not None:
+            return existing_pool
+
+        pool_ref: utils.ConnectionPool[AsyncStreamingTTSSession] | None = None
+
+        async def _connect(timeout: float) -> AsyncStreamingTTSSession:
+            session = AsyncStreamingTTSSession(
+                api_key=opts.api_key,
+                base_url=opts.base_url,
+                allow_insecure_base_url=opts.allow_insecure_base_url,
+                config=config,
+            )
+            await session.connect(timeout=timeout)
+            assert pool_ref is not None
+            self._start_keepalive(session, pool_ref)
+            return session
+
+        pool = utils.ConnectionPool(
+            connect_cb=_connect,
+            close_cb=self._close_ws,
+            max_session_duration=3600,
+            mark_refreshed_on_get=False,
+        )
+        pool_ref = pool
+        self._pools[config] = pool
+        return pool
+
+    async def _close_ws(self, session: AsyncStreamingTTSSession) -> None:
+        await self._stop_keepalive(session)
+        await session.close()
+
+    def _start_keepalive(
+        self,
+        session: AsyncStreamingTTSSession,
+        pool: utils.ConnectionPool[AsyncStreamingTTSSession],
+    ) -> None:
+        task = self._ws_keepalive_tasks.get(session)
+        if task is not None and not task.done():
+            return
+        self._ws_keepalive_tasks[session] = asyncio.create_task(
+            self._keepalive_loop(session, pool), name="vakyam-tts-ws-keepalive"
+        )
+
+    async def _stop_keepalive(self, session: AsyncStreamingTTSSession) -> None:
+        task = self._ws_keepalive_tasks.pop(session, None)
+        if task is None:
+            return
+        task.cancel()
+        with suppress(asyncio.CancelledError):
+            await task
+
+    async def _keepalive_loop(
+        self,
+        session: AsyncStreamingTTSSession,
+        pool: utils.ConnectionPool[AsyncStreamingTTSSession],
+    ) -> None:
+        try:
+            while True:
+                await asyncio.sleep(KEEPALIVE_INTERVAL_SECONDS)
+                if not session.connected:
+                    return
+                try:
+                    await session.ping()
+                except Exception as exc:
+                    logger.debug(
+                        "Vakyam TTS keepalive failed (%s); evicting session",
+                        type(exc).__name__,
+                    )
+                    pool.remove(session)
+                    return
+        except asyncio.CancelledError:
+            return
+        finally:
+            current = asyncio.current_task()
+            if self._ws_keepalive_tasks.get(session) is current:
+                self._ws_keepalive_tasks.pop(session, None)
+
+
+class ChunkedStream(tts.ChunkedStream):
+    """One-shot synthesis over HTTP ``POST /v1/tts/stream``."""
+
+    def __init__(self, *, tts: TTS, input_text: str, conn_options: APIConnectOptions) -> None:
+        super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
+        self._tts: TTS = tts
+        self._opts = replace(tts._opts)
+
+    async def _run(self, output_emitter: tts.AudioEmitter) -> None:
+        request_id = utils.shortuuid()
+        payload = speech_payload(
+            text=self._input_text,
+            model=str(self._opts.model),
+            voice=self._opts.voice,
+            language=str(self._opts.language),
+            sample_rate=int(self._opts.sample_rate),
+            speed=self._opts.speed,
+            output_format="pcm",
+        )
+        headers = {
+            "Authorization": f"Bearer {self._opts.api_key}",
+            "Content-Type": "application/json",
+            "User-Agent": f"LiveKit-Agents-Vakyam/{__version__}",
+        }
+        try:
+            async with self._tts._ensure_session().post(
+                http_stream_url(self._opts.base_url),
+                json=payload,
+                headers=headers,
+                timeout=aiohttp.ClientTimeout(
+                    total=None,
+                    sock_connect=self._conn_options.timeout,
+                    sock_read=self._conn_options.timeout,
+                ),
+            ) as resp:
+                if resp.status >= 400:
+                    body = await resp.text()
+                    raise_http_error(resp.status, body)
+
+                content_type = resp.headers.get("Content-Type", "").lower()
+                if content_type and not (
+                    content_type.startswith("audio/")
+                    or content_type.startswith("application/octet-stream")
+                ):
+                    body = await resp.text()
+                    raise APIStatusError(
+                        "Vakyam TTS returned a non-audio response",
+                        status_code=502,
+                        body=body,
+                    )
+
+                output_emitter.initialize(
+                    request_id=request_id,
+                    sample_rate=int(self._opts.sample_rate),
+                    num_channels=NUM_CHANNELS,
+                    mime_type="audio/pcm",
+                )
+                async for chunk, _ in resp.content.iter_chunks():
+                    if chunk:
+                        output_emitter.push(chunk)
+                output_emitter.flush()
+        except (APIStatusError, APIConnectionError, APITimeoutError):
+            raise
+        except asyncio.TimeoutError as exc:
+            raise APITimeoutError("Vakyam TTS HTTP stream timed out") from exc
+        except aiohttp.ClientError as exc:
+            raise APIConnectionError(f"Vakyam TTS HTTP stream connection error: {exc}") from exc
+
+
+class SynthesizeStream(tts.SynthesizeStream):
+    """Streaming synthesis with sentence tokenization over a persistent WebSocket."""
+
+    def __init__(self, *, tts: TTS, conn_options: APIConnectOptions) -> None:
+        super().__init__(tts=tts, conn_options=conn_options)
+        self._tts: TTS = tts
+        self._opts = replace(tts._opts)
+
+    async def _run(self, output_emitter: tts.AudioEmitter) -> None:
+        segments_ch = utils.aio.Chan[tokenize.SentenceStream]()
+        request_id = utils.shortuuid()
+        output_emitter.initialize(
+            request_id=request_id,
+            sample_rate=int(self._opts.sample_rate),
+            num_channels=NUM_CHANNELS,
+            mime_type="audio/pcm",
+            stream=True,
+            frame_size_ms=50,
+        )
+
+        async def _tokenize_input() -> None:
+            sentence_stream: tokenize.SentenceStream | None = None
+            async for data in self._input_ch:
+                if isinstance(data, str):
+                    if sentence_stream is None:
+                        sentence_stream = self._tts._sentence_tokenizer.stream()
+                        segments_ch.send_nowait(sentence_stream)
+                    sentence_stream.push_text(data)
+                elif isinstance(data, self._FlushSentinel):
+                    if sentence_stream is not None:
+                        sentence_stream.end_input()
+                        sentence_stream = None
+            if sentence_stream is not None:
+                sentence_stream.end_input()
+            segments_ch.close()
+
+        async def _process_segments() -> None:
+            async for sentence_stream in segments_ch:
+                await self._run_segment(sentence_stream, output_emitter)
+
+        tasks = [
+            asyncio.create_task(_tokenize_input()),
+            asyncio.create_task(_process_segments()),
+        ]
+        try:
+            await asyncio.gather(*tasks)
+        except (APIStatusError, APIConnectionError, APITimeoutError):
+            raise
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            raise APIConnectionError(f"Vakyam TTS stream failed: {exc}") from exc
+        finally:
+            await utils.aio.gracefully_cancel(*tasks)
+            output_emitter.end_input()
+
+    async def _run_segment(
+        self, sentence_stream: tokenize.SentenceStream, output_emitter: tts.AudioEmitter
+    ) -> None:
+        segment_id = utils.shortuuid()
+        output_emitter.start_segment(segment_id=segment_id)
+        pool = self._tts._pool_for(self._opts)
+        deferred_error: APIStatusError | None = None
+        cancelled = False
+
+        async with pool.connection(timeout=self._conn_options.timeout) as session:
+            self._acquire_time = pool.last_acquire_time
+            self._connection_reused = pool.last_connection_reused
+            await self._tts._stop_keepalive(session)
+            reusable = False
+            try:
+                started = False
+                async for sentence in sentence_stream:
+                    for text in split_text(sentence.token):
+                        if not started:
+                            self._mark_started()
+                            started = True
+
+                        async for chunk in session.synthesize_stream(
+                            text, timeout=self._conn_options.timeout
+                        ):
+                            output_emitter.push(chunk)
+
+                        if session.last_result and session.last_result.cancelled:
+                            logger.debug("Vakyam TTS utterance cancelled")
+                            break
+                        if session.last_result and session.last_result.truncated:
+                            deferred_error = APIStatusError(
+                                "Vakyam TTS utterance was truncated",
+                                status_code=500,
+                                retryable=False,
+                            )
+                            break
+                    if deferred_error is not None or (
+                        session.last_result and session.last_result.cancelled
+                    ):
+                        break
+                reusable = True
+            except asyncio.CancelledError:
+                # Reuse only after Vakyam acknowledges cancel and trailing audio is drained.
+                reusable = bool(session.last_result and session.last_result.cancelled)
+                if not reusable:
+                    raise
+                cancelled = True
+            except APIStatusError as exc:
+                # Only JSON per-message errors leave the WebSocket open. Close-code
+                # status errors represent a dead transport and must be evicted.
+                if isinstance(exc.body, dict) and exc.body.get("type") == "error":
+                    reusable = True
+                    deferred_error = exc
+                else:
+                    raise
+            finally:
+                if reusable:
+                    self._tts._start_keepalive(session, pool)
+
+        if cancelled:
+            raise asyncio.CancelledError
+        if deferred_error is not None:
+            raise deferred_error
+        output_emitter.end_segment()

livekit/plugins/vakyam/version.py

@@ -0,0 +1,15 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "1.8.0"

livekit_plugins_vakyam-1.8.0.dist-info/METADATA

@@ -0,0 +1,80 @@
+Metadata-Version: 2.5
+Name: livekit-plugins-vakyam
+Version: 1.8.0
+Summary: LiveKit Agents plugin for Vakyam AI TTS (Raaga 1) — Indian-language text-to-speech
+Project-URL: Documentation, https://docs.livekit.io
+Project-URL: Website, https://livekit.io/
+Project-URL: Source, https://github.com/livekit/agents
+Author-email: LiveKit <hello@livekit.io>
+License-Expression: Apache-2.0
+Keywords: audio,indian-languages,livekit,raaga,realtime,text-to-speech,tts,vakyam,webrtc
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10.0
+Requires-Dist: livekit-agents[codecs]>=1.8.0
+Requires-Dist: websockets<16.0,>=14.0
+Description-Content-Type: text/markdown
+
+# Vakyam AI plugin for LiveKit Agents
+
+Support for voice synthesis with [Vakyam AI](https://vakyam.ai/) Raaga 1 —
+text-to-speech for Indian languages.
+
+See [https://docs.vakyam.ai/integrations/livekit](https://docs.vakyam.ai/integrations/livekit)
+for provider docs.
+
+## Installation
+
+```bash
+pip install livekit-plugins-vakyam
+```
+
+Or with the LiveKit Agents extra:
+
+```bash
+uv add "livekit-agents[vakyam]"
+```
+
+## Pre-requisites
+
+You'll need an API key from [Vakyam](https://dashboard.vakyam.ai/api-keys).
+Set it as an environment variable:
+
+```bash
+export VAKYAM_API_KEY="vak_live_..."
+```
+
+## Usage
+
+```python
+from livekit.agents import AgentSession
+from livekit.plugins import vakyam
+
+session = AgentSession(
+    tts=vakyam.TTS(
+        model="raaga-v1",
+        voice="Archana",
+        language="ta-IN",
+        sample_rate=24000,
+    ),
+    # ... stt, llm, vad
+)
+```
+
+`stream()` uses the realtime WebSocket API and sentence-tokenizes LLM text so
+each utterance is one complete sentence (Vakyam does not accept partial
+tokens). `synthesize()` uses HTTP streaming (`POST /v1/tts/stream`) and
+returns PCM audio.
+
+WebSocket connections are pooled and reused between sequential agent turns.
+Each active synthesis stream has exclusive ownership of its connection, so an
+overlapping stream uses a separate connection. On interruption, the plugin
+sends `cancel`, drains through Vakyam's cancellation acknowledgement, and
+returns the healthy connection to the pool.

livekit_plugins_vakyam-1.8.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+livekit/plugins/vakyam/__init__.py,sha256=veB5JV1El0O2H1LRXiG9ajxFWWLkijuxHsJHT068oZQ,1294
+livekit/plugins/vakyam/_utils.py,sha256=Tmo3BLnMwx6deAOq8ijjl9yHgbpDtL4d-4XI8_IaW0Q,7132
+livekit/plugins/vakyam/_websocket.py,sha256=5z59N82zEoR7sQjpLTLFcH2-rByYcZliYHvr9-P_7-E,14093
+livekit/plugins/vakyam/log.py,sha256=rSyzLdpYK33mxW_1ZuxZ6oCLgJs0UWQpkc12mpjEQqA,69
+livekit/plugins/vakyam/models.py,sha256=Njf3AwJ1iR_PeGXE0Zk5VrBvd2C3ZRPqDfdfhmFmcCo,1539
+livekit/plugins/vakyam/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/vakyam/tts.py,sha256=hdZG-9wtKuqJo3Eua8A4tbLT0N0f7YXTUe5Kj8OZGXk,19008
+livekit/plugins/vakyam/version.py,sha256=IeTDs-sIIzY-W7jelf8ozYz_UfMZ153wLnSwlRVky74,600
+livekit_plugins_vakyam-1.8.0.dist-info/METADATA,sha256=YBJFSzQLRFskdPH9pFT5aq48NDhRLD9Q0yPogwJpMJM,2568
+livekit_plugins_vakyam-1.8.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+livekit_plugins_vakyam-1.8.0.dist-info/RECORD,,

livekit_plugins_vakyam-1.8.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any