vision-agents-plugins-sarvam 0.5.3__tar.gz

vision_agents_plugins_sarvam-0.5.3/.gitignore

@@ -0,0 +1,101 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.cursor/*
+# Distribution / packaging
+.Python
+build/
+dist/
+downloads/
+develop-eggs/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+pip-wheel-metadata/
+MANIFEST
+*.egg-info/
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+coverage.xml
+nosetests.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Type checker / lint caches
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pytype/
+.pyre/
+.ruff_cache/
+
+# Environments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.env
+.env.local
+.env.*.local
+.env.bak
+pyvenv.cfg
+.python-version
+
+# Editors / IDEs
+.vscode/
+.idea/
+
+# Jupyter Notebook
+.ipynb_checkpoints/
+
+# OS / Misc
+.DS_Store
+*.log
+
+# Tooling & repo-specific
+pyrightconfig.json
+shell.nix
+bin/*
+lib/*
+stream-py/
+
+# Example lock files (regenerated by uv sync)
+examples/*/uv.lock
+plugins/*/example/uv.lock
+
+# Artifacts / assets
+*.pt
+*.kef
+*.onnx
+profile.html
+
+/opencode.json
+.ralph-tui/
+.claude/
+
+.uv-cache/
+
+# pytest json report
+.report.json

vision_agents_plugins_sarvam-0.5.3/PKG-INFO

@@ -0,0 +1,58 @@
+Metadata-Version: 2.4
+Name: vision-agents-plugins-sarvam
+Version: 0.5.3
+Summary: Sarvam AI STT, TTS, and LLM integration for Vision Agents
+Project-URL: Documentation, https://visionagents.ai/
+Project-URL: Website, https://visionagents.ai/
+Project-URL: Source, https://github.com/GetStream/Vision-Agents
+License-Expression: MIT
+Keywords: AI,LLM,STT,TTS,agents,indian-languages,sarvam,speech-to-text,text-to-speech,voice agents
+Requires-Python: >=3.10
+Requires-Dist: aiohttp>=3.13.3
+Requires-Dist: vision-agents
+Requires-Dist: vision-agents-plugins-openai
+Description-Content-Type: text/markdown
+
+# Sarvam AI Plugin
+
+This plugin provides STT, TTS, and LLM capabilities using Sarvam AI, a suite of
+AI models built for Indian languages.
+
+## Features
+
+- **STT**: WebSocket streaming speech-to-text (Saarika / Saaras) with Voice
+  Activity Detection for turn events.
+- **TTS**: WebSocket streaming text-to-speech (Bulbul) with configurable
+  speaker, pace, and language.
+- **LLM**: OpenAI-compatible chat completions (Sarvam-30B / Sarvam-105B /
+  Sarvam-M) via the existing `ChatCompletionsLLM` from the OpenAI plugin.
+
+## Installation
+
+```bash
+uv add vision-agents-plugins-sarvam
+```
+
+## Usage
+
+```python
+from vision_agents.core import Agent, User
+from vision_agents.plugins import getstream, sarvam, smart_turn
+
+agent = Agent(
+    edge=getstream.Edge(),
+    agent_user=User(name="Sarvam AI"),
+    instructions="Reply in Hindi or English, whichever the user speaks",
+    llm=sarvam.LLM(model="sarvam-30b"),
+    stt=sarvam.STT(language="hi-IN"),
+    tts=sarvam.TTS(speaker="shubh"),
+    turn_detection=smart_turn.TurnDetection(),
+)
+```
+
+All three services read the same `SARVAM_API_KEY` environment variable and send
+it via the `api-subscription-key` header.
+
+## References
+
+- [Sarvam API docs](https://docs.sarvam.ai/)

vision_agents_plugins_sarvam-0.5.3/README.md

@@ -0,0 +1,43 @@
+# Sarvam AI Plugin
+
+This plugin provides STT, TTS, and LLM capabilities using Sarvam AI, a suite of
+AI models built for Indian languages.
+
+## Features
+
+- **STT**: WebSocket streaming speech-to-text (Saarika / Saaras) with Voice
+  Activity Detection for turn events.
+- **TTS**: WebSocket streaming text-to-speech (Bulbul) with configurable
+  speaker, pace, and language.
+- **LLM**: OpenAI-compatible chat completions (Sarvam-30B / Sarvam-105B /
+  Sarvam-M) via the existing `ChatCompletionsLLM` from the OpenAI plugin.
+
+## Installation
+
+```bash
+uv add vision-agents-plugins-sarvam
+```
+
+## Usage
+
+```python
+from vision_agents.core import Agent, User
+from vision_agents.plugins import getstream, sarvam, smart_turn
+
+agent = Agent(
+    edge=getstream.Edge(),
+    agent_user=User(name="Sarvam AI"),
+    instructions="Reply in Hindi or English, whichever the user speaks",
+    llm=sarvam.LLM(model="sarvam-30b"),
+    stt=sarvam.STT(language="hi-IN"),
+    tts=sarvam.TTS(speaker="shubh"),
+    turn_detection=smart_turn.TurnDetection(),
+)
+```
+
+All three services read the same `SARVAM_API_KEY` environment variable and send
+it via the `api-subscription-key` header.
+
+## References
+
+- [Sarvam API docs](https://docs.sarvam.ai/)

vision_agents_plugins_sarvam-0.5.3/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vision-agents-plugins-sarvam"
+dynamic = ["version"]
+description = "Sarvam AI STT, TTS, and LLM integration for Vision Agents"
+readme = "README.md"
+keywords = [
+    "sarvam",
+    "STT",
+    "TTS",
+    "LLM",
+    "speech-to-text",
+    "text-to-speech",
+    "indian-languages",
+    "AI",
+    "voice agents",
+    "agents",
+]
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+    "vision-agents",
+    "vision-agents-plugins-openai",
+    "aiohttp>=3.13.3",
+]
+
+[project.urls]
+Documentation = "https://visionagents.ai/"
+Website = "https://visionagents.ai/"
+Source = "https://github.com/GetStream/Vision-Agents"
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "..", search_parent_directories = true, fallback_version = "0.0.0" }
+
+[tool.hatch.build.targets.wheel]
+packages = [".", "vision_agents"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/vision_agents"]
+
+[tool.uv.sources]
+vision-agents = { workspace = true }
+vision-agents-plugins-openai = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "pytest-asyncio>=1.0.0",
+]

vision_agents_plugins_sarvam-0.5.3/vision_agents/plugins/sarvam/__init__.py

@@ -0,0 +1,5 @@
+from .llm import SarvamLLM as LLM
+from .stt import STT
+from .tts import TTS, SarvamTTSError
+
+__all__ = ["LLM", "STT", "TTS", "SarvamTTSError"]

vision_agents_plugins_sarvam-0.5.3/vision_agents/plugins/sarvam/llm.py

@@ -0,0 +1,232 @@
+"""Sarvam AI LLM using the OpenAI-compatible Chat Completions endpoint.
+
+Sarvam exposes ``/v1/chat/completions`` with the same shape as OpenAI, so we
+point an ``AsyncOpenAI`` client at Sarvam's base URL and inject the
+``api-subscription-key`` header. Streaming, tool calling, and conversation
+history are all inherited from :class:`ChatCompletionsLLM`.
+
+Sarvam-m supports "hybrid thinking" which emits ``<think>…</think>`` blocks
+before the actual answer. This plugin strips those blocks from the streamed
+output so they don't reach TTS.
+
+Docs: https://docs.sarvam.ai/api-reference-docs/chat/chat-completions
+"""
+
+import logging
+import os
+import re
+import time
+from typing import Any, Dict, List, Optional, cast
+
+from openai import AsyncStream
+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
+from vision_agents.core.llm.events import (
+    LLMResponseChunkEvent,
+    LLMResponseCompletedEvent,
+)
+from vision_agents.core.llm.llm import LLMResponseEvent
+from vision_agents.core.llm.llm_types import NormalizedToolCallItem
+from vision_agents.plugins.openai import ChatCompletionsLLM
+
+from openai import AsyncOpenAI
+
+logger = logging.getLogger(__name__)
+
+SARVAM_BASE_URL = "https://api.sarvam.ai/v1"
+DEFAULT_MODEL = "sarvam-m"
+SUPPORTED_MODELS = {"sarvam-m", "sarvam-30b", "sarvam-105b"}
+
+PLUGIN_NAME = "sarvam"
+
+_THINK_RE = re.compile(r"<think>.*?</think>", re.DOTALL)
+
+
+class _ThinkTagFilter:
+    """Streaming filter that strips ``<think>…</think>`` blocks.
+
+    Feed each streamed delta via :meth:`feed` and use the return value
+    (possibly empty) as the filtered delta to emit.
+    """
+
+    def __init__(self) -> None:
+        self._inside = False
+        self._buf = ""
+
+    def feed(self, delta: str) -> str:
+        """Process *delta* and return the portion that should be emitted."""
+        self._buf += delta
+        out_parts: list[str] = []
+
+        while self._buf:
+            if self._inside:
+                end = self._buf.find("</think>")
+                if end == -1:
+                    # Still inside — keep a partial ``</think>`` prefix so we
+                    # can detect the closing tag when it spans multiple chunks.
+                    lt = self._buf.rfind("<")
+                    if lt != -1 and "</think>".startswith(self._buf[lt:]):
+                        self._buf = self._buf[lt:]
+                    else:
+                        self._buf = ""
+                    break
+                # Skip past closing tag
+                self._buf = self._buf[end + len("</think>") :]
+                self._inside = False
+            else:
+                start = self._buf.find("<think>")
+                if start == -1:
+                    # No opening tag — check for a possible partial tag at the
+                    # end (e.g. "<thi") and keep it buffered.
+                    lt = self._buf.rfind("<")
+                    if lt != -1 and "<think>".startswith(self._buf[lt:]):
+                        out_parts.append(self._buf[:lt])
+                        self._buf = self._buf[lt:]
+                    else:
+                        out_parts.append(self._buf)
+                        self._buf = ""
+                    break
+                # Emit text before the tag, consume the tag
+                out_parts.append(self._buf[:start])
+                self._buf = self._buf[start + len("<think>") :]
+                self._inside = True
+
+        return "".join(out_parts)
+
+    def flush(self, text: str) -> str:
+        """Strip think tags from the final accumulated text."""
+        return _THINK_RE.sub("", text).strip()
+
+
+class SarvamLLM(ChatCompletionsLLM):
+    """Sarvam AI Chat Completions LLM.
+
+    Thin wrapper around :class:`ChatCompletionsLLM` that configures the OpenAI
+    client for Sarvam's OpenAI-compatible endpoint and strips ``<think>``
+    blocks from streamed output so TTS doesn't speak the reasoning text.
+
+    Examples:
+
+        from vision_agents.plugins import sarvam
+        llm = sarvam.LLM(model="sarvam-30b")
+    """
+
+    def __init__(
+        self,
+        model: str = DEFAULT_MODEL,
+        api_key: Optional[str] = None,
+        base_url: str = SARVAM_BASE_URL,
+        client: Optional[AsyncOpenAI] = None,
+    ) -> None:
+        """Initialize the Sarvam LLM.
+
+        Args:
+            model: The Sarvam model id. Defaults to ``sarvam-m``. Supported:
+                ``sarvam-m``, ``sarvam-30b``, ``sarvam-105b``.
+            api_key: Sarvam API key. Defaults to ``SARVAM_API_KEY`` env var.
+            base_url: API base URL. Defaults to ``https://api.sarvam.ai/v1``.
+            client: Optional pre-configured ``AsyncOpenAI`` client. Takes
+                precedence over ``api_key`` / ``base_url``.
+        """
+        resolved_key = (
+            api_key if api_key is not None else os.environ.get("SARVAM_API_KEY")
+        )
+        if client is None and not resolved_key:
+            raise ValueError(
+                "SARVAM_API_KEY env var or api_key parameter required for Sarvam LLM"
+            )
+
+        if client is None:
+            client = AsyncOpenAI(
+                api_key=resolved_key,
+                base_url=base_url,
+                default_headers={"api-subscription-key": resolved_key or ""},
+            )
+
+        super().__init__(model=model, client=client)
+
+    async def _process_streaming_response(
+        self,
+        response: Any,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]],
+        kwargs: Dict[str, Any],
+        request_start_time: float,
+    ) -> LLMResponseEvent:
+        """Process streaming response, stripping ``<think>`` blocks."""
+        llm_response: LLMResponseEvent = LLMResponseEvent(original=None, text="")
+        text_chunks: list[str] = []
+        total_text = ""
+        self._pending_tool_calls: Dict[int, Dict[str, Any]] = {}
+        accumulated_tool_calls: List[NormalizedToolCallItem] = []
+        seq = 0
+        first_token_time: Optional[float] = None
+        think_filter = _ThinkTagFilter()
+
+        async for chunk in cast(AsyncStream[ChatCompletionChunk], response):
+            if not chunk.choices:
+                continue
+
+            choice = chunk.choices[0]
+            content = choice.delta.content
+            finish_reason = choice.finish_reason
+
+            if choice.delta.tool_calls:
+                for tc in choice.delta.tool_calls:
+                    self._accumulate_tool_call_chunk(tc)
+
+            if content:
+                if first_token_time is None:
+                    first_token_time = time.perf_counter()
+
+                text_chunks.append(content)
+
+                filtered = think_filter.feed(content)
+                if filtered:
+                    is_first = seq == 0
+                    ttft_ms = None
+                    if is_first and first_token_time is not None:
+                        ttft_ms = (first_token_time - request_start_time) * 1000
+                    self.events.send(
+                        LLMResponseChunkEvent(
+                            plugin_name=PLUGIN_NAME,
+                            content_index=None,
+                            item_id=chunk.id,
+                            output_index=0,
+                            sequence_number=seq,
+                            delta=filtered,
+                            is_first_chunk=is_first,
+                            time_to_first_token_ms=ttft_ms,
+                        )
+                    )
+                    seq += 1
+
+            if finish_reason:
+                if finish_reason == "tool_calls":
+                    accumulated_tool_calls = self._finalize_pending_tool_calls()
+
+                total_text = think_filter.flush("".join(text_chunks))
+                latency_ms = (time.perf_counter() - request_start_time) * 1000
+                ttft_ms_final = None
+                if first_token_time is not None:
+                    ttft_ms_final = (first_token_time - request_start_time) * 1000
+
+                self.events.send(
+                    LLMResponseCompletedEvent(
+                        plugin_name=PLUGIN_NAME,
+                        original=chunk,
+                        text=total_text,
+                        item_id=chunk.id,
+                        latency_ms=latency_ms,
+                        time_to_first_token_ms=ttft_ms_final,
+                        model=self.model,
+                    )
+                )
+
+            llm_response = LLMResponseEvent(original=chunk, text=total_text)
+
+        if accumulated_tool_calls:
+            return await self._handle_tool_calls(
+                accumulated_tool_calls, messages, tools, kwargs
+            )
+
+        return llm_response

vision_agents_plugins_sarvam-0.5.3/vision_agents/plugins/sarvam/stt.py

@@ -0,0 +1,347 @@
+"""Sarvam AI Speech-to-Text via WebSocket streaming.
+
+Docs: https://docs.sarvam.ai/api-reference-docs/api-guides-tutorials/speech-to-text/streaming-api
+
+Supported models:
+    - ``saaras:v3`` (default, recommended) – transcription + translation
+    - ``saarika:v2.5`` – legacy transcription-only
+    - ``saaras:v2.5`` – legacy translation
+"""
+
+import asyncio
+import base64
+import json
+import logging
+import os
+import time
+from typing import Any, Optional
+from urllib.parse import urlencode
+
+import aiohttp
+from getstream.video.rtc.track_util import PcmData
+from vision_agents.core import stt
+from vision_agents.core.edge.types import Participant
+from vision_agents.core.stt import TranscriptResponse
+
+logger = logging.getLogger(__name__)
+
+WS_STT_URL = "wss://api.sarvam.ai/speech-to-text/ws"
+WS_STT_TRANSLATE_URL = "wss://api.sarvam.ai/speech-to-text-translate/ws"
+
+SUPPORTED_SAMPLE_RATES = {8000, 16000}
+SUPPORTED_MODES = {"transcribe", "translate", "verbatim", "translit", "codemix"}
+
+MODELS_USING_TRANSLATE_ENDPOINT = {"saaras:v2.5"}
+MODELS_SUPPORTING_PROMPT = {"saaras:v2.5", "saaras:v3"}
+MODELS_SUPPORTING_MODE = {"saaras:v3"}
+SUPPORTED_MODELS = {"saaras:v3", "saarika:v2.5", "saaras:v2.5"}
+
+
+class STT(stt.STT):
+    """Sarvam AI streaming Speech-to-Text.
+
+    Uses aiohttp for a fully-async WebSocket connection to Sarvam's streaming
+    endpoint. Audio is sent as base64-encoded PCM inside JSON messages.
+    Transcript and VAD events are emitted as STT and turn events.
+
+    Turn detection is supported natively via Sarvam's VAD signals
+    (``speech_start`` / ``speech_end``).
+    """
+
+    turn_detection: bool = True
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model: str = "saaras:v3",
+        language: Optional[str] = None,
+        mode: Optional[str] = None,
+        sample_rate: int = 16000,
+        high_vad_sensitivity: bool = False,
+        vad_signals: bool = True,
+        prompt: Optional[str] = None,
+    ) -> None:
+        """Initialize Sarvam STT.
+
+        Args:
+            api_key: Sarvam API key. Falls back to ``SARVAM_API_KEY`` env var.
+            model: Streaming model id. Defaults to ``saaras:v3``.
+            language: Language code (e.g. ``hi-IN``, ``en-IN``). ``None`` lets
+                Sarvam auto-detect.
+            mode: One of ``transcribe``, ``translate``, ``verbatim``,
+                ``translit``, ``codemix``. Saaras defaults are model-dependent.
+            sample_rate: Input sample rate, 8000 or 16000 Hz.
+            high_vad_sensitivity: Increase VAD sensitivity for noisy input.
+            vad_signals: Emit ``speech_start`` / ``speech_end`` events used
+                for turn detection.
+            prompt: Optional biasing prompt sent once after connect.
+        """
+        super().__init__(provider_name="sarvam")
+
+        if model not in SUPPORTED_MODELS:
+            raise ValueError(
+                f"Unsupported Sarvam STT model '{model}'. "
+                f"Expected one of: {sorted(SUPPORTED_MODELS)}"
+            )
+        if sample_rate not in SUPPORTED_SAMPLE_RATES:
+            raise ValueError(
+                f"Unsupported sample_rate {sample_rate}. "
+                f"Expected one of: {sorted(SUPPORTED_SAMPLE_RATES)}"
+            )
+        if mode is not None and mode not in SUPPORTED_MODES:
+            raise ValueError(
+                f"Unsupported mode '{mode}'. Expected one of: {sorted(SUPPORTED_MODES)}"
+            )
+
+        self._api_key = api_key or os.environ.get("SARVAM_API_KEY")
+        if not self._api_key:
+            raise ValueError(
+                "SARVAM_API_KEY env var or api_key parameter required for Sarvam STT"
+            )
+
+        self.model = model
+        self.language = language
+        self.mode = mode
+        self.sample_rate = sample_rate
+        self.high_vad_sensitivity = high_vad_sensitivity
+        self.vad_signals = vad_signals
+        self._prompt = prompt
+
+        self._session: Optional[aiohttp.ClientSession] = None
+        self._ws: Optional[aiohttp.ClientWebSocketResponse] = None
+        self._receive_task: Optional[asyncio.Task[Any]] = None
+        self._connection_ready = asyncio.Event()
+        self._current_participant: Optional[Participant] = None
+        self._audio_start_time: Optional[float] = None
+
+        self._in_speech: bool = False
+        self._pending_transcript: Optional[str] = None
+        self._pending_response: Optional[TranscriptResponse] = None
+        self._turn_end_pending: bool = False
+
+    def _build_ws_url(self) -> str:
+        base = (
+            WS_STT_TRANSLATE_URL
+            if self.model in MODELS_USING_TRANSLATE_ENDPOINT
+            else WS_STT_URL
+        )
+        params: dict[str, str | int] = {
+            "model": self.model,
+            "sample_rate": self.sample_rate,
+            "vad_signals": "true" if self.vad_signals else "false",
+        }
+        if self.language is not None:
+            params["language-code"] = self.language
+        if self.mode is not None and self.model in MODELS_SUPPORTING_MODE:
+            params["mode"] = self.mode
+        if self.high_vad_sensitivity:
+            params["high_vad_sensitivity"] = "true"
+        return f"{base}?{urlencode(params)}"
+
+    async def start(self) -> None:
+        """Open the Sarvam WebSocket and start the receive loop."""
+        await super().start()
+
+        url = self._build_ws_url()
+        headers = {"api-subscription-key": self._api_key or ""}
+
+        self._session = aiohttp.ClientSession()
+        self._ws = await self._session.ws_connect(url, headers=headers)
+
+        if self._prompt and self.model in MODELS_SUPPORTING_PROMPT:
+            await self._ws.send_str(
+                json.dumps({"type": "config", "prompt": self._prompt})
+            )
+
+        self._receive_task = asyncio.create_task(self._receive_loop())
+        self._connection_ready.set()
+
+    async def process_audio(
+        self,
+        pcm_data: PcmData,
+        participant: Participant,
+    ) -> None:
+        """Send a PCM audio chunk to Sarvam.
+
+        The chunk is resampled to the configured sample rate and wrapped in
+        the JSON schema expected by Sarvam's WebSocket.
+        """
+        if self.closed:
+            logger.warning("Sarvam STT is closed, ignoring audio")
+            return
+
+        await self._connection_ready.wait()
+
+        if self._ws is None or self._ws.closed:
+            logger.warning("Sarvam STT WebSocket not open, dropping audio")
+            return
+
+        resampled = pcm_data.resample(self.sample_rate, 1)
+        audio_bytes = resampled.samples.tobytes()
+
+        self._current_participant = participant
+        if self._audio_start_time is None:
+            self._audio_start_time = time.perf_counter()
+
+        message = {
+            "audio": {
+                "data": base64.b64encode(audio_bytes).decode("ascii"),
+                "encoding": "audio/wav",
+                "sample_rate": self.sample_rate,
+            }
+        }
+        await self._ws.send_str(json.dumps(message))
+
+    async def _receive_loop(self) -> None:
+        ws = self._ws
+        if ws is None:
+            return
+        try:
+            async for msg in ws:
+                if msg.type == aiohttp.WSMsgType.TEXT:
+                    try:
+                        parsed = json.loads(msg.data)
+                    except json.JSONDecodeError:
+                        logger.warning("Sarvam STT sent non-JSON text: %s", msg.data)
+                        continue
+                    if logger.isEnabledFor(logging.DEBUG):
+                        logger.debug("Sarvam STT message: %s", parsed)
+                    self._handle_message(parsed)
+                elif msg.type in (
+                    aiohttp.WSMsgType.CLOSED,
+                    aiohttp.WSMsgType.CLOSING,
+                    aiohttp.WSMsgType.ERROR,
+                ):
+                    break
+        except asyncio.CancelledError:
+            raise
+        except aiohttp.ClientError:
+            logger.exception("Sarvam STT receive loop error")
+
+        if not self.closed:
+            self._emit_error_event(
+                ConnectionError("Sarvam STT WebSocket closed unexpectedly"),
+                self._current_participant,
+                "sarvam_ws_closed",
+            )
+
+    def _handle_message(self, data: dict[str, Any]) -> None:
+        """Dispatch a parsed Sarvam WebSocket message.
+
+        Sarvam's streaming STT sends three message shapes:
+
+        - ``{"type": "events", "data": {"signal_type": "START_SPEECH" | "END_SPEECH"}}``
+          VAD boundaries used to drive turn events.
+        - ``{"type": "data", "data": {"transcript": "...", "language_code": ...}}``
+          Transcript updates during an utterance. Sarvam may send multiple
+          ``data`` messages per utterance as it refines the text. Only the
+          last one before ``END_SPEECH`` is treated as final.
+        - ``{"type": "error", ...}`` or any message with an ``error`` key.
+        """
+        msg_type = data.get("type", "")
+        payload = data.get("data") or {}
+        participant = self._current_participant
+
+        if msg_type == "events":
+            signal = payload.get("signal_type", "")
+            if participant is None:
+                return
+            if signal == "START_SPEECH":
+                self._in_speech = True
+                self._pending_transcript = None
+                self._pending_response = None
+                self._turn_end_pending = False
+                self._emit_turn_started_event(participant)
+            elif signal == "END_SPEECH":
+                self._in_speech = False
+                self._audio_start_time = None
+                if self._pending_transcript and self._pending_response:
+                    self._emit_transcript_event(
+                        self._pending_transcript,
+                        participant,
+                        self._pending_response,
+                    )
+                    self._pending_transcript = None
+                    self._pending_response = None
+                    self._emit_turn_ended_event(participant)
+                else:
+                    self._turn_end_pending = True
+            return
+
+        if msg_type == "error" or "error" in data:
+            err_msg = data.get("error") or payload.get("message") or "Sarvam STT error"
+            self._emit_error_event(
+                Exception(str(err_msg)),
+                participant,
+                "sarvam_streaming",
+            )
+            return
+
+        transcript_text = payload.get("transcript") or data.get("transcript") or ""
+        if not transcript_text:
+            return
+
+        if participant is None:
+            logger.warning("Sarvam transcript received but no participant set")
+            return
+
+        processing_time_ms: Optional[float] = None
+        if self._audio_start_time is not None:
+            processing_time_ms = (time.perf_counter() - self._audio_start_time) * 1000
+
+        language_code = (
+            payload.get("language_code")
+            or data.get("language_code")
+            or self.language
+            or "auto"
+        )
+        metrics = payload.get("metrics") or {}
+        audio_duration = metrics.get("audio_duration")
+        audio_duration_ms: Optional[int] = (
+            int(audio_duration * 1000) if audio_duration is not None else None
+        )
+
+        response = TranscriptResponse(
+            language=language_code,
+            model_name=self.model,
+            processing_time_ms=processing_time_ms,
+            audio_duration_ms=audio_duration_ms,
+        )
+
+        if self._in_speech:
+            self._pending_transcript = transcript_text
+            self._pending_response = response
+            self._emit_partial_transcript_event(transcript_text, participant, response)
+        elif self._turn_end_pending:
+            self._turn_end_pending = False
+            self._emit_transcript_event(transcript_text, participant, response)
+            self._emit_turn_ended_event(participant)
+        else:
+            self._emit_transcript_event(transcript_text, participant, response)
+
+    async def close(self) -> None:
+        """Send end_of_stream, close the WebSocket, and clean up."""
+        await super().close()
+
+        if self._ws is not None and not self._ws.closed:
+            try:
+                await self._ws.send_str(json.dumps({"type": "end_of_stream"}))
+            except (aiohttp.ClientError, ConnectionError):
+                logger.debug("Could not send end_of_stream to Sarvam")
+            await self._ws.close()
+        self._ws = None
+
+        if self._receive_task is not None:
+            self._receive_task.cancel()
+            try:
+                await self._receive_task
+            except asyncio.CancelledError:
+                pass
+            self._receive_task = None
+
+        if self._session is not None and not self._session.closed:
+            await self._session.close()
+        self._session = None
+
+        self._connection_ready.clear()
+        self._audio_start_time = None

vision_agents_plugins_sarvam-0.5.3/vision_agents/plugins/sarvam/tts.py

@@ -0,0 +1,340 @@
+"""Sarvam AI Text-to-Speech via WebSocket streaming.
+
+Docs: https://docs.sarvam.ai/api-reference-docs/api-guides-tutorials/text-to-speech/streaming-api
+
+The WebSocket stays open across ``stream_audio`` calls to avoid per-call
+connection overhead. Text is sent as a JSON message; audio chunks arrive as
+base64-encoded PCM which we decode into ``PcmData``.
+"""
+
+import asyncio
+import base64
+import json
+import logging
+import os
+from typing import Any, AsyncIterator, Optional
+
+import aiohttp
+from getstream.video.rtc.track_util import AudioFormat, PcmData
+from vision_agents.core import tts
+
+logger = logging.getLogger(__name__)
+
+WS_BASE_URL = "wss://api.sarvam.ai/text-to-speech/ws"
+
+SUPPORTED_MODELS = {"bulbul:v2", "bulbul:v3-beta", "bulbul:v3"}
+
+KEEPALIVE_INTERVAL_S = 20
+
+MODEL_SPEAKER_COMPATIBILITY: dict[str, set[str]] = {
+    "bulbul:v2": {
+        "anushka",
+        "manisha",
+        "vidya",
+        "arya",
+        "abhilash",
+        "karun",
+        "hitesh",
+    },
+    "bulbul:v3-beta": {
+        "shubh",
+        "ritu",
+        "rahul",
+        "pooja",
+        "simran",
+        "kavya",
+        "amit",
+        "ratan",
+        "rohan",
+        "dev",
+        "ishita",
+        "shreya",
+        "manan",
+        "sumit",
+        "priya",
+        "aditya",
+        "kabir",
+        "neha",
+        "varun",
+        "roopa",
+        "aayan",
+        "ashutosh",
+        "advait",
+        "amelia",
+        "sophia",
+    },
+    "bulbul:v3": {
+        "shubh",
+        "ritu",
+        "rahul",
+        "pooja",
+        "simran",
+        "kavya",
+        "amit",
+        "ratan",
+        "rohan",
+        "dev",
+        "ishita",
+        "shreya",
+        "manan",
+        "sumit",
+        "priya",
+        "aditya",
+        "kabir",
+        "neha",
+        "varun",
+        "roopa",
+        "aayan",
+        "ashutosh",
+        "advait",
+        "amelia",
+        "sophia",
+    },
+}
+
+MODELS_SUPPORTING_PITCH = {"bulbul:v2"}
+MODELS_SUPPORTING_LOUDNESS = {"bulbul:v2"}
+MODELS_SUPPORTING_TEMPERATURE = {"bulbul:v3-beta", "bulbul:v3"}
+
+
+class SarvamTTSError(Exception):
+    """Raised when Sarvam TTS returns an error message over WebSocket."""
+
+
+class TTS(tts.TTS):
+    """Sarvam AI streaming Text-to-Speech.
+
+    Keeps a persistent WebSocket open across synthesis calls. Sends a config
+    message on first connect, then text + flush.
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model: str = "bulbul:v3",
+        language: str = "hi-IN",
+        speaker: str = "shubh",
+        sample_rate: int = 24000,
+        pace: Optional[float] = None,
+        pitch: Optional[float] = None,
+        loudness: Optional[float] = None,
+        temperature: Optional[float] = None,
+        enable_preprocessing: bool = True,
+        idle_timeout: float = 5.0,
+    ) -> None:
+        """Initialize Sarvam TTS.
+
+        Args:
+            api_key: Sarvam API key. Falls back to ``SARVAM_API_KEY`` env var.
+            model: TTS model. Defaults to ``bulbul:v3``.
+            language: Target language code (e.g. ``hi-IN``, ``en-IN``).
+            speaker: Speaker voice id (e.g. ``shubh``, ``anushka``).
+            sample_rate: Output sample rate in Hz. Defaults to 24000.
+            pace: Speech pace. Range depends on model
+                (bulbul:v3 supports 0.5-2.0).
+            pitch: Speech pitch. Only supported on bulbul:v2.
+            loudness: Speech loudness. Only supported on bulbul:v2.
+            temperature: Sampling temperature. Only supported on
+                bulbul:v3 / bulbul:v3-beta.
+            enable_preprocessing: Normalize mixed-language / numeric text.
+            idle_timeout: Fallback seconds of server silence before treating
+                synthesis as complete. Normally the server sends an explicit
+                completion event; this is a safety net.
+        """
+        super().__init__(provider_name="sarvam")
+
+        if model not in SUPPORTED_MODELS:
+            raise ValueError(
+                f"Unsupported Sarvam TTS model '{model}'. "
+                f"Expected one of: {sorted(SUPPORTED_MODELS)}"
+            )
+
+        self._api_key = api_key or os.environ.get("SARVAM_API_KEY")
+        if not self._api_key:
+            raise ValueError(
+                "SARVAM_API_KEY env var or api_key parameter required for Sarvam TTS"
+            )
+
+        compatible = MODEL_SPEAKER_COMPATIBILITY.get(model)
+        if compatible is not None and speaker not in compatible:
+            raise ValueError(
+                f"Speaker '{speaker}' is not compatible with model '{model}'. "
+                f"Compatible speakers: {sorted(compatible)}"
+            )
+
+        self.model = model
+        self.language = language
+        self.speaker = speaker
+        self.sample_rate = sample_rate
+        self.pace = pace
+        self.pitch = pitch
+        self.loudness = loudness
+        self.temperature = temperature
+        self.enable_preprocessing = enable_preprocessing
+        self._idle_timeout = idle_timeout
+
+        self._session: Optional[aiohttp.ClientSession] = None
+        self._ws: Optional[aiohttp.ClientWebSocketResponse] = None
+        self._lock = asyncio.Lock()
+        self._stop_event = asyncio.Event()
+        self._keepalive_task: Optional[asyncio.Task[None]] = None
+
+    async def start(self) -> None:
+        """Open the persistent WebSocket connection."""
+        await self._ensure_connection()
+
+    async def close(self) -> None:
+        """Close the WebSocket and release the aiohttp session."""
+        await self._reset_connection()
+        await super().close()
+
+    async def stream_audio(
+        self, text: str, *_: Any, **__: Any
+    ) -> AsyncIterator[PcmData]:
+        """Stream TTS audio chunks for ``text`` over the persistent WebSocket.
+
+        Returns:
+            Async iterator yielding ``PcmData`` chunks.
+        """
+
+        async def _stream() -> AsyncIterator[PcmData]:
+            self._stop_event.clear()
+            async with self._lock:
+                ws = await self._ensure_connection()
+                await ws.send_str(json.dumps({"type": "text", "data": {"text": text}}))
+                await ws.send_str(json.dumps({"type": "flush"}))
+                async for chunk in self._receive_audio(ws):
+                    yield chunk
+
+        return _stream()
+
+    async def stop_audio(self) -> None:
+        """Cancel any in-flight synthesis and tear down the connection."""
+        self._stop_event.set()
+        if self._ws is not None and not self._ws.closed:
+            try:
+                await self._ws.send_str(json.dumps({"type": "cancel"}))
+            except (aiohttp.ClientError, ConnectionError):
+                pass
+        await self._reset_connection()
+
+    async def _ensure_connection(self) -> aiohttp.ClientWebSocketResponse:
+        if self._ws is not None and not self._ws.closed:
+            return self._ws
+
+        if self._session is None or self._session.closed:
+            self._session = aiohttp.ClientSession()
+
+        url = f"{WS_BASE_URL}?model={self.model}&send_completion_event=true"
+        headers = {"api-subscription-key": self._api_key or ""}
+        ws = await self._session.ws_connect(url, headers=headers)
+
+        config: dict[str, Any] = {
+            "model": self.model,
+            "target_language_code": self.language,
+            "speaker": self.speaker,
+            "speech_sample_rate": self.sample_rate,
+            "enable_preprocessing": self.enable_preprocessing,
+            "output_audio_codec": "linear16",
+        }
+        if self.pace is not None:
+            config["pace"] = self.pace
+        if self.pitch is not None and self.model in MODELS_SUPPORTING_PITCH:
+            config["pitch"] = self.pitch
+        if self.loudness is not None and self.model in MODELS_SUPPORTING_LOUDNESS:
+            config["loudness"] = self.loudness
+        if self.temperature is not None and self.model in MODELS_SUPPORTING_TEMPERATURE:
+            config["temperature"] = self.temperature
+
+        await ws.send_str(json.dumps({"type": "config", "data": config}))
+        self._ws = ws
+        self._start_keepalive()
+        logger.debug("Sarvam TTS websocket connected at %dHz", self.sample_rate)
+        return ws
+
+    def _start_keepalive(self) -> None:
+        self._stop_keepalive()
+        self._keepalive_task = asyncio.create_task(self._keepalive_loop())
+
+    def _stop_keepalive(self) -> None:
+        if self._keepalive_task is not None:
+            self._keepalive_task.cancel()
+            self._keepalive_task = None
+
+    async def _keepalive_loop(self) -> None:
+        try:
+            while True:
+                await asyncio.sleep(KEEPALIVE_INTERVAL_S)
+                if self._ws is not None and not self._ws.closed:
+                    await self._ws.send_str(json.dumps({"type": "ping"}))
+        except asyncio.CancelledError:
+            pass
+        except (aiohttp.ClientError, ConnectionError):
+            logger.debug("Sarvam TTS keepalive send failed")
+
+    async def _reset_connection(self) -> None:
+        self._stop_keepalive()
+
+        if self._ws is not None and not self._ws.closed:
+            try:
+                await self._ws.close()
+            except (aiohttp.ClientError, ConnectionError):
+                logger.debug("Error closing Sarvam TTS websocket")
+        self._ws = None
+
+        if self._session is not None and not self._session.closed:
+            await self._session.close()
+        self._session = None
+
+    async def _receive_audio(
+        self, ws: aiohttp.ClientWebSocketResponse
+    ) -> AsyncIterator[PcmData]:
+        """Yield PcmData chunks until completion event, cancel, idle, or disconnect."""
+        while True:
+            if self._stop_event.is_set():
+                break
+            try:
+                msg = await asyncio.wait_for(ws.receive(), timeout=self._idle_timeout)
+            except asyncio.TimeoutError:
+                break
+
+            if msg.type in (
+                aiohttp.WSMsgType.CLOSED,
+                aiohttp.WSMsgType.CLOSING,
+                aiohttp.WSMsgType.ERROR,
+            ):
+                break
+            if msg.type != aiohttp.WSMsgType.TEXT:
+                continue
+
+            try:
+                data = json.loads(msg.data)
+            except json.JSONDecodeError:
+                logger.warning("Sarvam TTS sent non-JSON text: %s", msg.data)
+                continue
+
+            msg_type = data.get("type", "")
+            if msg_type in ("audio", "audio_chunk"):
+                payload = data.get("data") or {}
+                b64_audio = payload.get("audio") or data.get("audio")
+                if not b64_audio:
+                    continue
+                audio_bytes = base64.b64decode(b64_audio)
+                yield PcmData.from_bytes(
+                    audio_bytes,
+                    sample_rate=self.sample_rate,
+                    channels=1,
+                    format=AudioFormat.S16,
+                )
+            elif msg_type == "event":
+                event_data = data.get("data") or {}
+                if event_data.get("event_type") == "final":
+                    break
+            elif msg_type in ("flushed", "complete", "done"):
+                break
+            elif msg_type == "error":
+                error_data = data.get("data") or {}
+                error_msg = (
+                    error_data.get("message") or data.get("error") or "Sarvam TTS error"
+                )
+                raise SarvamTTSError(str(error_msg))