livekit-plugins-clevrlabs 0.1.0__tar.gz

livekit_plugins_clevrlabs-0.1.0/.gitignore

@@ -0,0 +1,21 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Virtual environments
+.venv/
+venv/
+
+# Tooling caches
+.ruff_cache/
+.mypy_cache/
+.pytest_cache/
+
+# OS / editor
+.DS_Store
+.idea/
+.vscode/

livekit_plugins_clevrlabs-0.1.0/LICENSE

@@ -0,0 +1,11 @@
+Copyright (c) 2026 Clevr Labs. All rights reserved.
+
+This software and associated documentation files (the "Software") are the
+proprietary and confidential property of Clevr Labs. Unauthorized copying,
+distribution, modification, or use of the Software, in whole or in part,
+is strictly prohibited without the express prior written permission of
+Clevr Labs.
+
+The Software is provided to authorized users solely for use with the
+Clevr Labs API service. No license, express or implied, is granted to any
+intellectual property rights in the Software.

livekit_plugins_clevrlabs-0.1.0/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: livekit-plugins-clevrlabs
+Version: 0.1.0
+Summary: LiveKit Agents TTS plugin for the Clevr Labs conversational speech model
+Project-URL: Homepage, https://theclevr.com
+License: Copyright (c) 2026 Clevr Labs. All rights reserved.
+        
+        This software and associated documentation files (the "Software") are the
+        proprietary and confidential property of Clevr Labs. Unauthorized copying,
+        distribution, modification, or use of the Software, in whole or in part,
+        is strictly prohibited without the express prior written permission of
+        Clevr Labs.
+        
+        The Software is provided to authorized users solely for use with the
+        Clevr Labs API service. No license, express or implied, is granted to any
+        intellectual property rights in the Software.
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.28
+Requires-Dist: livekit-agents>=1.4
+Requires-Dist: num2words>=0.5
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10

livekit_plugins_clevrlabs-0.1.0/livekit/plugins/clevrlabs/__init__.py

@@ -0,0 +1,36 @@
+"""Clevr Labs TTS plugin for LiveKit Agents.
+
+Provides a streaming ``tts.TTS`` implementation backed by the Clevr Labs
+conversational speech model. Import and use it like any other LiveKit plugin::
+
+    from livekit.plugins import clevrlabs
+
+    session = AgentSession(tts=clevrlabs.TTS(api_key="clevr_..."), ...)
+"""
+
+from .stt import is_whisper_hallucination
+from .tts import TTS
+from .version import __version__
+
+__all__ = ["TTS", "is_whisper_hallucination", "__version__"]
+
+from livekit.agents import Plugin
+
+from .log import logger
+
+
+class ClevrLabsPlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(ClevrLabsPlugin())
+
+# Cleanup docs of unexported modules
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit_plugins_clevrlabs-0.1.0/livekit/plugins/clevrlabs/_text.py

@@ -0,0 +1,75 @@
+"""Text normalisation before TTS synthesis.
+
+Translation table is built once at import time — import this module at
+process startup so the C table is never rebuilt per-request.
+"""
+
+import re
+import unicodedata
+from decimal import ROUND_HALF_UP, Decimal
+
+from num2words import num2words
+
+_CURRENCY_MAP = {
+    "$": ("dollar", "cent"),
+    "£": ("pound", "penny"),
+    "€": ("euro", "cent"),
+}
+
+
+def _expand_currency(m: re.Match) -> str:
+    symbol = m.group(1)
+    number = m.group(2).replace(",", "")
+    name, frac_name = _CURRENCY_MAP.get(symbol, ("dollar", "cent"))
+    amount = Decimal(number)
+    whole = int(amount)
+    cents = int((amount - whole).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) * 100)
+    parts = [f"{num2words(whole)} {name}{'s' if whole != 1 else ''}"]
+    if cents:
+        parts.append(f"{num2words(cents)} {frac_name}{'s' if cents != 1 else ''}")
+    return " and ".join(parts)
+
+
+_CURRENCY_RE = re.compile(r"([$£€])([\d,]+(?:\.\d{1,2})?)")
+
+_KEEP = set(",'?.-%$!:/£⁇¿")
+
+_REPLACE = {
+    "—": ",",
+    "–": "-",
+    "’": "'",
+    "@": " at ",
+    "&": "and",
+    "×": " times ",
+    "÷": " divided by ",
+    "±": " plus or minus ",
+    "°": " degrees ",
+}
+
+_table = str.maketrans(
+    {
+        **{
+            chr(cp): None
+            for cp in range(0x110000)
+            if unicodedata.category(chr(cp)).startswith(("P", "S"))
+            and chr(cp) not in _KEEP
+            and chr(cp) not in _REPLACE
+        },
+        **_REPLACE,
+    }
+)
+
+_EMAIL = re.compile(r"\b[\w.+-]+@[\w.-]+\.[a-zA-Z]{2,}\b")
+_SPACE_BEFORE_COMMA = re.compile(r" ,")
+_MULTI_PUNCT = re.compile(r"[,'?.%$!:/£⁇¿-]{2,}")
+
+
+def clean_text(text: str) -> str:
+    text = _CURRENCY_RE.sub(_expand_currency, text)
+    text = re.sub(r"\s+", " ", text).strip()
+    text = _EMAIL.sub(lambda m: m.group().replace(".", " dot ").replace("@", " at "), text)
+    text = text.translate(_table)
+    text = re.sub(r" {2,}", " ", text)
+    text = _SPACE_BEFORE_COMMA.sub(",", text)
+    text = _MULTI_PUNCT.sub(lambda m: m.group()[-1], text)
+    return text

livekit_plugins_clevrlabs-0.1.0/livekit/plugins/clevrlabs/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.clevrlabs")

livekit_plugins_clevrlabs-0.1.0/livekit/plugins/clevrlabs/stt.py

@@ -0,0 +1,39 @@
+"""STT-side helpers for use with the Clevr Labs voice model.
+
+These utilities are not required to use ``clevrlabs.TTS`` — they exist to help you
+avoid feeding bad STT output into the conversational context, which can
+corrupt voice consistency.
+"""
+
+from __future__ import annotations
+
+_WHISPER_HALLUCINATION_PHRASES = (
+    "thanks for watching",
+    "thank you for watching",
+    "like and subscribe",
+    "subscribe to",
+    "don't forget to subscribe",
+    "see you in the next video",
+    "subtitles by",
+    "subtitle by",
+    "transcribed by",
+    "amara.org",
+)
+
+
+def is_whisper_hallucination(text: str) -> bool:
+    """True if ``text`` matches a known Whisper hallucination pattern.
+
+    Whisper (and Whisper-based providers like Groq) is partly trained on
+    YouTube subtitle data, so it can produce phantom transcripts such as
+    "Thanks for watching!" on silent or noisy audio. Feeding these into
+    ``clevrlabs.TTS.add_user_turn`` pairs fake text with real audio in the model's
+    context and can cause the voice to drift.
+
+    Use this to filter STT output before it reaches ``add_user_turn`` (and
+    before it reaches your LLM). Only applies to Whisper-family STT — other
+    providers (Deepgram, AssemblyAI, Google, Azure) have different failure
+    modes and typically don't need this filter.
+    """
+    lower = text.lower()
+    return any(phrase in lower for phrase in _WHISPER_HALLUCINATION_PHRASES)

livekit_plugins_clevrlabs-0.1.0/livekit/plugins/clevrlabs/tts.py

@@ -0,0 +1,313 @@
+"""LiveKit TTS plugin for the Clevr Labs conversational speech model."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import re
+import time
+from math import gcd
+
+import httpx
+import numpy as np
+from scipy.signal import resample_poly
+
+from livekit.agents import APIConnectOptions, tts, utils
+from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS
+
+from ._text import clean_text
+from .log import logger
+
+_DEFAULT_SERVER_URL = "https://api.theclevr.com"
+
+
+class TTS(tts.TTS):
+    """LiveKit TTS plugin for the Clevr Labs conversational speech model.
+
+    Usage::
+
+        from livekit.plugins import clevrlabs
+
+        tts = clevrlabs.TTS(api_key="clevr_...")
+
+        # Wire into a LiveKit AgentSession:
+        session = AgentSession(tts=tts, ...)
+
+        # After each user turn, feed context so the server keeps voice consistency:
+        tts.add_user_turn(text=transcript, audio=audio_np, sample_rate=48000)
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        server_url: str = _DEFAULT_SERVER_URL,
+        sample_rate: int = 24000,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ):
+        super().__init__(
+            capabilities=tts.TTSCapabilities(streaming=True),
+            sample_rate=sample_rate,
+            num_channels=1,
+        )
+        self._server_url = server_url.rstrip("/")
+        self._conn_options = conn_options
+
+        self._session_id: str | None = None
+        self._session_started: bool = False
+        self._session_ready = asyncio.Event()
+        self._session_error: Exception | None = None
+        self._pending_user_turn: dict | None = None
+
+        self._http_client = httpx.AsyncClient(
+            base_url=self._server_url,
+            headers={"Authorization": f"Bearer {api_key}"},
+            timeout=httpx.Timeout(90.0, connect=10.0),
+        )
+
+    def add_user_turn(self, *, text: str, audio: np.ndarray, sample_rate: int) -> None:
+        """Buffer a user turn to send with the next synthesis request.
+
+        The server owns all conversation context in its KV cache. The client
+        only needs to forward new user audio so the server can encode it.
+
+        Args:
+            text:        Transcript of what the user said.
+            audio:       Float32 numpy array of the user's speech. Int16 arrays
+                         are accepted and converted automatically.
+            sample_rate: Sample rate of the audio array (e.g. 48000, 16000).
+        """
+        if not text.strip() or audio.size == 0:
+            logger.debug(
+                "add_user_turn: dropping empty turn (text=%r, audio_size=%d)",
+                text,
+                audio.size,
+            )
+            return
+
+        if audio.dtype != np.float32:
+            audio = audio.astype(np.float32)
+            if np.abs(audio).max() > 1.0:
+                audio /= 32768.0
+
+        if sample_rate != 24000:
+            g = gcd(sample_rate, 24000)
+            audio = resample_poly(audio, 24000 // g, sample_rate // g).astype(np.float32)
+
+        audio_b64 = base64.b64encode(audio.tobytes()).decode("ascii")
+        self._pending_user_turn = {"speaker": 1, "text": text, "audio_b64": audio_b64}
+
+    def start_session(self) -> None:
+        """Eagerly start a session in the background.
+
+        Call this right after construction (from an async context) so the
+        session is ready before the first synthesis request. If not called,
+        the session is started lazily on first use.
+        """
+        asyncio.ensure_future(self._start_session_bg())
+
+    async def _start_session_bg(self) -> None:
+        try:
+            resp = await self._http_client.post("/tts/session/start")
+            resp.raise_for_status()
+            data = resp.json()
+            self._session_id = data["session_id"]
+            self._session_started = True
+            logger.info("Clevr session started: %s", self._session_id)
+        except Exception as e:
+            self._session_error = e
+            logger.error("Failed to start Clevr session: %s", e)
+        finally:
+            self._session_ready.set()
+
+    async def _ensure_session(self) -> None:
+        if self._session_started:
+            return
+        if not self._session_ready.is_set():
+            await self._session_ready.wait()
+        if self._session_error is not None:
+            raise self._session_error
+        if not self._session_started:
+            resp = await self._http_client.post("/tts/session/start")
+            resp.raise_for_status()
+            data = resp.json()
+            self._session_id = data["session_id"]
+            self._session_started = True
+            logger.info("Clevr session started: %s", self._session_id)
+
+    async def _end_session(self) -> None:
+        if not self._session_started or not self._session_id:
+            return
+        try:
+            resp = await self._http_client.post(
+                "/tts/session/end", params={"session_id": self._session_id}
+            )
+            resp.raise_for_status()
+            logger.info("Clevr session ended: %s", self._session_id)
+        except Exception:
+            logger.warning("Failed to end Clevr session %s", self._session_id, exc_info=True)
+        finally:
+            self._session_started = False
+            self._session_id = None
+
+    async def aclose(self) -> None:
+        await self._end_session()
+        await self._http_client.aclose()
+
+    @property
+    def model(self) -> str:
+        return "csm-1"
+
+    @property
+    def provider(self) -> str:
+        return "clevr"
+
+    def synthesize(
+        self, text: str, *, conn_options: APIConnectOptions | None = None
+    ) -> tts.ChunkedStream:
+        return self._synthesize_with_stream(
+            text=text, conn_options=conn_options or self._conn_options
+        )
+
+    def stream(self, *, conn_options: APIConnectOptions | None = None) -> ClevrLabsSynthesizeStream:
+        return ClevrLabsSynthesizeStream(tts=self, conn_options=conn_options or self._conn_options)
+
+
+class _SentenceBuffer:
+    """Buffers streamed LLM tokens and emits chunks of N complete sentences."""
+
+    def __init__(self, n: int = 3) -> None:
+        self._n = n
+        self._buf = ""
+        self._pattern = re.compile(r"([.!?]+(?:\s+|\n+|\Z))")
+
+    def push(self, text: str) -> list:
+        self._buf += text
+        chunks = []
+        while True:
+            matches = list(self._pattern.finditer(self._buf))
+            if len(matches) < self._n:
+                break
+            split_idx = matches[self._n - 1].end()
+            chunk = self._buf[:split_idx].strip()
+            self._buf = self._buf[split_idx:]
+            if chunk:
+                chunks.append(chunk)
+        return chunks
+
+    def flush(self) -> str:
+        text = self._buf.strip()
+        self._buf = ""
+        return text
+
+
+class ClevrLabsSynthesizeStream(tts.SynthesizeStream):
+    def __init__(self, *, tts: TTS, conn_options: APIConnectOptions) -> None:
+        super().__init__(tts=tts, conn_options=conn_options)
+        self._tts: TTS = tts
+
+    async def _run(self, output_emitter: tts.AudioEmitter) -> None:
+        request_id = utils.shortuuid()
+        output_emitter.initialize(
+            request_id=request_id,
+            sample_rate=self._tts.sample_rate,
+            num_channels=self._tts.num_channels,
+            mime_type="audio/pcm",
+            stream=True,
+        )
+        audio_bstream = utils.audio.AudioByteStream(
+            sample_rate=self._tts.sample_rate,
+            num_channels=1,
+        )
+        segment_id = utils.shortuuid()
+        output_emitter.start_segment(segment_id=segment_id)
+
+        buf = _SentenceBuffer(n=3)
+        async for data in self._input_ch:
+            if isinstance(data, tts.SynthesizeStream._FlushSentinel):
+                text = buf.flush()
+                if text:
+                    await self._synthesize_segment(text, output_emitter, audio_bstream)
+            else:
+                for chunk in buf.push(data):
+                    await self._synthesize_segment(chunk, output_emitter, audio_bstream)
+
+        text = buf.flush()
+        if text:
+            await self._synthesize_segment(text, output_emitter, audio_bstream)
+
+        output_emitter.flush()
+        output_emitter.end_segment()
+
+    async def _synthesize_segment(
+        self,
+        text: str,
+        output_emitter: tts.AudioEmitter,
+        audio_bstream: utils.audio.AudioByteStream,
+    ) -> None:
+        text = clean_text(text.strip())
+        if not text:
+            return
+
+        await self._tts._ensure_session()
+
+        t_start = time.perf_counter()
+
+        payload: dict = {
+            "text": text,
+            "speaker": 0,
+            "session_id": self._tts._session_id,
+        }
+        if self._tts._pending_user_turn is not None:
+            payload["context"] = [self._tts._pending_user_turn]
+            self._tts._pending_user_turn = None
+
+        audio_chunks: list = []
+        byte_buffer = b""
+
+        try:
+            async with self._tts._http_client.stream(
+                "POST", "/tts/synthesize/stream", json=payload
+            ) as resp:
+                if resp.status_code >= 400:
+                    await resp.aread()
+                    raise httpx.HTTPStatusError(
+                        f"HTTP {resp.status_code}",
+                        request=resp.request,
+                        response=resp,
+                    )
+                async for raw in resp.aiter_bytes():
+                    byte_buffer += raw
+                    complete = (len(byte_buffer) // 4) * 4
+                    if complete == 0:
+                        continue
+                    chunk_np = np.frombuffer(byte_buffer[:complete], dtype=np.float32).copy()
+                    byte_buffer = byte_buffer[complete:]
+
+                    audio_chunks.append(chunk_np)
+                    chunk_np = np.clip(chunk_np, -1.0, 1.0)
+                    chunk_int16 = (chunk_np * 32767).astype(np.int16)
+                    for frame in audio_bstream.write(chunk_int16.tobytes()):
+                        output_emitter.push(frame.data.tobytes())
+
+            for frame in audio_bstream.flush():
+                output_emitter.push(frame.data.tobytes())
+
+        except asyncio.CancelledError:
+            raise
+        except httpx.ConnectError as e:
+            raise RuntimeError(
+                f"Cannot connect to Clevr TTS server at {self._tts._server_url}. "
+                "Check your server_url or visit https://theclevr.com for status."
+            ) from e
+        except httpx.HTTPStatusError as e:
+            raise RuntimeError(
+                f"Clevr TTS server returned {e.response.status_code}: {e.response.text}"
+            ) from e
+        finally:
+            total_ms = (time.perf_counter() - t_start) * 1000
+            audio_np = (
+                np.concatenate(audio_chunks) if audio_chunks else np.zeros(0, dtype=np.float32)
+            )
+            duration_s = len(audio_np) / self._tts.sample_rate
+            logger.info("[clevr] %.1fs audio in %.0fms", duration_s, total_ms)

livekit_plugins_clevrlabs-0.1.0/livekit/plugins/clevrlabs/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

livekit_plugins_clevrlabs-0.1.0/pyproject.toml

@@ -0,0 +1,56 @@
+[project]
+name = "livekit-plugins-clevrlabs"
+dynamic = ["version"]
+description = "LiveKit Agents TTS plugin for the Clevr Labs conversational speech model"
+license = { file = "LICENSE" }
+requires-python = ">=3.11"
+dependencies = [
+    "httpx>=0.28",
+    "numpy>=1.24",
+    "livekit-agents>=1.4",
+    "scipy>=1.10",
+    "num2words>=0.5",
+]
+
+[project.urls]
+Homepage = "https://theclevr.com"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "livekit/plugins/clevrlabs/version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["livekit"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/livekit"]
+
+# Mirrors the livekit/agents repo ruff config so this plugin stays clean against
+# their CI (`make check`) and is ready to upstream without restyling.
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "W", "F", "I", "B", "C4", "UP"]
+ignore = ["E501"]
+
+[tool.ruff.lint.isort]
+combine-as-imports = true
+known-first-party = ["livekit"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+# Mirrors the livekit/agents strict mypy gate. num2words and scipy ship no type
+# stubs, so they're ignored the same way upstream ignores stub-less deps.
+[tool.mypy]
+strict = true
+disallow_any_generics = false
+
+[[tool.mypy.overrides]]
+module = ["num2words", "scipy.*"]
+ignore_missing_imports = true