livekit-plugins-lokutor 1.8.12__tar.gz

livekit_plugins_lokutor-1.8.12/.gitignore

@@ -0,0 +1,180 @@
+**/.vscode
+**/.DS_Store
+.env
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# trunk
+.trunk/
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+node_modules
+
+credentials.json
+pyrightconfig.json
+docs/
+
+# Database files
+*.db
+
+
+# Examples for development
+examples/dev/*

livekit_plugins_lokutor-1.8.12/PKG-INFO

@@ -0,0 +1,95 @@
+Metadata-Version: 2.5
+Name: livekit-plugins-lokutor
+Version: 1.8.12
+Summary: Lokutor speech-to-text and text-to-speech plugin for LiveKit agents
+Project-URL: Documentation, https://docs.livekit.io
+Project-URL: Website, https://livekit.io/
+Project-URL: Source, https://github.com/livekit/agents
+Author-email: LiveKit <hello@livekit.io>
+License-Expression: Apache-2.0
+Keywords: STT,TTS,audio,livekit,lokutor,realtime,voice,webrtc
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10.0
+Requires-Dist: livekit-agents>=1.8.12
+Description-Content-Type: text/markdown
+
+# LiveKit Plugins Lokutor
+
+**Lokutor** speech-to-text and text-to-speech for [LiveKit Agents](https://docs.livekit.io/agents/).
+
+Lokutor builds its own small speech models: Conv 2.0 for recognition and Versa 2.0 for synthesis, each one model for nine languages (English, Spanish, Catalan, Galician, Basque, Portuguese, French, Italian and German). See [lokutor.com](https://lokutor.com) and the [API docs](https://docs.lokutor.com).
+
+LiveKit docs: [speech-to-text](https://docs.livekit.io/agents/models/stt/lokutor/) and [text-to-speech](https://docs.livekit.io/agents/models/tts/lokutor/) (live with the next release).
+
+## Installation
+
+```bash
+pip install livekit-plugins-lokutor
+```
+
+Set `LOKUTOR_API_KEY`, or pass `api_key=` to `STT` and `TTS`.
+
+## Usage
+
+### In an agent
+
+```python
+from livekit.agents import AgentSession
+from livekit.plugins import lokutor, openai, silero
+
+session = AgentSession(
+    stt=lokutor.STT(language="es"),
+    llm=openai.LLM(),
+    tts=lokutor.TTS(voice="F1", language="es"),
+    vad=silero.VAD.load(),
+)
+```
+
+### Speech-to-text
+
+`STT` streams over `WS /ws/stt`: interim transcripts while the user speaks, then a final transcript when the server sees the utterance end (a pause of about 0.7 s). `recognize()` transcribes one clip with `POST /stt/transcribe`. The text comes back written out, with punctuation, casing, digits and e-mail addresses.
+
+```python
+stt = lokutor.STT(language="en")
+event = await stt.recognize(frames)  # a list of rtc.AudioFrame, any sample rate
+print(event.alternatives[0].text)
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `api_key` | `str` | env `LOKUTOR_API_KEY` | Lokutor API key |
+| `language` | `str` | detected | One of the nine languages; regional codes such as `es-ES` are accepted. Set it whenever you know it: it keeps one-word replies reliable and sets how the text is written. Any other value is left to detection |
+| `base_url` | `str` | `"wss://api.lokutor.com"` | API base URL |
+
+### Text-to-speech
+
+```python
+tts = lokutor.TTS(voice="F1", language="en")
+async with tts:
+    stream = tts.stream()
+    stream.push_text("Hello, world!")
+    stream.end_input()
+    async for audio in stream:
+        print(f"Got audio: {audio.frame.duration:.2f}s")
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `api_key` | `str` | env `LOKUTOR_API_KEY` | Lokutor API key |
+| `voice` | `str` | `"F1"` | Voice slot, `F1`-`F5` (female) or `M1`-`M5` (male). Each language has its own native speaker in each slot |
+| `language` | `str \| None` | `"en"` | One of the nine languages; any other value uses the English speakers |
+| `tokenizer` | `tokenize.SentenceTokenizer` | `blingfire` | Splits the streamed input into sentences; each sentence is one request, so the voice has per-sentence prosody |
+| `base_url` | `str` | `"wss://api.lokutor.com"` | API base URL |
+
+Audio comes out at 44.1 kHz mono; the pipeline resamples if it needs another rate.
+
+## License
+
+Apache 2.0

livekit_plugins_lokutor-1.8.12/README.md

@@ -0,0 +1,73 @@
+# LiveKit Plugins Lokutor
+
+**Lokutor** speech-to-text and text-to-speech for [LiveKit Agents](https://docs.livekit.io/agents/).
+
+Lokutor builds its own small speech models: Conv 2.0 for recognition and Versa 2.0 for synthesis, each one model for nine languages (English, Spanish, Catalan, Galician, Basque, Portuguese, French, Italian and German). See [lokutor.com](https://lokutor.com) and the [API docs](https://docs.lokutor.com).
+
+LiveKit docs: [speech-to-text](https://docs.livekit.io/agents/models/stt/lokutor/) and [text-to-speech](https://docs.livekit.io/agents/models/tts/lokutor/) (live with the next release).
+
+## Installation
+
+```bash
+pip install livekit-plugins-lokutor
+```
+
+Set `LOKUTOR_API_KEY`, or pass `api_key=` to `STT` and `TTS`.
+
+## Usage
+
+### In an agent
+
+```python
+from livekit.agents import AgentSession
+from livekit.plugins import lokutor, openai, silero
+
+session = AgentSession(
+    stt=lokutor.STT(language="es"),
+    llm=openai.LLM(),
+    tts=lokutor.TTS(voice="F1", language="es"),
+    vad=silero.VAD.load(),
+)
+```
+
+### Speech-to-text
+
+`STT` streams over `WS /ws/stt`: interim transcripts while the user speaks, then a final transcript when the server sees the utterance end (a pause of about 0.7 s). `recognize()` transcribes one clip with `POST /stt/transcribe`. The text comes back written out, with punctuation, casing, digits and e-mail addresses.
+
+```python
+stt = lokutor.STT(language="en")
+event = await stt.recognize(frames)  # a list of rtc.AudioFrame, any sample rate
+print(event.alternatives[0].text)
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `api_key` | `str` | env `LOKUTOR_API_KEY` | Lokutor API key |
+| `language` | `str` | detected | One of the nine languages; regional codes such as `es-ES` are accepted. Set it whenever you know it: it keeps one-word replies reliable and sets how the text is written. Any other value is left to detection |
+| `base_url` | `str` | `"wss://api.lokutor.com"` | API base URL |
+
+### Text-to-speech
+
+```python
+tts = lokutor.TTS(voice="F1", language="en")
+async with tts:
+    stream = tts.stream()
+    stream.push_text("Hello, world!")
+    stream.end_input()
+    async for audio in stream:
+        print(f"Got audio: {audio.frame.duration:.2f}s")
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `api_key` | `str` | env `LOKUTOR_API_KEY` | Lokutor API key |
+| `voice` | `str` | `"F1"` | Voice slot, `F1`-`F5` (female) or `M1`-`M5` (male). Each language has its own native speaker in each slot |
+| `language` | `str \| None` | `"en"` | One of the nine languages; any other value uses the English speakers |
+| `tokenizer` | `tokenize.SentenceTokenizer` | `blingfire` | Splits the streamed input into sentences; each sentence is one request, so the voice has per-sentence prosody |
+| `base_url` | `str` | `"wss://api.lokutor.com"` | API base URL |
+
+Audio comes out at 44.1 kHz mono; the pipeline resamples if it needs another rate.
+
+## License
+
+Apache 2.0

livekit_plugins_lokutor-1.8.12/livekit/plugins/lokutor/__init__.py

@@ -0,0 +1,30 @@
+"""Lokutor plugin for LiveKit Agents: speech-to-text and text-to-speech.
+
+See https://docs.lokutor.com for the API behind it.
+"""
+
+from .stt import STT, SpeechStream
+from .tts import TTS, ChunkedStream, SynthesizeStream
+from .version import __version__
+
+__all__ = ["STT", "SpeechStream", "TTS", "ChunkedStream", "SynthesizeStream", "__version__"]
+
+from livekit.agents import Plugin
+
+from .log import logger
+
+
+class LokutorPlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(LokutorPlugin())
+
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit_plugins_lokutor-1.8.12/livekit/plugins/lokutor/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.lokutor")

livekit_plugins_lokutor-1.8.12/livekit/plugins/lokutor/models.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Literal
+
+# Voice slots: five female, five male. Each language has its own native speaker in
+# each slot, so the speaker heard depends on the language too. See
+# https://docs.lokutor.com/voices-languages-models#voices
+VoiceID = Literal[
+    "F1",
+    "F2",
+    "F3",
+    "F4",
+    "F5",
+    "M1",
+    "M2",
+    "M3",
+    "M4",
+    "M5",
+]
+
+# The nine languages Lokutor speaks and recognises. A language outside this list
+# uses the English speakers for synthesis and is detected for recognition. See
+# https://docs.lokutor.com/voices-languages-models#languages
+TTSLanguage = Literal["en", "es", "ca", "gl", "eu", "pt", "fr", "it", "de"]
+STTLanguage = TTSLanguage
+STT_LANGUAGES: frozenset[str] = frozenset(("en", "es", "ca", "gl", "eu", "pt", "fr", "it", "de"))
+
+TTSModels = Literal["versa-2.0"]
+TTS_MODEL: TTSModels = "versa-2.0"
+STT_MODEL = "conv-2.0"
+
+DEFAULT_VOICE_ID: VoiceID = "F1"
+DEFAULT_LANGUAGE: TTSLanguage = "en"
+DEFAULT_SAMPLE_RATE: int = 44100
+DEFAULT_BASE_URL = "wss://api.lokutor.com"

livekit_plugins_lokutor-1.8.12/livekit/plugins/lokutor/stt.py

@@ -0,0 +1,449 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Lokutor speech-to-text.
+
+Streaming uses ``WS /ws/stt``. Wire contract:
+
+    client -> server   {"language": "en"}   ("" lets the model detect it)
+    server -> client   {"type": "status", "data": "ready"}
+    client -> server   binary frames: 16-bit PCM, 16 kHz, mono
+    server -> client   {"type": "transcript", "data": ..., "isFinal": bool}
+    client -> server   {"type": "end"}
+    server -> client   the last final, then {"type": "status", "data": "done"}
+
+The server decides where an utterance ends (a pause of about 0.7 s). ``data`` is the
+whole utterance so far on every partial, and the final carries the utterance once,
+transcribed in one pass. Single-shot recognition uses ``POST /stt/transcribe``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import os
+from dataclasses import dataclass
+
+import aiohttp
+
+from livekit import rtc
+from livekit.agents import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    APIConnectionError,
+    APIConnectOptions,
+    APIError,
+    APIStatusError,
+    APITimeoutError,
+    stt,
+    utils,
+)
+from livekit.agents.language import LanguageCode
+from livekit.agents.types import NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import AudioBuffer, is_given
+
+from .log import logger
+from .models import DEFAULT_BASE_URL, STT_LANGUAGES, STT_MODEL, STTLanguage
+
+_SAMPLE_RATE = 16000
+
+
+@dataclass
+class _STTOptions:
+    api_key: str
+    language: str | None
+    base_url: str
+
+
+def _api_language(language: str | None) -> str:
+    """The language as the API takes it ("en-US" -> "en"), or "" to have it detected.
+
+    The recogniser writes the text by the rules of the language it is given, so a
+    language it does not serve is left to detection rather than forced.
+    """
+    if not language:
+        return ""
+    code = LanguageCode(language).language
+    return code if code in STT_LANGUAGES else ""
+
+
+def _http_base(base_url: str) -> str:
+    if base_url.startswith("wss://"):
+        return "https://" + base_url[len("wss://") :]
+    if base_url.startswith("ws://"):
+        return "http://" + base_url[len("ws://") :]
+    return base_url
+
+
+class STT(stt.STT):
+    """Lokutor Speech-to-Text integration.
+
+    One model for nine languages: English, Spanish, Catalan, Galician, Basque,
+    Portuguese, French, Italian and German. The text comes back written out, with
+    punctuation, casing, digits and e-mail addresses.
+
+    Args:
+        api_key: Lokutor API key. If not provided, reads from ``LOKUTOR_API_KEY``
+            environment variable.
+        language: Language code (e.g. ``"en"``, ``"es-ES"``). Send it whenever it is
+            known: it keeps one-word replies reliable and sets how the text is
+            written. Left unset, or a language the model does not serve, the
+            language is detected.
+        base_url: Base URL for the Lokutor API. Defaults to ``"wss://api.lokutor.com"``.
+        http_session: Optional shared ``aiohttp.ClientSession``.
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        language: NotGivenOr[STTLanguage | str] = NOT_GIVEN,
+        base_url: str = DEFAULT_BASE_URL,
+        http_session: aiohttp.ClientSession | None = None,
+    ) -> None:
+        super().__init__(
+            capabilities=stt.STTCapabilities(
+                streaming=True,
+                interim_results=True,
+                offline_recognize=True,
+            ),
+        )
+        lokutor_api_key = api_key or os.environ.get("LOKUTOR_API_KEY")
+        if not lokutor_api_key:
+            raise ValueError(
+                "Lokutor API key is required, either as argument or set"
+                " LOKUTOR_API_KEY environment variable"
+            )
+        self._opts = _STTOptions(
+            api_key=lokutor_api_key,
+            language=language if is_given(language) else None,
+            base_url=base_url,
+        )
+        self._session = http_session
+
+    @property
+    def model(self) -> str:
+        """Returns the model name ``"conv-2.0"``."""
+        return STT_MODEL
+
+    @property
+    def provider(self) -> str:
+        """Returns the provider name ``"Lokutor"``."""
+        return "Lokutor"
+
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if not self._session:
+            self._session = utils.http_context.http_session()
+        return self._session
+
+    async def _recognize_impl(
+        self,
+        buffer: AudioBuffer,
+        *,
+        language: NotGivenOr[str] = NOT_GIVEN,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> stt.SpeechEvent:
+        lang = _api_language(language if is_given(language) else self._opts.language)
+        frame = rtc.combine_audio_frames(buffer)
+        body: dict[str, str] = {
+            "audio": base64.b64encode(frame.to_wav_bytes()).decode(),
+            "format": "wav",
+        }
+        if lang:
+            body["language"] = lang
+
+        try:
+            async with self._ensure_session().post(
+                f"{_http_base(self._opts.base_url)}/stt/transcribe",
+                json=body,
+                headers={"X-API-Key": self._opts.api_key},
+                timeout=aiohttp.ClientTimeout(total=conn_options.timeout),
+            ) as resp:
+                if resp.status != 200:
+                    raise APIStatusError(
+                        "Lokutor transcription failed",
+                        status_code=resp.status,
+                        request_id=None,
+                        body=None,
+                    )
+                data = await resp.json()
+        except asyncio.TimeoutError:
+            raise APITimeoutError() from None
+        except APIError:
+            raise
+        except Exception as e:
+            # Keep only the exception type: a transport error can carry the request
+            # URL, headers or a response body.
+            raise APIConnectionError(type(e).__name__) from None
+
+        return stt.SpeechEvent(
+            type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+            request_id=utils.shortuuid("lokutor-"),
+            alternatives=[
+                stt.SpeechData(
+                    # Without a requested language, the one the API reports (detected), if it serves it.
+                    language=LanguageCode(
+                        lang or _api_language(str(data.get("language") or "")) or "multi"
+                    ),
+                    text=str(data.get("text") or "").strip(),
+                    start_time=0.0,
+                    end_time=frame.duration,
+                )
+            ],
+        )
+
+    def stream(
+        self,
+        *,
+        language: NotGivenOr[str] = NOT_GIVEN,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> SpeechStream:
+        """Create a streaming recognition session over ``WS /ws/stt``."""
+        return SpeechStream(
+            stt=self,
+            conn_options=conn_options,
+            language=language if is_given(language) else self._opts.language,
+        )
+
+
+class SpeechStream(stt.SpeechStream):
+    """Streaming recognition: interim transcripts while the user speaks, a final one
+    when the server sees the utterance end."""
+
+    def __init__(
+        self,
+        *,
+        stt: STT,
+        conn_options: APIConnectOptions,
+        language: str | None,
+    ) -> None:
+        super().__init__(stt=stt, conn_options=conn_options, sample_rate=_SAMPLE_RATE)
+        self._stt: STT = stt
+        self._api_lang = _api_language(language)
+        self._language = LanguageCode(self._api_lang or "multi")
+        self._speaking = False
+        self._request_id = ""
+        self._input_done = False
+        self._done = False
+        self._audio_sent = 0.0
+        self._audio_reported = 0.0
+        # Audio position (seconds into this stream) where the last final transcript ended: the earliest
+        # the next utterance can start. The server sends no word timings.
+        self._utterance_floor = 0.0
+
+    async def _connect_ws(self) -> aiohttp.ClientWebSocketResponse:
+        try:
+            return await asyncio.wait_for(
+                self._stt._ensure_session().ws_connect(
+                    f"{self._stt._opts.base_url}/ws/stt",
+                    max_msg_size=0,
+                    # In a header, not the query string, so the key stays out of
+                    # exception messages, tracing and access logs.
+                    headers={"X-API-Key": self._stt._opts.api_key},
+                ),
+                self._conn_options.timeout,
+            )
+        except asyncio.TimeoutError:
+            raise APITimeoutError() from None
+        except aiohttp.WSServerHandshakeError as e:
+            # 401 bad key, 402 no balance: final. 429, the account's concurrent sessions in use, is
+            # retryable (APIStatusError's rule): a reconnect's old session is likely about to be released.
+            raise APIStatusError(
+                "Lokutor STT refused the connection",
+                status_code=e.status,
+                request_id=None,
+                body=None,
+            ) from None
+        except Exception as e:
+            raise APIConnectionError(type(e).__name__) from None
+
+    async def _run(self) -> None:
+        # A retry starts on a new connection, which knows nothing of an utterance the last one cut off: end
+        # it for the consumer, and let the new connection's first transcript open its own.
+        if self._speaking:
+            self._event_ch.send_nowait(
+                stt.SpeechEvent(type=stt.SpeechEventType.END_OF_SPEECH, request_id=self._request_id)
+            )
+        self._speaking = False
+        self._request_id = ""
+        self._input_done = False
+        self._done = False
+        # start_time_offset restarts at this connection's first frame (restart_provider_clock), so
+        # transcript times count from here too. The last connection's usage was reported when it ended.
+        self._audio_sent = 0.0
+        self._audio_reported = 0.0
+        self._utterance_floor = 0.0
+        ws = await self._connect_ws()
+        try:
+            await ws.send_str(json.dumps({"language": self._api_lang}))
+            await asyncio.wait_for(self._await_ready(ws), self._conn_options.timeout)
+
+            send_task = asyncio.create_task(self._send_task(ws))
+            recv_task = asyncio.create_task(self._recv_task(ws))
+            try:
+                await asyncio.wait({send_task, recv_task}, return_when=asyncio.FIRST_COMPLETED)
+                if send_task.done():
+                    send_task.result()
+                    # Input has ended: the last final transcript and "done" must follow within the
+                    # timeout. A live stream waits on the caller for as long as it lasts; this
+                    # bounds only the wait for a server that stops answering after "end".
+                    await asyncio.wait_for(recv_task, self._conn_options.timeout)
+                else:
+                    recv_task.result()
+            finally:
+                await utils.aio.gracefully_cancel(send_task, recv_task)
+        except asyncio.TimeoutError:
+            raise APITimeoutError() from None
+        except aiohttp.ClientError as e:
+            # A send on a socket that dropped mid-stream raises before the receiver sees the close;
+            # as a connection error it is retried like any other drop.
+            raise APIConnectionError(type(e).__name__) from None
+        finally:
+            self._report_usage()
+            await ws.close()
+
+    async def _await_ready(self, ws: aiohttp.ClientWebSocketResponse) -> None:
+        while True:
+            msg = await ws.receive()
+            if msg.type != aiohttp.WSMsgType.TEXT:
+                raise APIConnectionError("Lokutor STT closed before it was ready")
+            try:
+                data = json.loads(msg.data)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(data, dict):
+                continue
+            if data.get("type") == "error":
+                raise APIStatusError(
+                    "Lokutor STT rejected the session",
+                    status_code=-1,
+                    request_id=None,
+                    body=None,
+                    retryable=False,
+                )
+            if data.get("type") == "status" and data.get("data") == "ready":
+                self._note_language(data.get("lang"))
+                return
+
+    async def _send_task(self, ws: aiohttp.ClientWebSocketResponse) -> None:
+        async for data in self._input_ch:
+            # A flush needs nothing here: the server ends an utterance on its own pause.
+            if isinstance(data, rtc.AudioFrame):
+                audio = data.data.tobytes()
+                if audio:
+                    self._audio_sent += data.samples_per_channel / data.sample_rate
+                    await ws.send_bytes(audio)
+        self._input_done = True
+        await ws.send_str(json.dumps({"type": "end"}))
+
+    async def _recv_task(self, ws: aiohttp.ClientWebSocketResponse) -> None:
+        while True:
+            msg = await ws.receive()
+            if msg.type in (
+                aiohttp.WSMsgType.CLOSED,
+                aiohttp.WSMsgType.CLOSE,
+                aiohttp.WSMsgType.CLOSING,
+            ):
+                # Only "done" ends a stream: a close before it, even after end_input(), may have
+                # lost the last final transcript.
+                if self._done:
+                    return
+                raise APIConnectionError("Lokutor STT closed before it finished")
+            if msg.type == aiohttp.WSMsgType.ERROR:
+                raise APIConnectionError("Lokutor STT connection failed")
+            if msg.type != aiohttp.WSMsgType.TEXT:
+                continue
+            try:
+                data = json.loads(msg.data)
+            except json.JSONDecodeError:
+                logger.debug("Ignoring a non-JSON frame from Lokutor STT")
+                continue
+            if not isinstance(data, dict):
+                continue
+            kind = data.get("type")
+            if kind == "error":
+                # Not the provider's message: it may echo customer content.
+                raise APIStatusError(
+                    "Lokutor STT returned an error",
+                    status_code=-1,
+                    request_id=None,
+                    body=None,
+                    retryable=False,
+                )
+            if kind == "status" and data.get("data") == "done":
+                self._done = True
+                return
+            if kind == "transcript":
+                self._note_language(data.get("lang"))
+                self._on_transcript(
+                    str(data.get("data") or "").strip(), data.get("isFinal") is True
+                )
+
+    def _note_language(self, lang: object) -> None:
+        # With no language requested, label transcripts with the one the server reports, if it
+        # serves it; "multi" otherwise.
+        if not self._api_lang and isinstance(lang, str) and _api_language(lang):
+            self._language = LanguageCode(_api_language(lang))
+
+    def _on_transcript(self, text: str, final: bool) -> None:
+        if not text:
+            # An empty final after interims (a cough, noise) still closes the utterance they opened.
+            if final and self._speaking:
+                self._end_utterance()
+            return
+        if not self._speaking:
+            self._speaking = True
+            self._request_id = utils.shortuuid("lokutor-")
+            self._event_ch.send_nowait(stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH))
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(
+                type=(
+                    stt.SpeechEventType.FINAL_TRANSCRIPT
+                    if final
+                    else stt.SpeechEventType.INTERIM_TRANSCRIPT
+                ),
+                request_id=self._request_id,
+                alternatives=[
+                    stt.SpeechData(
+                        language=self._language,
+                        text=text,
+                        start_time=self.start_time_offset + self._utterance_floor,
+                        end_time=self.start_time_offset + self._audio_sent,
+                    )
+                ],
+            )
+        )
+        if final:
+            self._end_utterance()
+
+    def _end_utterance(self) -> None:
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(type=stt.SpeechEventType.END_OF_SPEECH, request_id=self._request_id)
+        )
+        self._report_usage()
+        self._utterance_floor = self._audio_sent
+        self._speaking = False
+        self._request_id = ""
+
+    def _report_usage(self) -> None:
+        duration = self._audio_sent - self._audio_reported
+        if duration <= 0:
+            return
+        self._audio_reported = self._audio_sent
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(
+                type=stt.SpeechEventType.RECOGNITION_USAGE,
+                recognition_usage=stt.RecognitionUsage(audio_duration=duration),
+            )
+        )

livekit_plugins_lokutor-1.8.12/livekit/plugins/lokutor/tts.py

@@ -0,0 +1,390 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import weakref
+from dataclasses import dataclass, replace
+from typing import Any
+
+import aiohttp
+
+from livekit.agents import (
+    APIConnectionError,
+    APIConnectOptions,
+    APIError,
+    APIStatusError,
+    APITimeoutError,
+    tokenize,
+    tts,
+    utils,
+)
+from livekit.agents.language import LanguageCode
+from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS, NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import is_given
+
+from .log import logger
+from .models import (
+    DEFAULT_BASE_URL,
+    DEFAULT_SAMPLE_RATE,
+    DEFAULT_VOICE_ID,
+    TTS_MODEL,
+    TTSLanguage,
+    VoiceID,
+)
+
+
+@dataclass
+class _TTSOptions:
+    api_key: str
+    voice: str
+    language: str | None
+    base_url: str
+    sample_rate: int
+
+    def get_ws_url(self) -> str:
+        return f"{self.base_url}/ws/tts"
+
+
+class TTS(tts.TTS):
+    """Lokutor Text-to-Speech integration.
+
+    Connects to the Lokutor WebSocket API to synthesize speech. Supports
+    streaming and non-streaming synthesis with multiple voices and languages.
+
+    Args:
+        api_key: Lokutor API key. If not provided, reads from ``LOKUTOR_API_KEY``
+            environment variable.
+        voice: Voice slot to use, ``"F1"``-``"F5"`` or ``"M1"``-``"M5"``. Each
+            language has its own native speaker in each slot. Defaults to ``"F1"``.
+        language: Language code: ``"en"``, ``"es"``, ``"ca"``, ``"gl"``,
+            ``"eu"``, ``"pt"``, ``"fr"``, ``"it"`` or ``"de"``. Any other value
+            uses the English speakers. Defaults to ``"en"``.
+        base_url: Base URL for the Lokutor API. Defaults to
+            ``"wss://api.lokutor.com"``.
+        tokenizer: Sentence tokenizer for streaming input. The agent pushes LLM
+            token deltas; each sentence becomes one request so the prosody is
+            per sentence. Defaults to ``tokenize.blingfire.SentenceTokenizer``.
+        http_session: Optional shared ``aiohttp.ClientSession``. If not
+            provided, a new session will be created.
+
+    Note:
+        Lokutor's TTS endpoint always returns audio at 44.1 kHz; the output
+        sample rate is fixed and not configurable. The LiveKit pipeline
+        resamples downstream if a different rate is required.
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        voice: VoiceID | str = DEFAULT_VOICE_ID,
+        language: TTSLanguage | str | None = "en",
+        base_url: str = DEFAULT_BASE_URL,
+        tokenizer: NotGivenOr[tokenize.SentenceTokenizer] = NOT_GIVEN,
+        http_session: aiohttp.ClientSession | None = None,
+    ) -> None:
+        lokutor_api_key = api_key or os.environ.get("LOKUTOR_API_KEY")
+        if not lokutor_api_key:
+            raise ValueError(
+                "Lokutor API key is required, either as argument or set"
+                " LOKUTOR_API_KEY environment variable"
+            )
+
+        super().__init__(
+            capabilities=tts.TTSCapabilities(
+                streaming=True,
+                aligned_transcript=False,
+            ),
+            sample_rate=DEFAULT_SAMPLE_RATE,
+            num_channels=1,
+        )
+
+        self._opts = _TTSOptions(
+            api_key=lokutor_api_key,
+            voice=voice,
+            language=language,
+            base_url=base_url,
+            sample_rate=DEFAULT_SAMPLE_RATE,
+        )
+
+        self._sentence_tokenizer = (
+            tokenizer if is_given(tokenizer) else tokenize.blingfire.SentenceTokenizer()
+        )
+        self._session = http_session
+        self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse](
+            connect_cb=self._connect_ws,
+            close_cb=self._close_ws,
+            max_session_duration=300,
+            mark_refreshed_on_get=True,
+        )
+        self._streams = weakref.WeakSet[SynthesizeStream]()
+
+    @property
+    def model(self) -> str:
+        """Returns the model name ``"versa-2.0"``."""
+        return TTS_MODEL
+
+    @property
+    def provider(self) -> str:
+        """Returns the provider name ``"Lokutor"``."""
+        return "Lokutor"
+
+    async def _connect_ws(self, timeout: float) -> aiohttp.ClientWebSocketResponse:
+        session = self._ensure_session()
+        # Send the API key in a header rather than the URL query string so the
+        # credential does not end up in exception messages, tracing, or
+        # intermediary access logs.
+        ws = await asyncio.wait_for(
+            session.ws_connect(
+                self._opts.get_ws_url(),
+                max_msg_size=0,
+                headers={"X-API-Key": self._opts.api_key},
+            ),
+            timeout,
+        )
+        return ws
+
+    async def _close_ws(self, ws: aiohttp.ClientWebSocketResponse) -> None:
+        await ws.close()
+
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if not self._session:
+            self._session = utils.http_context.http_session()
+        return self._session
+
+    def prewarm(self) -> None:
+        """Pre-warm the WebSocket connection pool."""
+        self._pool.prewarm()
+
+    def synthesize(
+        self,
+        text: str,
+        *,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> ChunkedStream:
+        """Synthesize text to speech in a single request.
+
+        Args:
+            text: The text to synthesize.
+            conn_options: Connection options for the request.
+
+        Returns:
+            A ``ChunkedStream`` that yields ``SynthesizedAudio`` events.
+        """
+        return ChunkedStream(tts=self, input_text=text, conn_options=conn_options)
+
+    def stream(
+        self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
+    ) -> SynthesizeStream:
+        """Create a streaming TTS session.
+
+        Args:
+            conn_options: Connection options for the stream.
+
+        Returns:
+            A ``SynthesizeStream`` for streaming synthesis.
+        """
+        stream = SynthesizeStream(tts=self, conn_options=conn_options)
+        self._streams.add(stream)
+        return stream
+
+    async def aclose(self) -> None:
+        """Clean up all stream and connection resources."""
+        for stream in list(self._streams):
+            await stream.aclose()
+        self._streams.clear()
+        await self._pool.aclose()
+
+
+async def _receive_audio(
+    ws: aiohttp.ClientWebSocketResponse,
+    output_emitter: tts.AudioEmitter,
+    *,
+    request_id: str,
+    timeout: float,
+) -> None:
+    """Read one request's reply: binary audio frames until the ``EOS`` text frame."""
+    while True:
+        msg = await ws.receive(timeout=timeout)
+        if msg.type in (
+            aiohttp.WSMsgType.CLOSED,
+            aiohttp.WSMsgType.CLOSE,
+            aiohttp.WSMsgType.CLOSING,
+        ):
+            raise APIStatusError(
+                "Lokutor connection closed unexpectedly",
+                request_id=request_id,
+                status_code=ws.close_code or -1,
+                body=None,
+            )
+
+        if msg.type == aiohttp.WSMsgType.BINARY:
+            output_emitter.push(msg.data)
+        elif msg.type == aiohttp.WSMsgType.TEXT:
+            if msg.data == "EOS":
+                return
+            try:
+                data = json.loads(msg.data)
+                if isinstance(data, dict) and data.get("type") == "error":
+                    # Don't interpolate the provider's raw message
+                    # (it may echo customer content) into the error.
+                    raise APIError("Lokutor returned an error response")
+            except json.JSONDecodeError:
+                logger.debug("Ignoring unexpected non-JSON text frame from Lokutor")
+
+
+class ChunkedStream(tts.ChunkedStream):
+    """Non-streaming TTS synthesis.
+
+    Sends the full text in a single WebSocket request and collects all
+    audio before yielding it.
+    """
+
+    def __init__(self, *, tts: TTS, input_text: str, conn_options: APIConnectOptions) -> None:
+        super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
+        self._tts: TTS = tts
+        self._opts = replace(tts._opts)
+
+    async def _run(self, output_emitter: tts.AudioEmitter) -> None:
+        request_id = utils.shortuuid()
+        output_emitter.initialize(
+            request_id=request_id,
+            sample_rate=self._opts.sample_rate,
+            num_channels=1,
+            mime_type="audio/pcm",
+        )
+
+        try:
+            async with self._tts._pool.connection(timeout=self._conn_options.timeout) as ws:
+                request = _build_request(self._opts, self._input_text)
+                await ws.send_str(json.dumps(request))
+                await _receive_audio(
+                    ws,
+                    output_emitter,
+                    request_id=request_id,
+                    timeout=self._conn_options.timeout,
+                )
+
+                output_emitter.flush()
+        except asyncio.TimeoutError:
+            raise APITimeoutError() from None
+        except APIError:
+            # Preserve typed errors raised above (APIStatusError, provider
+            # error) instead of collapsing them into APIConnectionError.
+            raise
+        except aiohttp.ClientResponseError as e:
+            raise APIStatusError(
+                message=e.message, status_code=e.status, request_id=None, body=None
+            ) from None
+        except Exception as e:
+            # Keep only the exception type, and drop the cause: an aiohttp
+            # transport error can carry the request URL, headers or a response
+            # body, and `from e` would preserve all of it on __cause__ for any
+            # telemetry that walks the chain.
+            raise APIConnectionError(type(e).__name__) from None
+
+
+class SynthesizeStream(tts.SynthesizeStream):
+    """Streaming TTS synthesis.
+
+    The input is split into sentences and each sentence is one request on a
+    pooled WebSocket, so audio starts at the first sentence and every sentence
+    has its own prosody. A flush ends the current segment; the next text opens
+    a new one.
+    """
+
+    def __init__(self, *, tts: TTS, conn_options: APIConnectOptions) -> None:
+        super().__init__(tts=tts, conn_options=conn_options)
+        self._tts: TTS = tts
+        self._opts = replace(tts._opts)
+
+    async def _run(self, output_emitter: tts.AudioEmitter) -> None:
+        request_id = utils.shortuuid()
+        output_emitter.initialize(
+            request_id=request_id,
+            sample_rate=self._opts.sample_rate,
+            num_channels=1,
+            mime_type="audio/pcm",
+            stream=True,
+        )
+
+        sent_stream = self._tts._sentence_tokenizer.stream(language=self._opts.language)
+
+        async def _input_task() -> None:
+            try:
+                async for data in self._input_ch:
+                    if isinstance(data, self._FlushSentinel):
+                        sent_stream.flush()
+                        continue
+                    sent_stream.push_text(data)
+            finally:
+                sent_stream.end_input()
+
+        async def _synthesize_task() -> None:
+            async with self._tts._pool.connection(timeout=self._conn_options.timeout) as ws:
+                self._acquire_time = self._tts._pool.last_acquire_time
+                self._connection_reused = self._tts._pool.last_connection_reused
+
+                # A segment is open from its first sentence to the flush (or the end of the
+                # input) that follows it; the tokenizer gives the sentences after a flush a
+                # new segment id.
+                segment_id: str | None = None
+                async for ev in sent_stream:
+                    text = ev.token.strip()
+                    if not text:
+                        continue
+                    if ev.segment_id != segment_id:
+                        if segment_id is not None:
+                            output_emitter.end_segment()
+                        segment_id = ev.segment_id
+                        output_emitter.start_segment(segment_id=segment_id)
+
+                    self._mark_started()
+                    await ws.send_str(json.dumps(_build_request(self._opts, text)))
+                    await _receive_audio(
+                        ws,
+                        output_emitter,
+                        request_id=request_id,
+                        timeout=self._conn_options.timeout,
+                    )
+                    output_emitter.flush()
+
+                if segment_id is not None:
+                    output_emitter.end_segment()
+
+        tasks = [
+            asyncio.create_task(_input_task()),
+            asyncio.create_task(_synthesize_task()),
+        ]
+        try:
+            await asyncio.gather(*tasks)
+        except asyncio.TimeoutError:
+            raise APITimeoutError() from None
+        except APIError:
+            # Preserve typed errors raised above instead of collapsing them
+            # into a generic APIConnectionError.
+            raise
+        except aiohttp.ClientResponseError as e:
+            raise APIStatusError(
+                message=e.message, status_code=e.status, request_id=None, body=None
+            ) from None
+        except Exception as e:
+            # Keep only the exception type, and drop the cause: an aiohttp
+            # transport error can carry the request URL, headers or a response
+            # body, and `from e` would preserve all of it on __cause__ for any
+            # telemetry that walks the chain.
+            raise APIConnectionError(type(e).__name__) from None
+        finally:
+            await utils.aio.cancel_and_wait(*tasks)
+            await sent_stream.aclose()
+
+
+def _build_request(opts: _TTSOptions, text: str) -> dict[str, Any]:
+    request: dict[str, Any] = {
+        "text": text,
+        "voice": opts.voice,
+    }
+    if opts.language:
+        request["language"] = LanguageCode(opts.language).language
+    return request

livekit_plugins_lokutor-1.8.12/livekit/plugins/lokutor/version.py

@@ -0,0 +1,15 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "1.8.12"

livekit_plugins_lokutor-1.8.12/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "livekit-plugins-lokutor"
+dynamic = ["version"]
+description = "Lokutor speech-to-text and text-to-speech plugin for LiveKit agents"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10.0"
+authors = [{ name = "LiveKit", email = "hello@livekit.io" }]
+keywords = ["webrtc", "realtime", "audio", "livekit", "lokutor", "STT", "TTS", "voice"]
+classifiers = [
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Topic :: Multimedia :: Sound/Audio",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3 :: Only",
+]
+dependencies = ["livekit-agents>=1.8.12"]
+
+[project.urls]
+Documentation = "https://docs.livekit.io"
+Website = "https://livekit.io/"
+Source = "https://github.com/livekit/agents"
+
+[tool.hatch.version]
+path = "livekit/plugins/lokutor/version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["livekit"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/livekit"]
+
+[tool.uv]
+exclude-newer = "7 days"
+exclude-newer-package = { livekit-agents = "0 days" }
+