livekit-plugins-meta 1.8.1__tar.gz

livekit_plugins_meta-1.8.1/.gitignore

@@ -0,0 +1,180 @@
+**/.vscode
+**/.DS_Store
+.env
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# trunk
+.trunk/
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+node_modules
+
+credentials.json
+pyrightconfig.json
+docs/
+
+# Database files
+*.db
+
+
+# Examples for development
+examples/dev/*

livekit_plugins_meta-1.8.1/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.5
+Name: livekit-plugins-meta
+Version: 1.8.1
+Summary: Meta Muse Voice Transcribe plugin for LiveKit Agents
+Project-URL: Documentation, https://docs.livekit.io
+Project-URL: Website, https://livekit.io/
+Project-URL: Source, https://github.com/livekit/agents
+Author-email: LiveKit <hello@livekit.io>
+License-Expression: Apache-2.0
+Keywords: audio,livekit,meta,muse,realtime,speech-to-text,stt,webrtc
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10.0
+Requires-Dist: livekit-agents>=1.8.1
+Description-Content-Type: text/markdown
+
+# Meta plugin for LiveKit Agents
+
+Support for Meta [Muse Voice Transcribe](https://dev.meta.ai/docs/speech-to-text) using its [realtime streaming speech-to-text interface](https://dev.meta.ai/docs/api-reference/voice/realtime).
+
+## Installation
+
+```bash
+pip install livekit-plugins-meta
+```
+
+## Pre-requisites
+
+Set a Meta Model API key in your environment using either variable:
+
+```bash
+MODEL_API_KEY=<your_model_api_key>
+# Or: META_API_KEY=<your_model_api_key>
+```
+
+You can also pass the key directly with `api_key=`. The resolution order is `api_key`, `MODEL_API_KEY`, then `META_API_KEY`.
+
+## Usage
+
+```python
+from livekit.agents import AgentSession
+from livekit.plugins import meta
+
+session = AgentSession(
+    stt=meta.STT(
+        keywords=["LiveKit", "Muse"],
+        language_bias=["English"],
+    ),
+    # ... llm, tts, etc.
+)
+```
+
+The plugin supports streaming recognition with server-side endpointing, cumulative interim transcripts, and mono PCM16 audio at 24 kHz. Connected streams must keep sending real-time PCM, including silence. `keywords` and `language_bias` are static hints applied during the initial handshake and cannot be changed on an active stream. Omit `language_bias` for automatic language detection.
+
+Supported language names are: Arabic, Bengali, Dutch, English, French, German, Hebrew, Hindi, Indonesian, Italian, Japanese, Kannada, Korean, Malay, Mandarin Chinese, Marathi, Polish, Portuguese, Spanish, Tagalog, Tamil, Telugu, Thai, Turkish, and Vietnamese. The per-stream `language=` argument also accepts corresponding language codes and locales, such as `en-US`, `pt-BR`, and `zh-CN`, and maps them to the documented names.
+
+Muse realtime sessions have a maximum duration of 60 minutes. This plugin reports the provider close and does not rotate an active LiveKit speech stream automatically; start a new stream to continue. Batch recognition, diarization, detected-language metadata, and active-stream keyterm updates are not supported.

livekit_plugins_meta-1.8.1/README.md

@@ -0,0 +1,41 @@
+# Meta plugin for LiveKit Agents
+
+Support for Meta [Muse Voice Transcribe](https://dev.meta.ai/docs/speech-to-text) using its [realtime streaming speech-to-text interface](https://dev.meta.ai/docs/api-reference/voice/realtime).
+
+## Installation
+
+```bash
+pip install livekit-plugins-meta
+```
+
+## Pre-requisites
+
+Set a Meta Model API key in your environment using either variable:
+
+```bash
+MODEL_API_KEY=<your_model_api_key>
+# Or: META_API_KEY=<your_model_api_key>
+```
+
+You can also pass the key directly with `api_key=`. The resolution order is `api_key`, `MODEL_API_KEY`, then `META_API_KEY`.
+
+## Usage
+
+```python
+from livekit.agents import AgentSession
+from livekit.plugins import meta
+
+session = AgentSession(
+    stt=meta.STT(
+        keywords=["LiveKit", "Muse"],
+        language_bias=["English"],
+    ),
+    # ... llm, tts, etc.
+)
+```
+
+The plugin supports streaming recognition with server-side endpointing, cumulative interim transcripts, and mono PCM16 audio at 24 kHz. Connected streams must keep sending real-time PCM, including silence. `keywords` and `language_bias` are static hints applied during the initial handshake and cannot be changed on an active stream. Omit `language_bias` for automatic language detection.
+
+Supported language names are: Arabic, Bengali, Dutch, English, French, German, Hebrew, Hindi, Indonesian, Italian, Japanese, Kannada, Korean, Malay, Mandarin Chinese, Marathi, Polish, Portuguese, Spanish, Tagalog, Tamil, Telugu, Thai, Turkish, and Vietnamese. The per-stream `language=` argument also accepts corresponding language codes and locales, such as `en-US`, `pt-BR`, and `zh-CN`, and maps them to the documented names.
+
+Muse realtime sessions have a maximum duration of 60 minutes. This plugin reports the provider close and does not rotate an active LiveKit speech stream automatically; start a new stream to continue. Batch recognition, diarization, detected-language metadata, and active-stream keyterm updates are not supported.

livekit_plugins_meta-1.8.1/livekit/plugins/meta/__init__.py

@@ -0,0 +1,40 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Meta Muse Voice Transcribe plugin for LiveKit Agents."""
+
+from .stt import STT, SpeechStream
+from .version import __version__
+
+__all__ = ["STT", "SpeechStream", "__version__"]
+
+from livekit.agents import Plugin
+
+from .log import logger
+
+
+class MetaPlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(MetaPlugin())
+
+# Cleanup docs of unexported modules.
+_module = dir()
+NOT_IN_ALL = [name for name in _module if name not in __all__]
+
+__pdoc__ = {}
+for name in NOT_IN_ALL:
+    __pdoc__[name] = False

livekit_plugins_meta-1.8.1/livekit/plugins/meta/log.py

@@ -0,0 +1,17 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+logger = logging.getLogger("livekit.plugins.meta")

livekit_plugins_meta-1.8.1/livekit/plugins/meta/stt.py

@@ -0,0 +1,801 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import asyncio
+import json
+import math
+import os
+import time
+import weakref
+from collections import OrderedDict, deque
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlparse
+
+import aiohttp
+
+from livekit.agents import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    APIConnectionError,
+    APIConnectOptions,
+    APIError,
+    APIStatusError,
+    APITimeoutError,
+    LanguageCode,
+    stt,
+    utils,
+)
+from livekit.agents.types import NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import AudioBuffer, is_given
+
+DEFAULT_URL = "wss://api.meta.ai/v1/asr/realtime"
+DEFAULT_MODEL = "muse-voice-transcribe-1.0"
+_SAMPLE_RATE = 24_000
+_CHANNELS = 1
+_SAMPLE_WIDTH_BYTES = 2
+_CHUNK_DURATION = 0.08
+_CHUNK_BYTES = int(_SAMPLE_RATE * _CHANNELS * _SAMPLE_WIDTH_BYTES * _CHUNK_DURATION)
+_MAX_MESSAGE_BYTES = 1024 * 1024
+_MAX_COMPLETED_TURNS = 128
+_SUPPORTED_LANGUAGES = (
+    "Arabic",
+    "Bengali",
+    "Dutch",
+    "English",
+    "French",
+    "German",
+    "Hebrew",
+    "Hindi",
+    "Indonesian",
+    "Italian",
+    "Japanese",
+    "Kannada",
+    "Korean",
+    "Malay",
+    "Mandarin Chinese",
+    "Marathi",
+    "Polish",
+    "Portuguese",
+    "Spanish",
+    "Tagalog",
+    "Tamil",
+    "Telugu",
+    "Thai",
+    "Turkish",
+    "Vietnamese",
+)
+_LANGUAGE_NAMES = {language.casefold(): language for language in _SUPPORTED_LANGUAGES}
+_LANGUAGE_CODES = {
+    "ar": "Arabic",
+    "bn": "Bengali",
+    "de": "German",
+    "en": "English",
+    "es": "Spanish",
+    "fil": "Tagalog",
+    "fr": "French",
+    "he": "Hebrew",
+    "hi": "Hindi",
+    "id": "Indonesian",
+    "it": "Italian",
+    "iw": "Hebrew",
+    "ja": "Japanese",
+    "kn": "Kannada",
+    "ko": "Korean",
+    "ms": "Malay",
+    "mr": "Marathi",
+    "nl": "Dutch",
+    "pl": "Polish",
+    "pt": "Portuguese",
+    "ta": "Tamil",
+    "te": "Telugu",
+    "th": "Thai",
+    "tl": "Tagalog",
+    "tr": "Turkish",
+    "vi": "Vietnamese",
+    "zh": "Mandarin Chinese",
+}
+_RETRYABLE_CLOSE_CODES = frozenset((1011, 1013))
+_NON_RETRYABLE_CLOSE_CODES = frozenset((1008,))
+
+
+def _normalize_hints(values: list[str] | None, *, name: str) -> list[str]:
+    normalized: list[str] = []
+    for value in values or ():
+        hint = value.strip()
+        if not hint:
+            raise ValueError(f"{name} entries must be non-empty")
+        if hint not in normalized:
+            normalized.append(hint)
+    return normalized
+
+
+def _normalize_access_token(api_key: str) -> str:
+    parts = api_key.split(None, 1)
+    if parts and parts[0].casefold() == "bearer":
+        if len(parts) != 2 or not parts[1].strip():
+            raise ValueError("Meta Model API key must include a token after Bearer")
+        return f"Bearer {parts[1].strip()}"
+    return f"Bearer {api_key}"
+
+
+def _normalize_language_bias(values: list[str] | None) -> list[str]:
+    normalized: list[str] = []
+    for value in values or ():
+        documented_name = _LANGUAGE_NAMES.get(value.strip().casefold())
+        if documented_name is None:
+            supported = ", ".join(_SUPPORTED_LANGUAGES)
+            raise ValueError(f"unsupported language_bias entry {value!r}; supported: {supported}")
+        if documented_name not in normalized:
+            normalized.append(documented_name)
+    return normalized
+
+
+def _normalize_language_hint(language: str) -> str:
+    value = language.strip()
+    if not value:
+        raise ValueError("language must be non-empty")
+
+    documented_name = _LANGUAGE_NAMES.get(value.casefold())
+    if documented_name is not None:
+        return documented_name
+
+    primary = value.replace("_", "-").split("-", 1)[0].casefold()
+    mapped_name = _LANGUAGE_CODES.get(primary)
+    if mapped_name is None:
+        supported = ", ".join(_SUPPORTED_LANGUAGES)
+        raise ValueError(f"unsupported Muse Voice language {language!r}; supported: {supported}")
+    return mapped_name
+
+
+def _protocol_error(detail: str) -> APIConnectionError:
+    return APIConnectionError(
+        f"Meta Muse realtime ASR protocol error: {detail}",
+        retryable=False,
+    )
+
+
+def _server_error(*, phase: str) -> APIStatusError:
+    return APIStatusError(
+        f"Meta Muse realtime ASR {phase} error",
+        status_code=400,
+        request_id=None,
+        body=None,
+        retryable=False,
+    )
+
+
+def _close_error(
+    close_code: int | None,
+    *,
+    phase: str,
+    retryable_on_normal_close: bool = False,
+) -> APIStatusError:
+    code = close_code or -1
+    if code == 1000:
+        retryable = retryable_on_normal_close
+    elif code in _NON_RETRYABLE_CLOSE_CODES:
+        retryable = False
+    elif code in _RETRYABLE_CLOSE_CODES:
+        retryable = True
+    else:
+        retryable = True
+    return APIStatusError(
+        f"Meta Muse realtime ASR closed during {phase}",
+        status_code=code,
+        body=None,
+        retryable=retryable,
+    )
+
+
+def _parse_ws_message(
+    raw: aiohttp.WSMessage,
+    *,
+    phase: str,
+    close_code: int | None = None,
+    retryable_on_normal_close: bool = False,
+) -> dict[str, Any]:
+    if raw.type != aiohttp.WSMsgType.TEXT or not isinstance(raw.data, str):
+        if raw.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED):
+            raw_close_code = raw.data if isinstance(raw.data, int) else close_code
+            raise _close_error(
+                raw_close_code,
+                phase=phase,
+                retryable_on_normal_close=retryable_on_normal_close,
+            )
+        raise _protocol_error(f"unexpected message type during {phase}")
+    try:
+        message = json.loads(raw.data)
+    except (json.JSONDecodeError, TypeError):
+        raise _protocol_error(f"invalid JSON during {phase}") from None
+    if not isinstance(message, dict):
+        raise _protocol_error(f"non-object message during {phase}")
+    return message
+
+
+async def _close_quietly(ws: aiohttp.ClientWebSocketResponse) -> None:
+    try:
+        await ws.close()
+    except Exception:
+        pass
+
+
+def _normalize_turn_id(value: object, *, event: str) -> str:
+    if isinstance(value, bool) or not isinstance(value, (str, int)):
+        raise _protocol_error(f"{event} event has an invalid turnId")
+    turn_id = str(value).strip()
+    if not turn_id:
+        raise _protocol_error(f"{event} event has an invalid turnId")
+    return turn_id
+
+
+@dataclass(slots=True)
+class _TurnState:
+    provider_started: bool = False
+    emitted_start: bool = False
+    latest_interim: str | None = None
+    emitted_interim: str | None = None
+    final_text: str | None = None
+    final_emitted: bool = False
+    ended: bool = False
+    usage_seconds: float = 0.0
+    usage_captured: bool = False
+
+
+class STT(stt.STT[Any]):
+    """Streaming speech recognition with Meta Muse Voice Transcribe."""
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        model: str = DEFAULT_MODEL,
+        url: str = DEFAULT_URL,
+        keywords: list[str] | None = None,
+        language_bias: list[str] | None = None,
+        http_session: aiohttp.ClientSession | None = None,
+    ) -> None:
+        """Create a Meta Muse streaming STT provider.
+
+        Args:
+            api_key: Meta Model API key. Falls back to ``MODEL_API_KEY``, then
+                ``META_API_KEY``.
+            model: Muse Voice Transcribe model identifier.
+            url: Realtime Muse ASR WebSocket endpoint. Must use ``wss://``.
+            keywords: Static recognition keywords sent when each stream starts.
+            language_bias: Static supported language names sent when each stream starts.
+            http_session: Optional aiohttp session. By default, the LiveKit HTTP
+                context session is used.
+        """
+        if api_key is not None:
+            resolved_key = api_key.strip()
+        else:
+            resolved_key = (
+                os.getenv("MODEL_API_KEY", "").strip() or os.getenv("META_API_KEY", "").strip()
+            )
+        if not resolved_key:
+            raise ValueError(
+                "Meta Model API key is required. Pass api_key or set MODEL_API_KEY or META_API_KEY"
+            )
+        if not model.strip():
+            raise ValueError("model must be non-empty")
+
+        parsed_url = urlparse(url)
+        if (
+            parsed_url.scheme != "wss"
+            or not parsed_url.hostname
+            or parsed_url.username is not None
+            or parsed_url.password is not None
+            or parsed_url.fragment
+        ):
+            raise ValueError("url must be an absolute wss:// URL without credentials or a fragment")
+
+        super().__init__(
+            capabilities=stt.STTCapabilities(
+                streaming=True,
+                interim_results=True,
+                diarization=False,
+                aligned_transcript=False,
+                offline_recognize=False,
+                keyterms=False,
+            )
+        )
+        self._api_key = _normalize_access_token(resolved_key)
+        self._model = model.strip()
+        self._url = url
+        self._keywords = _normalize_hints(keywords, name="keywords")
+        self._language_bias = _normalize_language_bias(language_bias)
+        self._http_session = http_session
+        self._streams: weakref.WeakSet[SpeechStream] = weakref.WeakSet()
+        self._closed = False
+
+    @property
+    def model(self) -> str:
+        return self._model
+
+    @property
+    def provider(self) -> str:
+        return "Meta"
+
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if self._http_session is None:
+            self._http_session = utils.http_context.http_session()
+        return self._http_session
+
+    async def _recognize_impl(
+        self,
+        buffer: AudioBuffer,
+        *,
+        language: NotGivenOr[str] = NOT_GIVEN,
+        conn_options: APIConnectOptions,
+    ) -> stt.SpeechEvent:
+        raise APIError(
+            "Meta Muse Voice Transcribe supports streaming recognition only",
+            retryable=False,
+        )
+
+    def stream(
+        self,
+        *,
+        language: NotGivenOr[str] = NOT_GIVEN,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> SpeechStream:
+        if self._closed:
+            raise RuntimeError("Meta STT is closed")
+
+        language_bias = list(self._language_bias)
+        if is_given(language):
+            language_hint = _normalize_language_hint(str(language))
+            if language_hint not in language_bias:
+                language_bias.append(language_hint)
+
+        stream = SpeechStream(
+            stt=self,
+            conn_options=conn_options,
+            api_key=self._api_key,
+            model=self._model,
+            url=self._url,
+            keywords=list(self._keywords),
+            language_bias=language_bias,
+            http_session=self._ensure_session(),
+        )
+        self._streams.add(stream)
+        return stream
+
+    async def aclose(self) -> None:
+        self._closed = True
+        streams = tuple(self._streams)
+        if streams:
+            await asyncio.gather(*(stream.aclose() for stream in streams), return_exceptions=True)
+
+
+class SpeechStream(stt.RecognizeStream):
+    def __init__(
+        self,
+        *,
+        stt: STT,
+        conn_options: APIConnectOptions,
+        api_key: str,
+        model: str,
+        url: str,
+        keywords: list[str],
+        language_bias: list[str],
+        http_session: aiohttp.ClientSession,
+    ) -> None:
+        super().__init__(stt=stt, conn_options=conn_options, sample_rate=_SAMPLE_RATE)
+        self._api_key = api_key
+        self._model = model
+        self._url = url
+        self._keywords = keywords
+        self._language_bias = language_bias
+        self._session = http_session
+        self._session_id = ""
+        self._turns: OrderedDict[str, _TurnState] = OrderedDict()
+        self._provider_active_turn_id: str | None = None
+        self._completed_turn_ids: set[str] = set()
+        self._completed_turn_order: deque[str] = deque()
+        self._audio_consumed = False
+        self._end_stream_sent = False
+        self._last_audio_processed_ms = 0.0
+        self._pending_usage_seconds = 0.0
+
+    async def _run(self) -> None:
+        self._end_stream_sent = False
+        self._last_audio_processed_ms = 0.0
+        ws: aiohttp.ClientWebSocketResponse | None = None
+        tasks: list[asyncio.Task[None]] = []
+        try:
+            ws = await self._connect_ws()
+            sender = asyncio.create_task(self._send_audio(ws), name="meta-stt-send")
+            receiver = asyncio.create_task(self._receive_events(ws), name="meta-stt-receive")
+            tasks = [sender, receiver]
+            await self._drive_tasks(sender, receiver)
+        except asyncio.CancelledError:
+            raise
+        except APIError as exc:
+            if self._audio_consumed and exc.retryable:
+                raise APIConnectionError(
+                    "Meta Muse realtime ASR failed after audio was consumed",
+                    retryable=False,
+                ) from None
+            raise
+        except Exception as exc:
+            phase = "audio streaming" if self._audio_consumed else "connection"
+            raise APIConnectionError(
+                f"Meta Muse realtime ASR {phase} failed ({type(exc).__name__})",
+                retryable=not self._audio_consumed,
+            ) from None
+        finally:
+            if tasks:
+                await utils.aio.gracefully_cancel(*tasks)
+            self._flush_usage()
+            if ws is not None:
+                await _close_quietly(ws)
+
+    async def _connect_ws(self) -> aiohttp.ClientWebSocketResponse:
+        started_at = time.perf_counter()
+        try:
+            ws = await asyncio.wait_for(
+                self._session.ws_connect(
+                    self._url,
+                    max_msg_size=_MAX_MESSAGE_BYTES,
+                ),
+                timeout=self._conn_options.timeout,
+            )
+        except asyncio.TimeoutError:
+            raise APITimeoutError("Meta Muse realtime ASR connection timed out") from None
+        except aiohttp.ClientResponseError as exc:
+            raise APIStatusError(
+                "Meta Muse realtime ASR connection was rejected",
+                status_code=exc.status,
+                body=None,
+            ) from None
+        except Exception as exc:
+            raise APIConnectionError(
+                f"Meta Muse realtime ASR connection failed ({type(exc).__name__})"
+            ) from None
+
+        try:
+            await ws.send_str(json.dumps(self._handshake(), separators=(",", ":")))
+            raw = await asyncio.wait_for(ws.receive(), timeout=self._conn_options.timeout)
+            message = _parse_ws_message(
+                raw,
+                phase="handshake",
+                close_code=ws.close_code,
+                retryable_on_normal_close=True,
+            )
+            self._accept_handshake(message)
+        except asyncio.CancelledError:
+            await _close_quietly(ws)
+            raise
+        except asyncio.TimeoutError:
+            await _close_quietly(ws)
+            raise APITimeoutError("Meta Muse realtime ASR handshake timed out") from None
+        except APIError:
+            await _close_quietly(ws)
+            raise
+        except Exception as exc:
+            await _close_quietly(ws)
+            raise APIConnectionError(
+                f"Meta Muse realtime ASR handshake failed ({type(exc).__name__})"
+            ) from None
+
+        self._report_connection_acquired(time.perf_counter() - started_at, False)
+        return ws
+
+    def _handshake(self) -> dict[str, object]:
+        handshake: dict[str, object] = {
+            "mode": "ENDPOINTING",
+            "authorization": {"accessToken": self._api_key},
+            "audioEncoding": "PCM_24KHZ",
+            "model": self._model,
+            "partialMode": "CUMULATIVE",
+            "emitAudioProgress": True,
+        }
+        if self._keywords:
+            handshake["keywords"] = self._keywords
+        if self._language_bias:
+            handshake["languageBias"] = self._language_bias
+        return handshake
+
+    def _accept_handshake(self, message: dict[str, Any]) -> None:
+        if message.get("type") == "error":
+            raise _server_error(phase="handshake")
+        session_id = message.get("sessionId")
+        if not isinstance(session_id, str) or not session_id:
+            raise APIConnectionError(
+                "Meta Muse realtime ASR sent an invalid handshake response",
+                retryable=False,
+            )
+        self._session_id = session_id
+
+    async def _drive_tasks(self, sender: asyncio.Task[None], receiver: asyncio.Task[None]) -> None:
+        done, _ = await asyncio.wait((sender, receiver), return_when=asyncio.FIRST_COMPLETED)
+        if receiver in done:
+            receiver.result()
+            if not self._end_stream_sent:
+                raise APIConnectionError(
+                    "Meta Muse realtime ASR closed before input ended",
+                    retryable=not self._audio_consumed,
+                )
+            if not sender.done():
+                await sender
+        else:
+            sender.result()
+            try:
+                await asyncio.wait_for(receiver, timeout=self._conn_options.timeout)
+            except asyncio.TimeoutError:
+                raise APITimeoutError(
+                    "Meta Muse realtime ASR timed out while draining final events",
+                    retryable=not self._audio_consumed,
+                ) from None
+        self._validate_clean_close()
+
+    async def _send_audio(self, ws: aiohttp.ClientWebSocketResponse) -> None:
+        pending = bytearray()
+        loop = asyncio.get_running_loop()
+        pacing_origin: float | None = None
+        sent_duration = 0.0
+
+        async def send_packet(packet: bytes) -> None:
+            nonlocal pacing_origin, sent_duration
+            if not packet:
+                return
+            if pacing_origin is None:
+                pacing_origin = loop.time()
+            deadline = pacing_origin + sent_duration
+            delay = deadline - loop.time()
+            if delay > 0:
+                await asyncio.sleep(delay)
+            try:
+                await ws.send_bytes(packet)
+            except Exception as exc:
+                raise APIConnectionError(
+                    f"Meta Muse realtime ASR audio send failed ({type(exc).__name__})",
+                    retryable=False,
+                ) from None
+            duration = len(packet) / (_SAMPLE_RATE * _CHANNELS * _SAMPLE_WIDTH_BYTES)
+            sent_duration += duration
+
+        async for item in self._input_ch:
+            if isinstance(item, self._FlushSentinel):
+                if pending:
+                    await send_packet(bytes(pending))
+                    pending.clear()
+                continue
+
+            self._audio_consumed = True
+            if item.num_channels != _CHANNELS:
+                raise APIError("Meta Muse realtime ASR requires mono audio", retryable=False)
+            pending.extend(item.data.tobytes())
+            while len(pending) >= _CHUNK_BYTES:
+                await send_packet(bytes(pending[:_CHUNK_BYTES]))
+                del pending[:_CHUNK_BYTES]
+
+        if pending:
+            await send_packet(bytes(pending))
+        if not self._end_stream_sent:
+            try:
+                await ws.send_str('{"type":"endStream"}')
+            except Exception as exc:
+                raise APIConnectionError(
+                    f"Meta Muse realtime ASR end-of-input send failed ({type(exc).__name__})",
+                    retryable=False,
+                ) from None
+            self._end_stream_sent = True
+
+    async def _receive_events(self, ws: aiohttp.ClientWebSocketResponse) -> None:
+        while True:
+            try:
+                raw = await ws.receive()
+            except Exception as exc:
+                raise APIConnectionError(
+                    f"Meta Muse realtime ASR receive failed ({type(exc).__name__})",
+                    retryable=not self._audio_consumed,
+                ) from None
+
+            if raw.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED):
+                close_code = raw.data if isinstance(raw.data, int) else ws.close_code
+                if self._end_stream_sent and close_code == 1000:
+                    return
+                raise _close_error(
+                    close_code,
+                    phase="stream",
+                    retryable_on_normal_close=not self._audio_consumed,
+                )
+            if raw.type == aiohttp.WSMsgType.CLOSING:
+                continue
+            if raw.type == aiohttp.WSMsgType.ERROR:
+                raise APIConnectionError(
+                    "Meta Muse realtime ASR WebSocket failed",
+                    retryable=not self._audio_consumed,
+                ) from None
+
+            message = _parse_ws_message(raw, phase="stream")
+            event_type = message.get("type")
+            if event_type == "error":
+                raise _server_error(phase="stream")
+            if event_type == "speechStart":
+                self._speech_start(message)
+            elif event_type == "transcript":
+                self._transcript(message)
+            elif event_type == "speechEnd":
+                self._speech_end(message)
+            elif event_type == "speechComplete":
+                self._speech_complete(message)
+            elif event_type == "audioProgress":
+                self._audio_progress(message)
+
+    def _audio_progress(self, message: dict[str, Any]) -> None:
+        processed_ms = message.get("audioProcessedMs")
+        if (
+            isinstance(processed_ms, bool)
+            or not isinstance(processed_ms, (int, float))
+            or not math.isfinite(processed_ms)
+            or processed_ms < 0
+        ):
+            raise _protocol_error("audioProgress event has invalid audioProcessedMs")
+        if processed_ms <= self._last_audio_processed_ms:
+            return
+        delta_seconds = (processed_ms - self._last_audio_processed_ms) / 1000
+        self._last_audio_processed_ms = float(processed_ms)
+        self._pending_usage_seconds += delta_seconds
+
+    def _speech_start(self, message: dict[str, Any]) -> None:
+        turn_id = self._required_turn_id(message, event="speechStart")
+        if turn_id in self._completed_turn_ids:
+            return
+        turn = self._turns.setdefault(turn_id, _TurnState())
+        turn.provider_started = True
+        self._provider_active_turn_id = turn_id
+        self._drain_turns()
+
+    def _transcript(self, message: dict[str, Any]) -> None:
+        text = message.get("transcript")
+        if not isinstance(text, str):
+            raise _protocol_error("transcript event has invalid text")
+        if not text and message.get("turnId") is None and self._provider_active_turn_id is None:
+            return
+        turn_id = self._transcript_turn_id(message)
+        if turn_id in self._completed_turn_ids:
+            return
+        turn = self._turns.setdefault(turn_id, _TurnState())
+        if turn.final_text is not None or turn.latest_interim == text:
+            return
+        turn.latest_interim = text
+        self._drain_turns()
+
+    def _speech_end(self, message: dict[str, Any]) -> None:
+        turn_id = self._required_turn_id(message, event="speechEnd")
+        if turn_id in self._completed_turn_ids:
+            return
+        turn = self._turns.setdefault(turn_id, _TurnState())
+        turn.ended = True
+        if self._provider_active_turn_id == turn_id:
+            self._provider_active_turn_id = None
+        self._capture_turn_usage(turn)
+        self._drain_turns()
+
+    def _speech_complete(self, message: dict[str, Any]) -> None:
+        turn_id = self._required_turn_id(message, event="speechComplete")
+        if turn_id in self._completed_turn_ids:
+            return
+        text = message.get("transcript")
+        if not isinstance(text, str):
+            raise _protocol_error("speechComplete event has invalid transcript")
+        turn = self._turns.setdefault(turn_id, _TurnState())
+        if turn.final_text is None:
+            turn.final_text = text
+        self._capture_turn_usage(turn)
+        self._drain_turns()
+
+    def _capture_turn_usage(self, turn: _TurnState) -> None:
+        if turn.usage_captured or not turn.ended:
+            return
+        turn.usage_seconds = self._pending_usage_seconds
+        turn.usage_captured = True
+        self._pending_usage_seconds = 0.0
+
+    def _drain_turns(self) -> None:
+        while self._turns:
+            turn_id = next(iter(self._turns))
+            turn = self._turns[turn_id]
+            has_content = turn.latest_interim is not None or turn.final_text is not None
+            if not turn.emitted_start and (turn.provider_started or has_content):
+                turn.emitted_start = True
+                self._emit(stt.SpeechEventType.START_OF_SPEECH, turn_id)
+
+            if (
+                turn.emitted_start
+                and not turn.final_emitted
+                and turn.latest_interim is not None
+                and turn.latest_interim != turn.emitted_interim
+            ):
+                turn.emitted_interim = turn.latest_interim
+                self._emit(
+                    stt.SpeechEventType.INTERIM_TRANSCRIPT,
+                    turn_id,
+                    turn.latest_interim,
+                )
+
+            if turn.emitted_start and not turn.final_emitted and turn.final_text is not None:
+                turn.final_emitted = True
+                self._emit(stt.SpeechEventType.FINAL_TRANSCRIPT, turn_id, turn.final_text)
+
+            if not (turn.final_emitted and turn.ended):
+                return
+
+            self._emit(stt.SpeechEventType.END_OF_SPEECH, turn_id)
+            self._emit_usage(turn.usage_seconds)
+            del self._turns[turn_id]
+            self._remember_completed_turn(turn_id)
+
+    def _remember_completed_turn(self, turn_id: str) -> None:
+        if turn_id in self._completed_turn_ids:
+            return
+        if len(self._completed_turn_order) >= _MAX_COMPLETED_TURNS:
+            oldest = self._completed_turn_order.popleft()
+            self._completed_turn_ids.discard(oldest)
+        self._completed_turn_order.append(turn_id)
+        self._completed_turn_ids.add(turn_id)
+
+    def _emit(self, event_type: stt.SpeechEventType, turn_id: str, text: str | None = None) -> None:
+        alternatives: list[stt.SpeechData] = []
+        if text is not None:
+            alternatives.append(stt.SpeechData(language=LanguageCode(""), text=text))
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(type=event_type, request_id=turn_id, alternatives=alternatives)
+        )
+
+    def _emit_usage(self, duration: float) -> None:
+        if duration <= 0:
+            return
+        self._event_ch.send_nowait(
+            stt.SpeechEvent(
+                type=stt.SpeechEventType.RECOGNITION_USAGE,
+                request_id=self._session_id,
+                recognition_usage=stt.RecognitionUsage(audio_duration=duration),
+            )
+        )
+
+    def _flush_usage(self) -> None:
+        duration = self._pending_usage_seconds
+        self._pending_usage_seconds = 0.0
+        for turn in self._turns.values():
+            duration += turn.usage_seconds
+            turn.usage_seconds = 0.0
+        self._emit_usage(duration)
+
+    def _transcript_turn_id(self, message: dict[str, Any]) -> str:
+        value = message.get("turnId")
+        if value is not None:
+            return _normalize_turn_id(value, event="transcript")
+
+        if self._provider_active_turn_id is not None:
+            return self._provider_active_turn_id
+        raise _protocol_error("transcript event is missing turnId outside an active turn")
+
+    def _required_turn_id(self, message: dict[str, Any], *, event: str) -> str:
+        value = message.get("turnId")
+        if value is None:
+            raise _protocol_error(f"{event} event is missing turnId")
+        return _normalize_turn_id(value, event=event)
+
+    def _validate_clean_close(self) -> None:
+        if self._turns:
+            raise APIConnectionError(
+                "Meta Muse realtime ASR closed with incomplete speech turns",
+                retryable=False,
+            )

livekit_plugins_meta-1.8.1/livekit/plugins/meta/version.py

@@ -0,0 +1,15 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "1.8.1"

livekit_plugins_meta-1.8.1/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "livekit-plugins-meta"
+dynamic = ["version"]
+description = "Meta Muse Voice Transcribe plugin for LiveKit Agents"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10.0"
+authors = [{ name = "LiveKit", email = "hello@livekit.io" }]
+keywords = [
+    "webrtc",
+    "realtime",
+    "audio",
+    "livekit",
+    "meta",
+    "muse",
+    "speech-to-text",
+    "stt",
+]
+classifiers = [
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Topic :: Multimedia :: Sound/Audio",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3 :: Only",
+]
+dependencies = ["livekit-agents>=1.8.1"]
+
+[project.urls]
+Documentation = "https://docs.livekit.io"
+Website = "https://livekit.io/"
+Source = "https://github.com/livekit/agents"
+
+[tool.hatch.version]
+path = "livekit/plugins/meta/version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["livekit"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/livekit"]
+
+[tool.uv]
+exclude-newer = "7 days"
+exclude-newer-package = { livekit-agents = "0 days" }