livekit-plugins-speechmatics 0.0.2__py3-none-any.whl

livekit/plugins/speechmatics/__init__.py

@@ -0,0 +1,32 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .log import logger
+from .stt import STT, SpeechStream
+from .version import __version__
+
+__all__ = [
+    "STT",
+    "SpeechStream",
+    "logger",
+    "__version__",
+]
+
+from livekit.agents import Plugin
+
+
+class SpeechmaticsPlugin(Plugin):
+    def __init__(self):
+        super().__init__(__name__, __version__, __package__)
+
+
+Plugin.register_plugin(SpeechmaticsPlugin())

livekit/plugins/speechmatics/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.speechmatics")

livekit/plugins/speechmatics/stt.py

@@ -0,0 +1,328 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+import json
+import os
+import weakref
+from typing import Dict, List, Optional
+
+import aiohttp
+from livekit.agents import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    APIConnectOptions,
+    APIStatusError,
+    stt,
+    utils,
+)
+from livekit.agents.utils import AudioBuffer
+
+from .log import logger
+from .types import (
+    AudioSettings,
+    ClientMessageType,
+    ConnectionSettings,
+    ServerMessageType,
+    TranscriptionConfig,
+)
+from .utils import get_access_token, sanitize_url
+
+
+class STT(stt.STT):
+    def __init__(
+        self,
+        *,
+        transcription_config: TranscriptionConfig = TranscriptionConfig(
+            language="en",
+            operating_point="enhanced",
+            enable_partials=True,
+            max_delay=0.7,
+        ),
+        connection_settings: ConnectionSettings = ConnectionSettings(
+            url="wss://eu2.rt.speechmatics.com/v2",
+        ),
+        audio_settings: AudioSettings = AudioSettings(),
+        http_session: Optional[aiohttp.ClientSession] = None,
+        extra_headers: Optional[Dict] = None,
+    ):
+        super().__init__(
+            capabilities=stt.STTCapabilities(
+                streaming=True,
+                interim_results=True,
+            ),
+        )
+        self._transcription_config = transcription_config
+        self._audio_settings = audio_settings
+        self._connection_settings = connection_settings
+        self._extra_headers = extra_headers or {}
+        self._session = http_session
+        self._streams = weakref.WeakSet[SpeechStream]()
+
+    @property
+    def session(self) -> aiohttp.ClientSession:
+        if not self._session:
+            self._session = utils.http_context.http_session()
+        return self._session
+
+    async def _recognize_impl(
+        self,
+        buffer: AudioBuffer,
+        *,
+        language: str | None,
+        conn_options: APIConnectOptions,
+    ) -> stt.SpeechEvent:
+        raise NotImplementedError("Not implemented")
+
+    def stream(
+        self,
+        *,
+        language: Optional[str] = None,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> "SpeechStream":
+        config = dataclasses.replace(self._audio_settings)
+        stream = SpeechStream(
+            stt=self,
+            transcription_config=self._transcription_config,
+            audio_settings=config,
+            connection_settings=self._connection_settings,
+            conn_options=conn_options,
+            http_session=self.session,
+            extra_headers=self._extra_headers,
+        )
+        self._streams.add(stream)
+        return stream
+
+
+class SpeechStream(stt.SpeechStream):
+    def __init__(
+        self,
+        *,
+        stt: STT,
+        transcription_config: TranscriptionConfig,
+        audio_settings: AudioSettings,
+        connection_settings: ConnectionSettings,
+        conn_options: APIConnectOptions,
+        http_session: aiohttp.ClientSession,
+        extra_headers: Optional[Dict] = None,
+    ) -> None:
+        super().__init__(
+            stt=stt, conn_options=conn_options, sample_rate=audio_settings.sample_rate
+        )
+        self._transcription_config = transcription_config
+        self._audio_settings = audio_settings
+        self._connection_settings = connection_settings
+        self._session = http_session
+        self._extra_headers = extra_headers or {}
+        self._speech_duration: float = 0
+
+        self._reconnect_event = asyncio.Event()
+        self._recognition_started = asyncio.Event()
+        self._seq_no = 0
+
+    async def _run(self):
+        closing_ws = False
+
+        async def send_task(ws: aiohttp.ClientWebSocketResponse):
+            nonlocal closing_ws
+
+            start_recognition_msg = {
+                "message": ClientMessageType.StartRecognition,
+                "audio_format": self._audio_settings.asdict(),
+                "transcription_config": self._transcription_config.asdict(),
+            }
+            await ws.send_str(json.dumps(start_recognition_msg))
+
+            await self._recognition_started.wait()
+
+            audio_bstream = utils.audio.AudioByteStream(
+                sample_rate=self._audio_settings.sample_rate,
+                num_channels=1,
+            )
+
+            async for data in self._input_ch:
+                if isinstance(data, self._FlushSentinel):
+                    frames = audio_bstream.flush()
+                else:
+                    frames = audio_bstream.write(data.data.tobytes())
+
+                for frame in frames:
+                    self._seq_no += 1
+                    self._speech_duration += frame.duration
+                    await ws.send_bytes(frame.data.tobytes())
+
+            closing_ws = True
+            await ws.send_str(
+                json.dumps(
+                    {
+                        "message": ClientMessageType.EndOfStream,
+                        "last_seq_no": self._seq_no,
+                    }
+                )
+            )
+
+        async def recv_task(ws: aiohttp.ClientWebSocketResponse):
+            nonlocal closing_ws
+            while True:
+                msg = await ws.receive()
+                if msg.type in (
+                    aiohttp.WSMsgType.CLOSED,
+                    aiohttp.WSMsgType.CLOSE,
+                    aiohttp.WSMsgType.CLOSING,
+                ):
+                    if closing_ws:  # close is expected, see SpeechStream.aclose
+                        return
+
+                    # this will trigger a reconnection, see the _run loop
+                    raise APIStatusError(
+                        message="Speechmatics connection closed unexpectedly"
+                    )
+
+                try:
+                    data = json.loads(msg.data)
+                    self._process_stream_event(data, closing_ws)
+                except Exception:
+                    logger.exception("failed to process Speechmatics message")
+
+        ws: aiohttp.ClientWebSocketResponse | None = None
+
+        while True:
+            try:
+                ws = await self._connect_ws()
+                tasks = [
+                    asyncio.create_task(send_task(ws)),
+                    asyncio.create_task(recv_task(ws)),
+                ]
+                wait_reconnect_task = asyncio.create_task(self._reconnect_event.wait())
+
+                try:
+                    done, _ = await asyncio.wait(
+                        [asyncio.gather(*tasks), wait_reconnect_task],
+                        return_when=asyncio.FIRST_COMPLETED,
+                    )  # type: ignore
+                    for task in done:
+                        if task != wait_reconnect_task:
+                            task.result()
+
+                    if wait_reconnect_task not in done:
+                        break
+
+                    self._reconnect_event.clear()
+                finally:
+                    await utils.aio.gracefully_cancel(*tasks, wait_reconnect_task)
+            finally:
+                if ws is not None:
+                    await ws.close()
+
+    async def _connect_ws(self) -> aiohttp.ClientWebSocketResponse:
+        api_key = self._connection_settings.api_key or os.environ.get(
+            "SPEECHMATICS_API_KEY"
+        )
+        if api_key is None:
+            raise ValueError(
+                "Speechmatics API key is required. "
+                "Pass one in via ConnectionSettings.api_key parameter, "
+                "or set `SPEECHMATICS_API_KEY` environment variable"
+            )
+        if self._connection_settings.get_access_token:
+            api_key = await get_access_token(api_key)
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            **self._extra_headers,
+        }
+        url = sanitize_url(
+            self._connection_settings.url, self._transcription_config.language
+        )
+        return await self._session.ws_connect(
+            url,
+            ssl=self._connection_settings.ssl_context,
+            headers=headers,
+        )
+
+    def _process_stream_event(self, data: dict, closing_ws: bool) -> None:
+        message_type = data["message"]
+
+        if message_type == ServerMessageType.RecognitionStarted:
+            self._recognition_started.set()
+
+        elif message_type == ServerMessageType.AddPartialTranscript:
+            alts = live_transcription_to_speech_data(data)
+            if len(alts) > 0 and alts[0].text:
+                interim_event = stt.SpeechEvent(
+                    type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
+                    alternatives=alts,
+                )
+                self._event_ch.send_nowait(interim_event)
+
+        elif message_type == ServerMessageType.AddTranscript:
+            alts = live_transcription_to_speech_data(data)
+            if len(alts) > 0 and alts[0].text:
+                final_event = stt.SpeechEvent(
+                    type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+                    alternatives=alts,
+                )
+                self._event_ch.send_nowait(final_event)
+
+            if self._speech_duration > 0:
+                usage_event = stt.SpeechEvent(
+                    type=stt.SpeechEventType.RECOGNITION_USAGE,
+                    alternatives=[],
+                    recognition_usage=stt.RecognitionUsage(
+                        audio_duration=self._speech_duration
+                    ),
+                )
+                self._event_ch.send_nowait(usage_event)
+                self._speech_duration = 0
+
+        elif message_type == ServerMessageType.EndOfTranscript:
+            if closing_ws:
+                pass
+            else:
+                raise Exception("Speechmatics connection closed unexpectedly")
+
+
+def live_transcription_to_speech_data(data: dict) -> List[stt.SpeechData]:
+    speech_data: List[stt.SpeechData] = []
+
+    for result in data.get("results", []):
+        start_time, end_time, is_eos = (
+            result.get("start_time", 0),
+            result.get("end_time", 0),
+            result.get("is_eos", False),
+        )
+
+        for alt in result.get("alternatives", []):
+            content, confidence, language = (
+                alt.get("content", "").strip(),
+                alt.get("confidence", 1.0),
+                alt.get("language", "en"),
+            )
+
+            if not content:
+                continue
+
+            # append punctuation to the previous result
+            if is_eos and speech_data:
+                speech_data[-1].text += content
+            elif speech_data and start_time == speech_data[-1].end_time:
+                speech_data[-1].text += " " + content
+            else:
+                speech_data.append(
+                    stt.SpeechData(language, content, start_time, end_time, confidence)
+                )
+
+    return speech_data

livekit/plugins/speechmatics/types.py

@@ -0,0 +1,141 @@
+import ssl
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class TranscriptionConfig:
+    """Real-time: Defines transcription parameters."""
+
+    language: str = "en"
+    """ISO 639-1 language code. eg. `en`"""
+
+    operating_point: Optional[str] = None
+    """Specifies which acoustic model to use."""
+
+    output_locale: Optional[str] = None
+    """RFC-5646 language code for transcript output. eg. `en-AU`"""
+
+    diarization: Optional[str] = None
+    """Indicates type of diarization to use, if any."""
+
+    additional_vocab: Optional[Dict] = None
+    """Additional vocabulary that is not part of the standard language."""
+
+    punctuation_overrides: Optional[Dict] = None
+    """Permitted puctuation marks for advanced punctuation."""
+
+    enable_entities: Optional[bool] = None
+    """Indicates if inverse text normalization entity output is enabled."""
+
+    max_delay: Optional[float] = None
+    """Maximum acceptable delay."""
+
+    max_delay_mode: Optional[str] = None
+    """Determines whether the threshold specified in max_delay can be exceeded
+    if a potential entity is detected. Flexible means if a potential entity
+    is detected, then the max_delay can be overriden until the end of that
+    entity. Fixed means that max_delay specified ignores any potential
+    entity that would not be completed within that threshold."""
+
+    streaming_mode: Optional[bool] = None
+    """Indicates if we run the engine in streaming mode, or regular RT mode."""
+
+    enable_partials: Optional[bool] = None
+    """Indicates if partials for transcription, where words are produced
+    immediately, is enabled."""
+
+    def asdict(self) -> Dict[Any, Any]:
+        """Returns model as a dict while excluding None values recursively."""
+        return asdict(
+            self, dict_factory=lambda x: {k: v for (k, v) in x if v is not None}
+        )
+
+
+@dataclass
+class AudioSettings:
+    """Real-time: Defines audio parameters."""
+
+    encoding: str = "pcm_s16le"
+    """Encoding format when raw audio is used. Allowed values are
+    `pcm_f32le`, `pcm_s16le` and `mulaw`."""
+
+    sample_rate: int = 16000
+    """Sampling rate in hertz."""
+
+    def asdict(self):
+        return {
+            "type": "raw",
+            "encoding": self.encoding,
+            "sample_rate": self.sample_rate,
+        }
+
+
+@dataclass
+class ConnectionSettings:
+    """Defines connection parameters."""
+
+    url: str
+    """Websocket server endpoint."""
+
+    ssl_context: ssl.SSLContext = field(default_factory=ssl.create_default_context)
+    """SSL context."""
+
+    api_key: Optional[str] = None
+    """api key to authenticate a customer."""
+
+    get_access_token: Optional[bool] = True
+    """Automatically generate a temporary token for authentication."""
+
+
+class ClientMessageType(str, Enum):
+    # pylint: disable=invalid-name
+    """Real-time: Defines various messages sent from client to server."""
+
+    StartRecognition = "StartRecognition"
+    """Initiates a recognition job based on configuration set previously."""
+
+    AddAudio = "AddAudio"
+    """Adds more audio data to the recognition job. The server confirms
+    receipt by sending an :py:attr:`ServerMessageType.AudioAdded` message."""
+
+    EndOfStream = "EndOfStream"
+    """Indicates that the client has no more audio to send."""
+
+    SetRecognitionConfig = "SetRecognitionConfig"
+    """Allows the client to re-configure the recognition session."""
+
+
+class ServerMessageType(str, Enum):
+    """Real-time: Defines various message types sent from server to client."""
+
+    RecognitionStarted = "RecognitionStarted"
+    """Server response to :py:attr:`ClientMessageType.StartRecognition`,
+    acknowledging that a recognition session has started."""
+
+    AudioAdded = "AudioAdded"
+    """Server response to :py:attr:`ClientMessageType.AddAudio`, indicating
+    that audio has been added successfully."""
+
+    AddPartialTranscript = "AddPartialTranscript"
+    """Indicates a partial transcript, which is an incomplete transcript that
+    is immediately produced and may change as more context becomes available.
+    """
+
+    AddTranscript = "AddTranscript"
+    """Indicates the final transcript of a part of the audio."""
+
+    EndOfTranscript = "EndOfTranscript"
+    """Server response to :py:attr:`ClientMessageType.EndOfStream`,
+    after the server has finished sending all :py:attr:`AddTranscript`
+    messages."""
+
+    Info = "Info"
+    """Indicates a generic info message."""
+
+    Warning = "Warning"
+    """Indicates a generic warning message."""
+
+    Error = "Error"
+    """Indicates n generic error message."""

livekit/plugins/speechmatics/utils.py

@@ -0,0 +1,57 @@
+import importlib.metadata
+import os
+
+import aiohttp
+
+
+async def get_access_token(api_key: str) -> str:
+    mp_api_url = os.getenv(
+        "SPEECHMATICS_MANAGEMENT_PLATFORM_URL", "https://mp.speechmatics.com"
+    )
+    endpoint = f"{mp_api_url}/v1/api_keys"
+    params = {"type": "rt", "sm-sdk": get_sdk_version()}
+    json_body = {"ttl": 60}
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            endpoint, params=params, json=json_body, headers=headers
+        ) as resp:
+            if resp.status == 201:
+                try:
+                    data = await resp.json()
+                    return data["key_value"]
+                except (ValueError, KeyError) as e:
+                    raise Exception(
+                        f"Failed to parse Speechmatics access token response: {e}"
+                    )
+            else:
+                error_message = await resp.text()
+                raise Exception(
+                    f"Failed to get Speechmatics access token. "
+                    f"Status: {resp.status}, Error: {error_message}"
+                )
+
+
+def get_sdk_version():
+    version = importlib.metadata.version("livekit-plugins-speechmatics")
+    return f"livekit-plugins-{version}"
+
+
+def sanitize_url(url, language):
+    from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+    parsed_url = urlparse(url)
+
+    query_params = dict(parse_qsl(parsed_url.query))
+    query_params["sm-sdk"] = get_sdk_version()
+    updated_query = urlencode(query_params)
+
+    url_path = parsed_url.path
+    if not url_path.endswith(language):
+        if url_path.endswith("/"):
+            url_path += language
+        else:
+            url_path += f"/{language}"
+
+    return urlunparse(parsed_url._replace(path=url_path, query=updated_query))

livekit/plugins/speechmatics/version.py

@@ -0,0 +1,15 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.0.2"

livekit_plugins_speechmatics-0.0.2.dist-info/METADATA

@@ -0,0 +1,66 @@
+Metadata-Version: 2.2
+Name: livekit-plugins-speechmatics
+Version: 0.0.2
+Summary: Agent Framework plugin for Speechmatics
+Home-page: https://github.com/livekit/agents
+License: Apache-2.0
+Project-URL: Documentation, https://docs.livekit.io
+Project-URL: Website, https://livekit.io/
+Project-URL: Source, https://github.com/livekit/agents
+Keywords: webrtc,realtime,audio,video,livekit
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.9.0
+Description-Content-Type: text/markdown
+Requires-Dist: livekit-agents<1.0.0,>=0.12.16
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# LiveKit Plugins Speechmatics
+
+Agent Framework plugin for Speechmatics.
+
+## Installation
+
+```bash
+pip install livekit-plugins-speechmatics
+```
+
+Usage:
+
+```python
+agent = VoicePipelineAgent(
+    stt=speechmatics.STT(),
+    turn_detector=turn_detector.EOUModel(),
+    min_endpointing_delay=0.5,
+    max_endpointing_delay=5.0,
+    ...
+)
+```
+
+Note: The plugin was built with
+LiveKit's [end-of-turn detection feature](https://github.com/livekit/agents#in-house-phrase-endpointing-model) in mind,
+and it doesn't implement phrase endpointing. `AddTranscript` and `AddPartialTranscript` events are emitted as soon
+as they’re received from the Speechmatics STT engine. For the best user experience,
+we recommend running the agent with end-of-turn detection enabled (
+see [example](https://github.com/livekit-examples/voice-pipeline-agent-python/blob/main/agent.py)).
+
+## Pre-requisites
+
+You'll need to specify a Speechmatics API Key. It can be set as environment variable `SPEECHMATICS_API_KEY` or
+`.env.local` file.

livekit_plugins_speechmatics-0.0.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+livekit/plugins/speechmatics/__init__.py,sha256=u1BkFot3ggFvlbdohgrWSvPgcfqO79dsRLN6lJsyxnU,919
+livekit/plugins/speechmatics/log.py,sha256=O1iyAF7cHUu_iMXh6l7KRwwWeDB5QyABI_qzAb0cs04,75
+livekit/plugins/speechmatics/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/speechmatics/stt.py,sha256=R79Bh0rC04mnyYqfEr8nDv-A-LboH3Ol-c2f9wbqS-Q,11342
+livekit/plugins/speechmatics/types.py,sha256=pqcFr9jzyuoENUGsF_Bagh7mzYSMMB2Q69xYnD5pLYo,4637
+livekit/plugins/speechmatics/utils.py,sha256=0w8CV1rtQnfR5hTzeKhIpFBO5jluv-zq89UT3t8dgr8,1893
+livekit/plugins/speechmatics/version.py,sha256=eDKVGIi8wd5-S0J4-EIahcMFFXzqOGXghEfr_1sXpAc,600
+livekit_plugins_speechmatics-0.0.2.dist-info/METADATA,sha256=hoF3L17vOrULmaRz_FzMKN916G9G-u5UCoUsrCSxbBY,2190
+livekit_plugins_speechmatics-0.0.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+livekit_plugins_speechmatics-0.0.2.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
+livekit_plugins_speechmatics-0.0.2.dist-info/RECORD,,

livekit_plugins_speechmatics-0.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

livekit_plugins_speechmatics-0.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+livekit