livekit-plugins-soniox 1.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of livekit-plugins-soniox might be problematic. Click here for more details.
- livekit_plugins_soniox-1.2.7/.gitignore +175 -0
- livekit_plugins_soniox-1.2.7/PKG-INFO +64 -0
- livekit_plugins_soniox-1.2.7/README.md +42 -0
- livekit_plugins_soniox-1.2.7/livekit/plugins/soniox/__init__.py +45 -0
- livekit_plugins_soniox-1.2.7/livekit/plugins/soniox/log.py +3 -0
- livekit_plugins_soniox-1.2.7/livekit/plugins/soniox/py.typed +0 -0
- livekit_plugins_soniox-1.2.7/livekit/plugins/soniox/stt.py +428 -0
- livekit_plugins_soniox-1.2.7/livekit/plugins/soniox/version.py +15 -0
- livekit_plugins_soniox-1.2.7/pyproject.toml +45 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
**/.vscode
|
|
2
|
+
**/.DS_Store
|
|
3
|
+
|
|
4
|
+
# Byte-compiled / optimized / DLL files
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.py[cod]
|
|
7
|
+
*$py.class
|
|
8
|
+
|
|
9
|
+
# C extensions
|
|
10
|
+
*.so
|
|
11
|
+
|
|
12
|
+
# Distribution / packaging
|
|
13
|
+
.Python
|
|
14
|
+
build/
|
|
15
|
+
develop-eggs/
|
|
16
|
+
dist/
|
|
17
|
+
downloads/
|
|
18
|
+
eggs/
|
|
19
|
+
.eggs/
|
|
20
|
+
lib/
|
|
21
|
+
lib64/
|
|
22
|
+
parts/
|
|
23
|
+
sdist/
|
|
24
|
+
var/
|
|
25
|
+
wheels/
|
|
26
|
+
share/python-wheels/
|
|
27
|
+
*.egg-info/
|
|
28
|
+
.installed.cfg
|
|
29
|
+
*.egg
|
|
30
|
+
MANIFEST
|
|
31
|
+
|
|
32
|
+
# PyInstaller
|
|
33
|
+
# Usually these files are written by a python script from a template
|
|
34
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
35
|
+
*.manifest
|
|
36
|
+
*.spec
|
|
37
|
+
|
|
38
|
+
# Installer logs
|
|
39
|
+
pip-log.txt
|
|
40
|
+
pip-delete-this-directory.txt
|
|
41
|
+
|
|
42
|
+
# Unit test / coverage reports
|
|
43
|
+
htmlcov/
|
|
44
|
+
.tox/
|
|
45
|
+
.nox/
|
|
46
|
+
.coverage
|
|
47
|
+
.coverage.*
|
|
48
|
+
.cache
|
|
49
|
+
nosetests.xml
|
|
50
|
+
coverage.xml
|
|
51
|
+
*.cover
|
|
52
|
+
*.py,cover
|
|
53
|
+
.hypothesis/
|
|
54
|
+
.pytest_cache/
|
|
55
|
+
cover/
|
|
56
|
+
|
|
57
|
+
# Translations
|
|
58
|
+
*.mo
|
|
59
|
+
*.pot
|
|
60
|
+
|
|
61
|
+
# Django stuff:
|
|
62
|
+
*.log
|
|
63
|
+
local_settings.py
|
|
64
|
+
db.sqlite3
|
|
65
|
+
db.sqlite3-journal
|
|
66
|
+
|
|
67
|
+
# Flask stuff:
|
|
68
|
+
instance/
|
|
69
|
+
.webassets-cache
|
|
70
|
+
|
|
71
|
+
# Scrapy stuff:
|
|
72
|
+
.scrapy
|
|
73
|
+
|
|
74
|
+
# Sphinx documentation
|
|
75
|
+
docs/_build/
|
|
76
|
+
|
|
77
|
+
# PyBuilder
|
|
78
|
+
.pybuilder/
|
|
79
|
+
target/
|
|
80
|
+
|
|
81
|
+
# Jupyter Notebook
|
|
82
|
+
.ipynb_checkpoints
|
|
83
|
+
|
|
84
|
+
# IPython
|
|
85
|
+
profile_default/
|
|
86
|
+
ipython_config.py
|
|
87
|
+
|
|
88
|
+
# pyenv
|
|
89
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
90
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
91
|
+
# .python-version
|
|
92
|
+
|
|
93
|
+
# pipenv
|
|
94
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
95
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
96
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
97
|
+
# install all needed dependencies.
|
|
98
|
+
#Pipfile.lock
|
|
99
|
+
|
|
100
|
+
# poetry
|
|
101
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
102
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
103
|
+
# commonly ignored for libraries.
|
|
104
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
105
|
+
#poetry.lock
|
|
106
|
+
|
|
107
|
+
# pdm
|
|
108
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
109
|
+
#pdm.lock
|
|
110
|
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
|
111
|
+
# in version control.
|
|
112
|
+
# https://pdm.fming.dev/#use-with-ide
|
|
113
|
+
.pdm.toml
|
|
114
|
+
|
|
115
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
116
|
+
__pypackages__/
|
|
117
|
+
|
|
118
|
+
# Celery stuff
|
|
119
|
+
celerybeat-schedule
|
|
120
|
+
celerybeat.pid
|
|
121
|
+
|
|
122
|
+
# SageMath parsed files
|
|
123
|
+
*.sage.py
|
|
124
|
+
|
|
125
|
+
# Environments
|
|
126
|
+
.env
|
|
127
|
+
.venv
|
|
128
|
+
env/
|
|
129
|
+
venv/
|
|
130
|
+
ENV/
|
|
131
|
+
env.bak/
|
|
132
|
+
venv.bak/
|
|
133
|
+
|
|
134
|
+
# Spyder project settings
|
|
135
|
+
.spyderproject
|
|
136
|
+
.spyproject
|
|
137
|
+
|
|
138
|
+
# Rope project settings
|
|
139
|
+
.ropeproject
|
|
140
|
+
|
|
141
|
+
# mkdocs documentation
|
|
142
|
+
/site
|
|
143
|
+
|
|
144
|
+
# mypy
|
|
145
|
+
.mypy_cache/
|
|
146
|
+
.dmypy.json
|
|
147
|
+
dmypy.json
|
|
148
|
+
|
|
149
|
+
# trunk
|
|
150
|
+
.trunk/
|
|
151
|
+
|
|
152
|
+
# Pyre type checker
|
|
153
|
+
.pyre/
|
|
154
|
+
|
|
155
|
+
# pytype static type analyzer
|
|
156
|
+
.pytype/
|
|
157
|
+
|
|
158
|
+
# Cython debug symbols
|
|
159
|
+
cython_debug/
|
|
160
|
+
|
|
161
|
+
# PyCharm
|
|
162
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
163
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
164
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
165
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
166
|
+
.idea/
|
|
167
|
+
|
|
168
|
+
node_modules
|
|
169
|
+
|
|
170
|
+
credentials.json
|
|
171
|
+
pyrightconfig.json
|
|
172
|
+
docs/
|
|
173
|
+
|
|
174
|
+
# Database files
|
|
175
|
+
*.db
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: livekit-plugins-soniox
|
|
3
|
+
Version: 1.2.7
|
|
4
|
+
Summary: Agent Framework plugin for services using Soniox's API.
|
|
5
|
+
Project-URL: Documentation, https://docs.livekit.io
|
|
6
|
+
Project-URL: Website, https://livekit.io/
|
|
7
|
+
Project-URL: Source, https://github.com/livekit/agents
|
|
8
|
+
Author-email: Soniox <support@soniox.com>
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
Keywords: audio,livekit,realtime,soniox,speech-to-text,webrtc
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.9.0
|
|
20
|
+
Requires-Dist: livekit-agents>=1.2.7
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# Soniox plugin for LiveKit Agents
|
|
24
|
+
|
|
25
|
+
Support for Soniox Speech-to-Text [Soniox](https://soniox.com/) API, using WebSocket streaming interface.
|
|
26
|
+
|
|
27
|
+
See https://docs.livekit.io/agents/integrations/stt/soniox/ for more information.
|
|
28
|
+
|
|
29
|
+
## Installation
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install livekit-plugins-soniox
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Pre-requisites
|
|
36
|
+
|
|
37
|
+
The Soniox plugin requires an API key to authenticate. You can get your Soniox API key [here](https://console.soniox.com/).
|
|
38
|
+
|
|
39
|
+
Set API key in your `.env` file:
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
SONIOX_API_KEY=<your_soniox_api_key>
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Usage
|
|
46
|
+
|
|
47
|
+
Use Soniox in an `AgentSession` or as a standalone transcription service:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from livekit.plugins import soniox
|
|
51
|
+
|
|
52
|
+
session = AgentSession(
|
|
53
|
+
stt = soniox.STT(),
|
|
54
|
+
# ... llm, tts, etc.
|
|
55
|
+
)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Congratulations! You are now ready to use Soniox Speech-to-Text API in your LiveKit agents.
|
|
59
|
+
|
|
60
|
+
You can test Soniox Speech-to-Text API in the LiveKit's [Voice AI quickstart](https://docs.livekit.io/agents/start/voice-ai/).
|
|
61
|
+
|
|
62
|
+
## More information and reference
|
|
63
|
+
|
|
64
|
+
Explore integration details and find comprehensive examples in our [Soniox LiveKit integration guide](https://speechdev.soniox.com/docs/speech-to-text/integrations/livekit).
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Soniox plugin for LiveKit Agents
|
|
2
|
+
|
|
3
|
+
Support for Soniox Speech-to-Text [Soniox](https://soniox.com/) API, using WebSocket streaming interface.
|
|
4
|
+
|
|
5
|
+
See https://docs.livekit.io/agents/integrations/stt/soniox/ for more information.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install livekit-plugins-soniox
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Pre-requisites
|
|
14
|
+
|
|
15
|
+
The Soniox plugin requires an API key to authenticate. You can get your Soniox API key [here](https://console.soniox.com/).
|
|
16
|
+
|
|
17
|
+
Set API key in your `.env` file:
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
SONIOX_API_KEY=<your_soniox_api_key>
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Usage
|
|
24
|
+
|
|
25
|
+
Use Soniox in an `AgentSession` or as a standalone transcription service:
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from livekit.plugins import soniox
|
|
29
|
+
|
|
30
|
+
session = AgentSession(
|
|
31
|
+
stt = soniox.STT(),
|
|
32
|
+
# ... llm, tts, etc.
|
|
33
|
+
)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Congratulations! You are now ready to use Soniox Speech-to-Text API in your LiveKit agents.
|
|
37
|
+
|
|
38
|
+
You can test Soniox Speech-to-Text API in the LiveKit's [Voice AI quickstart](https://docs.livekit.io/agents/start/voice-ai/).
|
|
39
|
+
|
|
40
|
+
## More information and reference
|
|
41
|
+
|
|
42
|
+
Explore integration details and find comprehensive examples in our [Soniox LiveKit integration guide](https://speechdev.soniox.com/docs/speech-to-text/integrations/livekit).
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright 2025 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Soniox plugin for LiveKit Agents
|
|
16
|
+
|
|
17
|
+
See https://docs.livekit.io/agents/integrations/stt/soniox/ for more information.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from .stt import STT, STTOptions
|
|
21
|
+
from .version import __version__
|
|
22
|
+
|
|
23
|
+
__all__ = ["STT", "STTOptions", "__version__"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
from livekit.agents import Plugin
|
|
27
|
+
|
|
28
|
+
from .log import logger
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SonioxPlugin(Plugin):
|
|
32
|
+
def __init__(self):
|
|
33
|
+
super().__init__(__name__, __version__, __package__, logger)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
Plugin.register_plugin(SonioxPlugin())
|
|
37
|
+
|
|
38
|
+
# Cleanup docs of unexported modules
|
|
39
|
+
_module = dir()
|
|
40
|
+
NOT_IN_ALL = [m for m in _module if m not in __all__]
|
|
41
|
+
|
|
42
|
+
__pdoc__ = {}
|
|
43
|
+
|
|
44
|
+
for n in NOT_IN_ALL:
|
|
45
|
+
__pdoc__[n] = False
|
|
File without changes
|
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
# Copyright 2025 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import asyncio
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import time
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
|
|
23
|
+
import aiohttp
|
|
24
|
+
|
|
25
|
+
from livekit import rtc
|
|
26
|
+
from livekit.agents import (
|
|
27
|
+
APIConnectionError,
|
|
28
|
+
APIConnectOptions,
|
|
29
|
+
APIStatusError,
|
|
30
|
+
APITimeoutError,
|
|
31
|
+
stt,
|
|
32
|
+
utils,
|
|
33
|
+
vad,
|
|
34
|
+
)
|
|
35
|
+
from livekit.agents.stt import SpeechEventType
|
|
36
|
+
from livekit.agents.types import (
|
|
37
|
+
DEFAULT_API_CONNECT_OPTIONS,
|
|
38
|
+
NOT_GIVEN,
|
|
39
|
+
NotGivenOr,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
from .log import logger
|
|
43
|
+
|
|
44
|
+
# Base URL for Soniox Speech-to-Text API.
|
|
45
|
+
BASE_URL = "wss://stt-rt.soniox.com/transcribe-websocket"
|
|
46
|
+
|
|
47
|
+
# WebSocket messages and tokens.
|
|
48
|
+
KEEPALIVE_MESSAGE = '{"type": "keepalive"}'
|
|
49
|
+
FINALIZE_MESSAGE = '{"type": "finalize"}'
|
|
50
|
+
END_TOKEN = "<end>"
|
|
51
|
+
FINALIZED_TOKEN = "<fin>"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def is_end_token(token: dict) -> bool:
|
|
55
|
+
"""Return True if the given token marks an end or finalized event."""
|
|
56
|
+
return token.get("text") in (END_TOKEN, FINALIZED_TOKEN)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class STTOptions:
|
|
61
|
+
"""Configuration options for Soniox Speech-to-Text service."""
|
|
62
|
+
|
|
63
|
+
model: str | None = "stt-rt-preview"
|
|
64
|
+
language_hints: list[str] | None = None
|
|
65
|
+
context: str | None = None
|
|
66
|
+
|
|
67
|
+
num_channels: int = 1
|
|
68
|
+
sample_rate: int = 16000
|
|
69
|
+
|
|
70
|
+
enable_language_identification: bool = True
|
|
71
|
+
|
|
72
|
+
enable_non_final_tokens: bool = True
|
|
73
|
+
max_non_final_tokens_duration_ms: int | None = None
|
|
74
|
+
|
|
75
|
+
client_reference_id: str | None = None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class STT(stt.STT):
|
|
79
|
+
"""Speech-to-Text service using Soniox Speech-to-Text API.
|
|
80
|
+
|
|
81
|
+
This service connects to Soniox Speech-to-Text API for real-time transcription
|
|
82
|
+
with support for multiple languages, custom context, speaker diarization,
|
|
83
|
+
and more.
|
|
84
|
+
|
|
85
|
+
For complete API documentation, see: https://soniox.com/docs/speech-to-text/api-reference/websocket-api
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def __init__(
|
|
89
|
+
self,
|
|
90
|
+
*,
|
|
91
|
+
api_key: str | None = None,
|
|
92
|
+
base_url: str = BASE_URL,
|
|
93
|
+
http_session: aiohttp.ClientSession | None = None,
|
|
94
|
+
vad: vad.VAD | None = None,
|
|
95
|
+
params: STTOptions | None = None,
|
|
96
|
+
):
|
|
97
|
+
"""Initialize instance of Soniox Speech-to-Text API service.
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
api_key: Soniox API key, if not provided, will look for SONIOX_API_KEY env variable.
|
|
101
|
+
base_url: Base URL for Soniox Speech-to-Text API, default to BASE_URL defined in this
|
|
102
|
+
module.
|
|
103
|
+
http_session: Optional aiohttp.ClientSession to use for requests.
|
|
104
|
+
vad: If passed, enable Voice Activity Detection (VAD) for audio frames.
|
|
105
|
+
params: Additional configuration parameters, such as model, language hints, context and
|
|
106
|
+
speaker diarization.
|
|
107
|
+
"""
|
|
108
|
+
super().__init__(capabilities=stt.STTCapabilities(streaming=True, interim_results=True))
|
|
109
|
+
|
|
110
|
+
self._api_key = api_key or os.getenv("SONIOX_API_KEY")
|
|
111
|
+
self._base_url = base_url
|
|
112
|
+
self._http_session = http_session
|
|
113
|
+
self._vad_stream = vad.stream() if vad else None
|
|
114
|
+
self._params = params or STTOptions()
|
|
115
|
+
|
|
116
|
+
async def _recognize_impl(
|
|
117
|
+
self,
|
|
118
|
+
buffer: utils.AudioBuffer,
|
|
119
|
+
*,
|
|
120
|
+
language: NotGivenOr[str] = NOT_GIVEN,
|
|
121
|
+
conn_options: APIConnectOptions,
|
|
122
|
+
) -> stt.SpeechEvent:
|
|
123
|
+
"""Raise error since single-frame recognition is not supported
|
|
124
|
+
by Soniox Speech-to-Text API."""
|
|
125
|
+
raise NotImplementedError(
|
|
126
|
+
"Soniox Speech-to-Text API does not support single frame recognition"
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
def stream(
|
|
130
|
+
self,
|
|
131
|
+
*,
|
|
132
|
+
language: NotGivenOr[str] = NOT_GIVEN,
|
|
133
|
+
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
|
|
134
|
+
) -> SpeechStream:
|
|
135
|
+
"""Return a new LiveKit streaming speech-to-text session."""
|
|
136
|
+
return SpeechStream(
|
|
137
|
+
stt=self,
|
|
138
|
+
conn_options=conn_options,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class SpeechStream(stt.SpeechStream):
|
|
143
|
+
def __init__(
|
|
144
|
+
self,
|
|
145
|
+
stt: STT,
|
|
146
|
+
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
|
|
147
|
+
) -> None:
|
|
148
|
+
"""Set up state and queues for a WebSocket-based transcription stream."""
|
|
149
|
+
super().__init__(stt=stt, conn_options=conn_options, sample_rate=stt._params.sample_rate)
|
|
150
|
+
self._stt = stt
|
|
151
|
+
self._ws: aiohttp.ClientWebSocketResponse | None = None
|
|
152
|
+
self._reconnect_event = asyncio.Event()
|
|
153
|
+
|
|
154
|
+
self.audio_queue = asyncio.Queue()
|
|
155
|
+
|
|
156
|
+
self._last_tokens_received: float | None = None
|
|
157
|
+
|
|
158
|
+
def _ensure_session(self) -> aiohttp.ClientSession:
|
|
159
|
+
"""Get or create an aiohttp ClientSession for WebSocket connections."""
|
|
160
|
+
if not self._stt._http_session:
|
|
161
|
+
self._stt._http_session = utils.http_context.http_session()
|
|
162
|
+
|
|
163
|
+
return self._stt._http_session
|
|
164
|
+
|
|
165
|
+
async def _connect_ws(self):
|
|
166
|
+
"""Open a WebSocket connection to the Soniox Speech-to-Text API and send the
|
|
167
|
+
initial configuration."""
|
|
168
|
+
# If VAD was passed, disable endpoint detection, otherwise enable it.
|
|
169
|
+
enable_endpoint_detection = not self._stt._vad_stream
|
|
170
|
+
|
|
171
|
+
# Create initial config object.
|
|
172
|
+
config = {
|
|
173
|
+
"api_key": self._stt._api_key,
|
|
174
|
+
"model": self._stt._params.model,
|
|
175
|
+
"audio_format": "pcm_s16le",
|
|
176
|
+
"num_channels": self._stt._params.num_channels or 1,
|
|
177
|
+
"enable_endpoint_detection": enable_endpoint_detection,
|
|
178
|
+
"sample_rate": self._stt._params.sample_rate,
|
|
179
|
+
"language_hints": self._stt._params.language_hints,
|
|
180
|
+
"context": self._stt._params.context,
|
|
181
|
+
"enable_non_final_tokens": self._stt._params.enable_non_final_tokens,
|
|
182
|
+
"max_non_final_tokens_duration_ms": self._stt._params.max_non_final_tokens_duration_ms,
|
|
183
|
+
"enable_language_identification": self._stt._params.enable_language_identification,
|
|
184
|
+
"client_reference_id": self._stt._params.client_reference_id,
|
|
185
|
+
}
|
|
186
|
+
# Connect to the Soniox Speech-to-Text API.
|
|
187
|
+
ws = await asyncio.wait_for(
|
|
188
|
+
self._ensure_session().ws_connect(self._stt._base_url),
|
|
189
|
+
timeout=self._conn_options.timeout,
|
|
190
|
+
)
|
|
191
|
+
# Set initial configuration message.
|
|
192
|
+
await ws.send_str(json.dumps(config))
|
|
193
|
+
logger.debug("Soniox Speech-to-Text API connection established!")
|
|
194
|
+
return ws
|
|
195
|
+
|
|
196
|
+
async def _run(self) -> None:
|
|
197
|
+
"""Manage connection lifecycle, spawning tasks and handling reconnection."""
|
|
198
|
+
while True:
|
|
199
|
+
try:
|
|
200
|
+
ws = await self._connect_ws()
|
|
201
|
+
self._ws = ws
|
|
202
|
+
# Create task for audio processing, voice turn detection and message handling.
|
|
203
|
+
tasks = [
|
|
204
|
+
asyncio.create_task(self._prepare_audio_task()),
|
|
205
|
+
asyncio.create_task(self._handle_vad_task()),
|
|
206
|
+
asyncio.create_task(self._send_audio_task()),
|
|
207
|
+
asyncio.create_task(self._recv_messages_task()),
|
|
208
|
+
asyncio.create_task(self._keepalive_task()),
|
|
209
|
+
]
|
|
210
|
+
wait_reconnect_task = asyncio.create_task(self._reconnect_event.wait())
|
|
211
|
+
try:
|
|
212
|
+
done, _ = await asyncio.wait(
|
|
213
|
+
[asyncio.gather(*tasks), wait_reconnect_task],
|
|
214
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
for task in done:
|
|
218
|
+
if task != wait_reconnect_task:
|
|
219
|
+
task.result()
|
|
220
|
+
|
|
221
|
+
if wait_reconnect_task not in done:
|
|
222
|
+
break
|
|
223
|
+
|
|
224
|
+
self._reconnect_event.clear()
|
|
225
|
+
finally:
|
|
226
|
+
await utils.aio.gracefully_cancel(*tasks, wait_reconnect_task)
|
|
227
|
+
# Handle errors.
|
|
228
|
+
except asyncio.TimeoutError as e:
|
|
229
|
+
logger.error(
|
|
230
|
+
f"Timeout during Soniox Speech-to-Text API connection/initialization: {e}"
|
|
231
|
+
)
|
|
232
|
+
raise APITimeoutError(
|
|
233
|
+
"Timeout connecting to or initializing Soniox Speech-to-Text API session"
|
|
234
|
+
) from e
|
|
235
|
+
|
|
236
|
+
except aiohttp.ClientResponseError as e:
|
|
237
|
+
logger.error(
|
|
238
|
+
"Soniox Speech-to-Text API status error during session init:"
|
|
239
|
+
+ f"{e.status} {e.message}"
|
|
240
|
+
)
|
|
241
|
+
raise APIStatusError(
|
|
242
|
+
message=e.message, status_code=e.status, request_id=None, body=None
|
|
243
|
+
) from e
|
|
244
|
+
|
|
245
|
+
except aiohttp.ClientError as e:
|
|
246
|
+
logger.error(f"Soniox Speech-to-Text API connection error: {e}")
|
|
247
|
+
raise APIConnectionError(f"Soniox Speech-to-Text API connection error: {e}") from e
|
|
248
|
+
|
|
249
|
+
except Exception as e:
|
|
250
|
+
logger.exception(f"Unexpected error occurred: {e}")
|
|
251
|
+
raise APIConnectionError(f"An unexpected error occurred: {e}") from e
|
|
252
|
+
# Close the WebSocket connection on finish.
|
|
253
|
+
finally:
|
|
254
|
+
if self._ws is not None:
|
|
255
|
+
await self._ws.close()
|
|
256
|
+
self._ws = None
|
|
257
|
+
|
|
258
|
+
async def _keepalive_task(self):
|
|
259
|
+
"""Periodically send keepalive messages (while no audio is being sent)
|
|
260
|
+
to maintain the WebSocket connection."""
|
|
261
|
+
try:
|
|
262
|
+
while self._ws:
|
|
263
|
+
await self._ws.send_str(KEEPALIVE_MESSAGE)
|
|
264
|
+
await asyncio.sleep(5)
|
|
265
|
+
except Exception as e:
|
|
266
|
+
logger.error(f"Error while sending keep alive message: {e}")
|
|
267
|
+
|
|
268
|
+
async def _prepare_audio_task(self):
|
|
269
|
+
"""Read audio frames, process VAD, and enqueue PCM data for sending."""
|
|
270
|
+
if not self._ws:
|
|
271
|
+
logger.error("WebSocket connection to Soniox Speech-to-Text API is not established")
|
|
272
|
+
return
|
|
273
|
+
|
|
274
|
+
async for data in self._input_ch:
|
|
275
|
+
if self._stt._vad_stream:
|
|
276
|
+
# If VAD is enabled, push the audio frame to the VAD stream.
|
|
277
|
+
if isinstance(data, self._FlushSentinel):
|
|
278
|
+
self._stt._vad_stream.flush()
|
|
279
|
+
else:
|
|
280
|
+
self._stt._vad_stream.push_frame(data)
|
|
281
|
+
|
|
282
|
+
if isinstance(data, rtc.AudioFrame):
|
|
283
|
+
# Get the raw bytes from the audio frame.
|
|
284
|
+
pcm_data = data.data.tobytes()
|
|
285
|
+
self.audio_queue.put_nowait(pcm_data)
|
|
286
|
+
|
|
287
|
+
async def _send_audio_task(self):
|
|
288
|
+
"""Take queued audio data and transmit it over the WebSocket."""
|
|
289
|
+
if not self._ws:
|
|
290
|
+
logger.error("WebSocket connection to Soniox Speech-to-Text API is not established")
|
|
291
|
+
return
|
|
292
|
+
|
|
293
|
+
while self._ws:
|
|
294
|
+
try:
|
|
295
|
+
data = await self.audio_queue.get()
|
|
296
|
+
|
|
297
|
+
if isinstance(data, bytes):
|
|
298
|
+
await self._ws.send_bytes(data)
|
|
299
|
+
else:
|
|
300
|
+
await self._ws.send_str(data)
|
|
301
|
+
except asyncio.CancelledError:
|
|
302
|
+
break
|
|
303
|
+
except Exception as e:
|
|
304
|
+
logger.error(f"Error while sending audio data: {e}")
|
|
305
|
+
break
|
|
306
|
+
|
|
307
|
+
async def _handle_vad_task(self):
|
|
308
|
+
"""Listen for VAD events to trigger finalize or keepalive messages."""
|
|
309
|
+
if not self._stt._vad_stream:
|
|
310
|
+
logger.debug("VAD stream is not enabled, skipping VAD task")
|
|
311
|
+
return
|
|
312
|
+
|
|
313
|
+
async for event in self._stt._vad_stream:
|
|
314
|
+
if event.type == vad.VADEventType.END_OF_SPEECH:
|
|
315
|
+
self.audio_queue.put_nowait(FINALIZE_MESSAGE)
|
|
316
|
+
|
|
317
|
+
async def _recv_messages_task(self):
|
|
318
|
+
"""Receive transcription messages, handle tokens, errors, and dispatch events."""
|
|
319
|
+
|
|
320
|
+
# Transcription frame will be only sent after we get the "endpoint" event.
|
|
321
|
+
final_transcript_buffer = ""
|
|
322
|
+
# Language code sent by Soniox if language detection is enabled (e.g. "en", "de", "fr")
|
|
323
|
+
final_transcript_language: str = ""
|
|
324
|
+
|
|
325
|
+
def send_endpoint_transcript():
|
|
326
|
+
nonlocal final_transcript_buffer, final_transcript_language
|
|
327
|
+
if final_transcript_buffer:
|
|
328
|
+
event = stt.SpeechEvent(
|
|
329
|
+
type=SpeechEventType.FINAL_TRANSCRIPT,
|
|
330
|
+
alternatives=[
|
|
331
|
+
stt.SpeechData(
|
|
332
|
+
text=final_transcript_buffer, language=final_transcript_language
|
|
333
|
+
)
|
|
334
|
+
],
|
|
335
|
+
)
|
|
336
|
+
self._event_ch.send_nowait(event)
|
|
337
|
+
final_transcript_buffer = ""
|
|
338
|
+
final_transcript_language = ""
|
|
339
|
+
|
|
340
|
+
# Method handles receiving messages from the Soniox Speech-to-Text API.
|
|
341
|
+
while self._ws:
|
|
342
|
+
try:
|
|
343
|
+
async for msg in self._ws:
|
|
344
|
+
if msg.type == aiohttp.WSMsgType.TEXT:
|
|
345
|
+
try:
|
|
346
|
+
content = json.loads(msg.data)
|
|
347
|
+
tokens = content["tokens"]
|
|
348
|
+
|
|
349
|
+
if tokens:
|
|
350
|
+
if len(tokens) == 1 and tokens[0]["text"] == FINALIZED_TOKEN:
|
|
351
|
+
# Ignore finalized token, prevent auto finalize cycle.
|
|
352
|
+
pass
|
|
353
|
+
else:
|
|
354
|
+
# Got at least one token, reset the auto finalize delay.
|
|
355
|
+
self._last_tokens_received = time.time()
|
|
356
|
+
|
|
357
|
+
# We will only send the final tokens after we get the "endpoint" event.
|
|
358
|
+
non_final_transcription = ""
|
|
359
|
+
non_final_transcription_language: str = ""
|
|
360
|
+
|
|
361
|
+
for token in tokens:
|
|
362
|
+
if token["is_final"]:
|
|
363
|
+
if is_end_token(token):
|
|
364
|
+
# Found an endpoint, tokens until here will be sent as
|
|
365
|
+
# transcript, the rest will be sent as interim tokens
|
|
366
|
+
# (even final tokens).
|
|
367
|
+
send_endpoint_transcript()
|
|
368
|
+
else:
|
|
369
|
+
final_transcript_buffer += token["text"]
|
|
370
|
+
|
|
371
|
+
# Soniox provides language for each token,
|
|
372
|
+
# LiveKit requires only a single language for the entire transcription chunk.
|
|
373
|
+
# Current heuristic is to take the first language we see.
|
|
374
|
+
if token.get("language") and not final_transcript_language:
|
|
375
|
+
final_transcript_language = token.get("language")
|
|
376
|
+
else:
|
|
377
|
+
non_final_transcription += token["text"]
|
|
378
|
+
if (
|
|
379
|
+
token.get("language")
|
|
380
|
+
and not non_final_transcription_language
|
|
381
|
+
):
|
|
382
|
+
non_final_transcription_language = token.get("language")
|
|
383
|
+
|
|
384
|
+
if final_transcript_buffer or non_final_transcription:
|
|
385
|
+
event = stt.SpeechEvent(
|
|
386
|
+
type=SpeechEventType.INTERIM_TRANSCRIPT,
|
|
387
|
+
alternatives=[
|
|
388
|
+
stt.SpeechData(
|
|
389
|
+
text=final_transcript_buffer + non_final_transcription,
|
|
390
|
+
language=final_transcript_language
|
|
391
|
+
if final_transcript_language
|
|
392
|
+
else non_final_transcription_language,
|
|
393
|
+
)
|
|
394
|
+
],
|
|
395
|
+
)
|
|
396
|
+
self._event_ch.send_nowait(event)
|
|
397
|
+
|
|
398
|
+
error_code = content.get("error_code")
|
|
399
|
+
error_message = content.get("error_message")
|
|
400
|
+
|
|
401
|
+
if error_code or error_message:
|
|
402
|
+
# In case of error, still send the final transcript.
|
|
403
|
+
send_endpoint_transcript()
|
|
404
|
+
logger.error(f"WebSocket error: {error_code} - {error_message}")
|
|
405
|
+
|
|
406
|
+
finished = content.get("finished")
|
|
407
|
+
|
|
408
|
+
if finished:
|
|
409
|
+
# When finished, still send the final transcript.
|
|
410
|
+
send_endpoint_transcript()
|
|
411
|
+
logger.debug("Transcription finished")
|
|
412
|
+
|
|
413
|
+
except Exception as e:
|
|
414
|
+
logger.exception(f"Error processing message: {e}")
|
|
415
|
+
elif msg.type in (
|
|
416
|
+
aiohttp.WSMsgType.CLOSED,
|
|
417
|
+
aiohttp.WSMsgType.CLOSE,
|
|
418
|
+
aiohttp.WSMsgType.CLOSING,
|
|
419
|
+
):
|
|
420
|
+
break
|
|
421
|
+
else:
|
|
422
|
+
logger.warning(
|
|
423
|
+
f"Unexpected message type from Soniox Speech-to-Text API: {msg.type}"
|
|
424
|
+
)
|
|
425
|
+
except aiohttp.ClientError as e:
|
|
426
|
+
logger.error(f"WebSocket error while receiving: {e}")
|
|
427
|
+
except Exception as e:
|
|
428
|
+
logger.error(f"Unexpected error while receiving messages: {e}")
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2025 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
__version__ = "1.2.7"
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "livekit-plugins-soniox"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Agent Framework plugin for services using Soniox's API."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "Apache-2.0"
|
|
11
|
+
requires-python = ">=3.9.0"
|
|
12
|
+
authors = [{ name = "Soniox", email = "support@soniox.com" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"webrtc",
|
|
15
|
+
"realtime",
|
|
16
|
+
"audio",
|
|
17
|
+
"livekit",
|
|
18
|
+
"soniox",
|
|
19
|
+
"speech-to-text",
|
|
20
|
+
]
|
|
21
|
+
classifiers = [
|
|
22
|
+
"Intended Audience :: Developers",
|
|
23
|
+
"License :: OSI Approved :: Apache Software License",
|
|
24
|
+
"Topic :: Multimedia :: Sound/Audio",
|
|
25
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
26
|
+
"Programming Language :: Python :: 3",
|
|
27
|
+
"Programming Language :: Python :: 3.9",
|
|
28
|
+
"Programming Language :: Python :: 3.10",
|
|
29
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
30
|
+
]
|
|
31
|
+
dependencies = ["livekit-agents>=1.2.7"]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Documentation = "https://docs.livekit.io"
|
|
35
|
+
Website = "https://livekit.io/"
|
|
36
|
+
Source = "https://github.com/livekit/agents"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.version]
|
|
39
|
+
path = "livekit/plugins/soniox/version.py"
|
|
40
|
+
|
|
41
|
+
[tool.hatch.build.targets.wheel]
|
|
42
|
+
packages = ["livekit"]
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.targets.sdist]
|
|
45
|
+
include = ["/livekit"]
|