livekit-plugins-volcengine 1.0.0rc4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- livekit_plugins_volcengine-1.0.0rc4/.gitignore +179 -0
- livekit_plugins_volcengine-1.0.0rc4/LICENSE +21 -0
- livekit_plugins_volcengine-1.0.0rc4/PKG-INFO +66 -0
- livekit_plugins_volcengine-1.0.0rc4/README.md +44 -0
- livekit_plugins_volcengine-1.0.0rc4/livekit/plugins/volcengine/__init__.py +25 -0
- livekit_plugins_volcengine-1.0.0rc4/livekit/plugins/volcengine/log.py +4 -0
- livekit_plugins_volcengine-1.0.0rc4/livekit/plugins/volcengine/py.typed +0 -0
- livekit_plugins_volcengine-1.0.0rc4/livekit/plugins/volcengine/tts.py +460 -0
- livekit_plugins_volcengine-1.0.0rc4/livekit/plugins/volcengine/version.py +1 -0
- livekit_plugins_volcengine-1.0.0rc4/pyproject.toml +55 -0
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py,cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
#Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
#uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
#poetry.lock
|
|
109
|
+
|
|
110
|
+
# pdm
|
|
111
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
112
|
+
#pdm.lock
|
|
113
|
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
|
114
|
+
# in version control.
|
|
115
|
+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
|
116
|
+
.pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
121
|
+
__pypackages__/
|
|
122
|
+
|
|
123
|
+
# Celery stuff
|
|
124
|
+
celerybeat-schedule
|
|
125
|
+
celerybeat.pid
|
|
126
|
+
|
|
127
|
+
# SageMath parsed files
|
|
128
|
+
*.sage.py
|
|
129
|
+
|
|
130
|
+
# Environments
|
|
131
|
+
.env
|
|
132
|
+
.venv
|
|
133
|
+
env/
|
|
134
|
+
venv/
|
|
135
|
+
ENV/
|
|
136
|
+
env.bak/
|
|
137
|
+
venv.bak/
|
|
138
|
+
|
|
139
|
+
# Spyder project settings
|
|
140
|
+
.spyderproject
|
|
141
|
+
.spyproject
|
|
142
|
+
|
|
143
|
+
# Rope project settings
|
|
144
|
+
.ropeproject
|
|
145
|
+
|
|
146
|
+
# mkdocs documentation
|
|
147
|
+
/site
|
|
148
|
+
|
|
149
|
+
# mypy
|
|
150
|
+
.mypy_cache/
|
|
151
|
+
.dmypy.json
|
|
152
|
+
dmypy.json
|
|
153
|
+
|
|
154
|
+
# Pyre type checker
|
|
155
|
+
.pyre/
|
|
156
|
+
|
|
157
|
+
# pytype static type analyzer
|
|
158
|
+
.pytype/
|
|
159
|
+
|
|
160
|
+
# Cython debug symbols
|
|
161
|
+
cython_debug/
|
|
162
|
+
|
|
163
|
+
# PyCharm
|
|
164
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
165
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
166
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
167
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
168
|
+
#.idea/
|
|
169
|
+
|
|
170
|
+
# Ruff stuff:
|
|
171
|
+
.ruff_cache/
|
|
172
|
+
|
|
173
|
+
# PyPI configuration file
|
|
174
|
+
.pypirc
|
|
175
|
+
|
|
176
|
+
# tests
|
|
177
|
+
test.py
|
|
178
|
+
|
|
179
|
+
.python-version
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 王梦迪
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: livekit-plugins-volcengine
|
|
3
|
+
Version: 1.0.0rc4
|
|
4
|
+
Summary: LiveKit Agent Plugins for Volcengine
|
|
5
|
+
Author-email: wangmengdi <790990241@qq.com>
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: audio,livekit,realtime,video,webrtc
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
15
|
+
Classifier: Topic :: Multimedia :: Video
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Requires-Dist: livekit-agents~=1.0.0rc4
|
|
19
|
+
Requires-Dist: pydantic>=2.0
|
|
20
|
+
Requires-Dist: python-dotenv>=1.1.0
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# LiveKit Plugins Volcengine
|
|
24
|
+
|
|
25
|
+
Agent Framework plugin for services from Volcengine(火山引擎). Currently supports [TTS](https://www.volcengine.com/docs/6561/79817)
|
|
26
|
+
|
|
27
|
+
## Installation
|
|
28
|
+
```python
|
|
29
|
+
pip install livekit-plugins-volcengine
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Pre-requisites
|
|
33
|
+
|
|
34
|
+
- Volcengine TTS environment variable: `VOLCENGINE_TTS_ACCESS_TOKEN`
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from livekit.agents import Agent, AgentSession, JobContext, cli, WorkerOptions
|
|
40
|
+
from livekit.plugins import openai, volcengine, deepgram, silero
|
|
41
|
+
from dotenv import load_dotenv
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
async def entry_point(ctx: JobContext):
|
|
45
|
+
|
|
46
|
+
await ctx.connect()
|
|
47
|
+
|
|
48
|
+
agent = Agent(instructions="You are a helpful assistant.")
|
|
49
|
+
|
|
50
|
+
session = AgentSession(
|
|
51
|
+
vad=silero.VAD.load(),
|
|
52
|
+
stt=deepgram.STT(language="zh"),
|
|
53
|
+
## app_id and cluster can be found in the Volcengine TTS console
|
|
54
|
+
tts=volcengine.TTS(app_id="xxx", cluster="xxx", streaming=True),
|
|
55
|
+
llm=openai.LLM(model="gpt-4o-mini"),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
await session.start(agent=agent, room=ctx.room)
|
|
59
|
+
|
|
60
|
+
await session.generate_reply()
|
|
61
|
+
|
|
62
|
+
if __name__ == "__main__":
|
|
63
|
+
load_dotenv()
|
|
64
|
+
cli.run_app(WorkerOptions(entrypoint_fnc=entry_point))
|
|
65
|
+
```
|
|
66
|
+
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# LiveKit Plugins Volcengine
|
|
2
|
+
|
|
3
|
+
Agent Framework plugin for services from Volcengine(火山引擎). Currently supports [TTS](https://www.volcengine.com/docs/6561/79817)
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
```python
|
|
7
|
+
pip install livekit-plugins-volcengine
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
## Pre-requisites
|
|
11
|
+
|
|
12
|
+
- Volcengine TTS environment variable: `VOLCENGINE_TTS_ACCESS_TOKEN`
|
|
13
|
+
|
|
14
|
+
## Usage
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from livekit.agents import Agent, AgentSession, JobContext, cli, WorkerOptions
|
|
18
|
+
from livekit.plugins import openai, volcengine, deepgram, silero
|
|
19
|
+
from dotenv import load_dotenv
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def entry_point(ctx: JobContext):
|
|
23
|
+
|
|
24
|
+
await ctx.connect()
|
|
25
|
+
|
|
26
|
+
agent = Agent(instructions="You are a helpful assistant.")
|
|
27
|
+
|
|
28
|
+
session = AgentSession(
|
|
29
|
+
vad=silero.VAD.load(),
|
|
30
|
+
stt=deepgram.STT(language="zh"),
|
|
31
|
+
## app_id and cluster can be found in the Volcengine TTS console
|
|
32
|
+
tts=volcengine.TTS(app_id="xxx", cluster="xxx", streaming=True),
|
|
33
|
+
llm=openai.LLM(model="gpt-4o-mini"),
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
await session.start(agent=agent, room=ctx.room)
|
|
37
|
+
|
|
38
|
+
await session.generate_reply()
|
|
39
|
+
|
|
40
|
+
if __name__ == "__main__":
|
|
41
|
+
load_dotenv()
|
|
42
|
+
cli.run_app(WorkerOptions(entrypoint_fnc=entry_point))
|
|
43
|
+
```
|
|
44
|
+
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from .tts import TTS
|
|
2
|
+
from .version import __version__
|
|
3
|
+
|
|
4
|
+
__all__ = ["TTS", "__version__"]
|
|
5
|
+
|
|
6
|
+
from livekit.agents import Plugin
|
|
7
|
+
|
|
8
|
+
from .log import logger
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class VolcenginePlugin(Plugin):
|
|
12
|
+
def __init__(self):
|
|
13
|
+
super().__init__(__name__, __version__, __package__, logger)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
Plugin.register_plugin(VolcenginePlugin())
|
|
17
|
+
|
|
18
|
+
# Cleanup docs of unexported modules
|
|
19
|
+
_module = dir()
|
|
20
|
+
NOT_IN_ALL = [m for m in _module if m not in __all__]
|
|
21
|
+
|
|
22
|
+
__pdoc__ = {}
|
|
23
|
+
|
|
24
|
+
for n in NOT_IN_ALL:
|
|
25
|
+
__pdoc__[n] = False
|
|
File without changes
|
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from typing import Dict, Literal, ByteString, List, Tuple
|
|
3
|
+
import os
|
|
4
|
+
import base64
|
|
5
|
+
import gzip
|
|
6
|
+
import json
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
from livekit.agents import (
|
|
10
|
+
tts,
|
|
11
|
+
utils,
|
|
12
|
+
APIConnectionError,
|
|
13
|
+
APITimeoutError,
|
|
14
|
+
APIStatusError,
|
|
15
|
+
APIConnectOptions,
|
|
16
|
+
)
|
|
17
|
+
from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS
|
|
18
|
+
from pydantic import BaseModel, Field
|
|
19
|
+
import aiohttp
|
|
20
|
+
import asyncio
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class _TTSOptions(BaseModel):
|
|
24
|
+
app_id: str
|
|
25
|
+
cluster: str
|
|
26
|
+
access_token: str | None = None
|
|
27
|
+
voice_type: str = "BV001_V2_streaming"
|
|
28
|
+
base_url: str = "https://openspeech.bytedance.com/api/v1"
|
|
29
|
+
sample_rate: Literal[24000, 16000, 8000] = 24000
|
|
30
|
+
encoding: Literal["mp3", "pcm"] = "pcm"
|
|
31
|
+
speed: float = Field(1.0, ge=0.2, le=3.0)
|
|
32
|
+
volume: float = Field(1.0, gt=0.1, le=3.0)
|
|
33
|
+
pitch: float = Field(1.0, ge=0.1, le=3.0)
|
|
34
|
+
|
|
35
|
+
def get_http_url(self):
|
|
36
|
+
return f"{self.base_url}/tts"
|
|
37
|
+
|
|
38
|
+
def get_http_header(self):
|
|
39
|
+
if self.access_token is None:
|
|
40
|
+
self.access_token = os.getenv("VOLCENGINE_TTS_ACCESS_TOKEN")
|
|
41
|
+
if self.access_token is None:
|
|
42
|
+
raise ValueError("VOLCENGINE_TTS_ACCESS_TOKEN is not set")
|
|
43
|
+
return {
|
|
44
|
+
"Authorization": f"Bearer;{self.access_token}",
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
def get_http_query_params(self, text: str, uid: str | None = None) -> Dict:
|
|
48
|
+
if uid is None:
|
|
49
|
+
uid = utils.shortuuid()
|
|
50
|
+
request_json = {
|
|
51
|
+
"app": {
|
|
52
|
+
"appid": self.app_id,
|
|
53
|
+
"token": self.access_token,
|
|
54
|
+
"cluster": self.cluster,
|
|
55
|
+
},
|
|
56
|
+
"user": {"uid": uid},
|
|
57
|
+
"audio": {
|
|
58
|
+
"voice_type": self.voice_type,
|
|
59
|
+
"encoding": self.encoding,
|
|
60
|
+
"speed_ratio": self.speed,
|
|
61
|
+
"volume_ratio": self.volume,
|
|
62
|
+
"pitch_ratio": 1.0,
|
|
63
|
+
"rate": self.sample_rate,
|
|
64
|
+
},
|
|
65
|
+
"request": {
|
|
66
|
+
"reqid": utils.shortuuid(),
|
|
67
|
+
"text": text,
|
|
68
|
+
"text_type": "plain",
|
|
69
|
+
"operation": "query",
|
|
70
|
+
"with_frontend": self.pitch,
|
|
71
|
+
"frontend_type": "unitTson",
|
|
72
|
+
},
|
|
73
|
+
}
|
|
74
|
+
return request_json
|
|
75
|
+
|
|
76
|
+
def get_ws_url(self):
|
|
77
|
+
return f"{self.base_url}/tts/ws_binary"
|
|
78
|
+
|
|
79
|
+
def get_ws_query_params(self, text: str, uid: str | None = None) -> bytearray:
|
|
80
|
+
if uid is None:
|
|
81
|
+
uid = utils.shortuuid()
|
|
82
|
+
submit_request_json = {
|
|
83
|
+
"app": {
|
|
84
|
+
"appid": self.app_id,
|
|
85
|
+
"token": self.access_token,
|
|
86
|
+
"cluster": self.cluster,
|
|
87
|
+
},
|
|
88
|
+
"user": {"uid": uid},
|
|
89
|
+
"audio": {
|
|
90
|
+
"voice_type": self.voice_type,
|
|
91
|
+
"encoding": self.encoding,
|
|
92
|
+
"speed_ratio": self.speed,
|
|
93
|
+
"volume_ratio": self.volume,
|
|
94
|
+
"pitch_ratio": self.pitch,
|
|
95
|
+
"rate": self.sample_rate,
|
|
96
|
+
},
|
|
97
|
+
"request": {
|
|
98
|
+
"reqid": utils.shortuuid(),
|
|
99
|
+
"text": text,
|
|
100
|
+
"text_type": "plain",
|
|
101
|
+
"operation": "submit",
|
|
102
|
+
"with_frontend": 1,
|
|
103
|
+
"frontend_type": "unitTson",
|
|
104
|
+
},
|
|
105
|
+
}
|
|
106
|
+
default_header = bytearray(b"\x11\x10\x11\x00")
|
|
107
|
+
payload_bytes = str.encode(json.dumps(submit_request_json))
|
|
108
|
+
payload_bytes = gzip.compress(
|
|
109
|
+
payload_bytes
|
|
110
|
+
) # if no compression, comment this line
|
|
111
|
+
full_client_request = bytearray(default_header)
|
|
112
|
+
full_client_request.extend(
|
|
113
|
+
(len(payload_bytes)).to_bytes(4, "big")
|
|
114
|
+
) # payload size(4 bytes)
|
|
115
|
+
full_client_request.extend(payload_bytes) # payload
|
|
116
|
+
return full_client_request
|
|
117
|
+
|
|
118
|
+
def get_ws_header(self):
|
|
119
|
+
if self.access_token is None:
|
|
120
|
+
self.access_token = os.getenv("VOLCENGINE_TTS_ACCESS_TOKEN")
|
|
121
|
+
if self.access_token is None:
|
|
122
|
+
raise ValueError("VOLCENGINE_TTS_ACCESS_TOKEN is not set")
|
|
123
|
+
return {
|
|
124
|
+
"Authorization": f"Bearer;{self.access_token}",
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class TTS(tts.TTS):
|
|
129
|
+
def __init__(
|
|
130
|
+
self,
|
|
131
|
+
app_id: str,
|
|
132
|
+
cluster: str,
|
|
133
|
+
access_token: str | None = None,
|
|
134
|
+
voice_type: str = "BV001_V2_streaming",
|
|
135
|
+
sample_rate: Literal[24000, 16000, 8000] = 24000,
|
|
136
|
+
streaming: bool = True,
|
|
137
|
+
http_session: aiohttp.ClientSession | None = None,
|
|
138
|
+
max_session_duration: float = 600,
|
|
139
|
+
):
|
|
140
|
+
"""VolcEngine TTS
|
|
141
|
+
|
|
142
|
+
Args:
|
|
143
|
+
app_id (str): the app id of the tts, you can get it from the console.
|
|
144
|
+
cluster (str): the cluster of the tts, you can get it from the console.
|
|
145
|
+
access_token (str | None, optional): the access token of the tts, if not provided, the value of the environment variable VOLCENGINE_TTS_ACCESS_TOKEN will be used. Defaults to None.
|
|
146
|
+
voice_type (str, optional): the voice type of the tts, you can get it from https://www.volcengine.com/docs/6561/97465. Defaults to "BV001_V2_streaming". if you want to use the streaming api, you must ensure the voice type is end with "_streaming".
|
|
147
|
+
sample_rate (Literal[24000, 16000, 8000], optional): the sample rate of the tts. Defaults to 24000.
|
|
148
|
+
streaming (bool, optional): whether to use the streaming api. Defaults to True.
|
|
149
|
+
http_session (aiohttp.ClientSession | None, optional): the http session to use. Defaults to None.
|
|
150
|
+
max_session_duration (float, optional): the max duration of the http session. Defaults to 600.
|
|
151
|
+
"""
|
|
152
|
+
super().__init__(
|
|
153
|
+
capabilities=tts.TTSCapabilities(streaming=streaming),
|
|
154
|
+
sample_rate=sample_rate,
|
|
155
|
+
num_channels=1,
|
|
156
|
+
)
|
|
157
|
+
self._opts = _TTSOptions(
|
|
158
|
+
app_id=app_id,
|
|
159
|
+
cluster=cluster,
|
|
160
|
+
access_token=access_token,
|
|
161
|
+
voice_type=voice_type,
|
|
162
|
+
sample_rate=sample_rate,
|
|
163
|
+
)
|
|
164
|
+
self._session = http_session
|
|
165
|
+
|
|
166
|
+
self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse](
|
|
167
|
+
connect_cb=self._connect_ws,
|
|
168
|
+
close_cb=self._close_ws,
|
|
169
|
+
max_session_duration=max_session_duration,
|
|
170
|
+
mark_refreshed_on_get=True,
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
def _ensure_session(self) -> aiohttp.ClientSession:
|
|
174
|
+
if self._session is None:
|
|
175
|
+
self._session = utils.http_context.http_session()
|
|
176
|
+
|
|
177
|
+
return self._session
|
|
178
|
+
|
|
179
|
+
async def _connect_ws(self) -> aiohttp.ClientWebSocketResponse:
|
|
180
|
+
session = self._ensure_session()
|
|
181
|
+
url = self._opts.get_ws_url()
|
|
182
|
+
headers = self._opts.get_ws_header()
|
|
183
|
+
return await asyncio.wait_for(
|
|
184
|
+
session.ws_connect(url, headers=headers), self._conn_options.timeout
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
async def _close_ws(self, ws: aiohttp.ClientWebSocketResponse):
|
|
188
|
+
await ws.close()
|
|
189
|
+
|
|
190
|
+
def synthesize(
|
|
191
|
+
self, text, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
|
|
192
|
+
) -> ChunedStream:
|
|
193
|
+
return ChunedStream(
|
|
194
|
+
opts=self._opts,
|
|
195
|
+
session=self._ensure_session(),
|
|
196
|
+
tts=self,
|
|
197
|
+
input_text=text,
|
|
198
|
+
conn_options=conn_options,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
def stream(self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS):
|
|
202
|
+
return SynthesizeStream(
|
|
203
|
+
tts=self,
|
|
204
|
+
conn_options=conn_options,
|
|
205
|
+
opts=self._opts,
|
|
206
|
+
pool=self._pool,
|
|
207
|
+
session=self._ensure_session(),
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class ChunedStream(tts.ChunkedStream):
|
|
212
|
+
def __init__(
|
|
213
|
+
self,
|
|
214
|
+
*,
|
|
215
|
+
opts: _TTSOptions,
|
|
216
|
+
session: aiohttp.ClientSession,
|
|
217
|
+
tts: TTS,
|
|
218
|
+
input_text,
|
|
219
|
+
conn_options=None,
|
|
220
|
+
):
|
|
221
|
+
super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
|
|
222
|
+
self._opts: _TTSOptions = opts
|
|
223
|
+
self._session = session
|
|
224
|
+
|
|
225
|
+
async def _run(self) -> None:
|
|
226
|
+
request_id = utils.shortuuid()
|
|
227
|
+
bstream = utils.audio.AudioByteStream(
|
|
228
|
+
sample_rate=self._opts.sample_rate, num_channels=1
|
|
229
|
+
)
|
|
230
|
+
data = self._opts.get_http_query_params(text=self._input_text)
|
|
231
|
+
headers = self._opts.get_http_header()
|
|
232
|
+
try:
|
|
233
|
+
async with self._session.post(
|
|
234
|
+
self._opts.get_http_url(),
|
|
235
|
+
json=data,
|
|
236
|
+
headers=headers,
|
|
237
|
+
timeout=aiohttp.ClientTimeout(
|
|
238
|
+
total=30,
|
|
239
|
+
sock_connect=self._conn_options.timeout,
|
|
240
|
+
),
|
|
241
|
+
) as resp:
|
|
242
|
+
resp.raise_for_status()
|
|
243
|
+
emitter = tts.SynthesizedAudioEmitter(
|
|
244
|
+
event_ch=self._event_ch,
|
|
245
|
+
request_id=request_id,
|
|
246
|
+
)
|
|
247
|
+
data = await resp.json()
|
|
248
|
+
if "data" in data:
|
|
249
|
+
data = data["data"]
|
|
250
|
+
data = base64.b64decode(data)
|
|
251
|
+
frames = bstream.write(data)
|
|
252
|
+
for frame in frames:
|
|
253
|
+
emitter.push(frame)
|
|
254
|
+
for frame in bstream.flush():
|
|
255
|
+
emitter.push(frame)
|
|
256
|
+
emitter.flush()
|
|
257
|
+
except asyncio.TimeoutError as e:
|
|
258
|
+
raise APITimeoutError() from e
|
|
259
|
+
except aiohttp.ClientResponseError as e:
|
|
260
|
+
raise APIStatusError(
|
|
261
|
+
message=e.message,
|
|
262
|
+
status_code=e.status,
|
|
263
|
+
request_id=None,
|
|
264
|
+
body=None,
|
|
265
|
+
) from e
|
|
266
|
+
except Exception as e:
|
|
267
|
+
raise APIConnectionError() from e
|
|
268
|
+
finally:
|
|
269
|
+
emitter.flush()
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
class SynthesizeStream(tts.SynthesizeStream):
|
|
273
|
+
def __init__(
|
|
274
|
+
self,
|
|
275
|
+
*,
|
|
276
|
+
opts: _TTSOptions,
|
|
277
|
+
session: aiohttp.ClientSession,
|
|
278
|
+
pool: utils.ConnectionPool[aiohttp.ClientWebSocketResponse],
|
|
279
|
+
tts: TTS,
|
|
280
|
+
conn_options=None,
|
|
281
|
+
):
|
|
282
|
+
super().__init__(tts=tts, conn_options=conn_options)
|
|
283
|
+
self._opts: _TTSOptions = opts
|
|
284
|
+
self._session = session
|
|
285
|
+
self._pool = pool
|
|
286
|
+
|
|
287
|
+
async def _run(self):
|
|
288
|
+
request_id = utils.shortuuid()
|
|
289
|
+
|
|
290
|
+
sentence_splitter = ChineseSentenceSplitter()
|
|
291
|
+
bstream = utils.audio.AudioByteStream(
|
|
292
|
+
sample_rate=self._opts.sample_rate,
|
|
293
|
+
num_channels=1,
|
|
294
|
+
)
|
|
295
|
+
emitter = tts.SynthesizedAudioEmitter(
|
|
296
|
+
event_ch=self._event_ch,
|
|
297
|
+
request_id=request_id,
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
async def _send_task(sentence: str, ws: aiohttp.ClientWebSocketResponse):
|
|
301
|
+
data = self._opts.get_ws_query_params(text=sentence)
|
|
302
|
+
await ws.send_bytes(data)
|
|
303
|
+
|
|
304
|
+
async def _recv_task(ws: aiohttp.ClientWebSocketResponse):
|
|
305
|
+
while True:
|
|
306
|
+
res = await ws.receive_bytes()
|
|
307
|
+
done, data = parse_response(res)
|
|
308
|
+
if data is not None:
|
|
309
|
+
frames = bstream.write(data)
|
|
310
|
+
for frame in frames:
|
|
311
|
+
emitter.push(frame)
|
|
312
|
+
if done:
|
|
313
|
+
for frame in bstream.flush():
|
|
314
|
+
emitter.push(frame)
|
|
315
|
+
emitter.flush()
|
|
316
|
+
break
|
|
317
|
+
|
|
318
|
+
async for token in self._input_ch:
|
|
319
|
+
if isinstance(token, self._FlushSentinel):
|
|
320
|
+
sentences = sentence_splitter.process_text(text="", is_last=True)
|
|
321
|
+
else:
|
|
322
|
+
sentences = sentence_splitter.process_text(text=token, is_last=False)
|
|
323
|
+
for sentence in sentences:
|
|
324
|
+
if len(sentence) == 0:
|
|
325
|
+
continue
|
|
326
|
+
async with self._pool.connection() as ws:
|
|
327
|
+
await asyncio.gather(
|
|
328
|
+
_send_task(sentence, ws),
|
|
329
|
+
_recv_task(ws),
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def parse_response(res) -> Tuple[bool, ByteString | None]:
|
|
334
|
+
header_size = res[0] & 0x0F
|
|
335
|
+
message_type = res[1] >> 4
|
|
336
|
+
message_type_specific_flags = res[1] & 0x0F
|
|
337
|
+
message_compression = res[2] & 0x0F
|
|
338
|
+
payload = res[header_size * 4 :]
|
|
339
|
+
if message_type == 0xB: # audio-only server response
|
|
340
|
+
if message_type_specific_flags == 0: # no sequence number as ACK
|
|
341
|
+
return False, None
|
|
342
|
+
else:
|
|
343
|
+
sequence_number = int.from_bytes(payload[:4], "big", signed=True)
|
|
344
|
+
payload = payload[8:]
|
|
345
|
+
if sequence_number < 0:
|
|
346
|
+
return True, payload
|
|
347
|
+
else:
|
|
348
|
+
return False, payload
|
|
349
|
+
elif message_type == 0xF:
|
|
350
|
+
error_msg = payload[8:]
|
|
351
|
+
if message_compression == 1:
|
|
352
|
+
error_msg = gzip.decompress(error_msg)
|
|
353
|
+
error_msg = str(error_msg, "utf-8")
|
|
354
|
+
|
|
355
|
+
return True, None
|
|
356
|
+
elif message_type == 0xC:
|
|
357
|
+
payload = payload[4:]
|
|
358
|
+
if message_compression == 1:
|
|
359
|
+
payload = gzip.decompress(payload)
|
|
360
|
+
return False, None
|
|
361
|
+
else:
|
|
362
|
+
return True, None
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
@dataclass
|
|
366
|
+
class ChineseSentenceSplitter:
|
|
367
|
+
buffer: str = ""
|
|
368
|
+
use_level2_threshold: int = 100
|
|
369
|
+
use_level3_threshold: int = 200
|
|
370
|
+
|
|
371
|
+
def process_text(
|
|
372
|
+
self,
|
|
373
|
+
text: str,
|
|
374
|
+
is_last: bool = False,
|
|
375
|
+
special_text: str | None = None,
|
|
376
|
+
) -> List[str]:
|
|
377
|
+
self.buffer = self.buffer + text
|
|
378
|
+
if special_text is not None:
|
|
379
|
+
if self.buffer.endswith(special_text):
|
|
380
|
+
return [self.buffer]
|
|
381
|
+
sentences, indices = self.split_sentences(self.buffer)
|
|
382
|
+
assert len(sentences) == len(indices), (
|
|
383
|
+
"The number of sentences and indices do not match"
|
|
384
|
+
)
|
|
385
|
+
if not is_last:
|
|
386
|
+
if len(indices) != 0:
|
|
387
|
+
self.buffer = self.buffer[indices[-1] + 1 :]
|
|
388
|
+
return sentences
|
|
389
|
+
else:
|
|
390
|
+
if len(sentences) == 0:
|
|
391
|
+
sentences = [self.buffer]
|
|
392
|
+
self.buffer = ""
|
|
393
|
+
return sentences
|
|
394
|
+
if indices[-1] == len(self.buffer) - 1:
|
|
395
|
+
self.buffer = ""
|
|
396
|
+
return sentences
|
|
397
|
+
else:
|
|
398
|
+
self.buffer = ""
|
|
399
|
+
return sentences + [text[indices[-1] + 1 :]]
|
|
400
|
+
|
|
401
|
+
def split_sentences(self, text: str) -> List[str]:
|
|
402
|
+
indices = self.get_sentence_end_indices(text)
|
|
403
|
+
sentences = []
|
|
404
|
+
start = 0
|
|
405
|
+
for i in indices:
|
|
406
|
+
t = text[start : i + 1]
|
|
407
|
+
if len(t) > 0:
|
|
408
|
+
sentences.append(t)
|
|
409
|
+
start = i + 1
|
|
410
|
+
return sentences, indices
|
|
411
|
+
|
|
412
|
+
def is_sentence_end_level1(self, text: str) -> bool:
|
|
413
|
+
return text.endswith(
|
|
414
|
+
(
|
|
415
|
+
"!",
|
|
416
|
+
"?",
|
|
417
|
+
"。",
|
|
418
|
+
"?",
|
|
419
|
+
"!",
|
|
420
|
+
";",
|
|
421
|
+
";",
|
|
422
|
+
)
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
def is_sentence_end_level2(self, text: str) -> bool:
|
|
426
|
+
return text.endswith(
|
|
427
|
+
(
|
|
428
|
+
"、",
|
|
429
|
+
"...",
|
|
430
|
+
"…",
|
|
431
|
+
",",
|
|
432
|
+
",",
|
|
433
|
+
)
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
def is_sentence_end_level3(self, text: str) -> bool:
|
|
437
|
+
return text.endswith(
|
|
438
|
+
(
|
|
439
|
+
":",
|
|
440
|
+
":",
|
|
441
|
+
)
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
def get_sentence_end_indices(self, text: str) -> List[int]:
|
|
445
|
+
sents_l1 = [i for i, c in enumerate(text) if self.is_sentence_end_level1(c)]
|
|
446
|
+
if len(sents_l1) == 0 and len(text) > self.use_level2_threshold:
|
|
447
|
+
sents_l2 = [i for i, c in enumerate(text) if self.is_sentence_end_level2(c)]
|
|
448
|
+
if len(sents_l2) == 0 and len(text) > self.use_level3_threshold:
|
|
449
|
+
sents_l3 = [
|
|
450
|
+
i for i, c in enumerate(text) if self.is_sentence_end_level3(c)
|
|
451
|
+
]
|
|
452
|
+
return sents_l3
|
|
453
|
+
else:
|
|
454
|
+
return sents_l2
|
|
455
|
+
|
|
456
|
+
else:
|
|
457
|
+
return sents_l1
|
|
458
|
+
|
|
459
|
+
def reset(self):
|
|
460
|
+
self.buffer = ""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.0.0.rc4"
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "livekit-plugins-volcengine"
|
|
3
|
+
dynamic = ["version"]
|
|
4
|
+
description = "LiveKit Agent Plugins for Volcengine"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "wangmengdi", email = "790990241@qq.com" }
|
|
8
|
+
]
|
|
9
|
+
keywords = ["webrtc", "realtime", "audio", "video", "livekit"]
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"livekit-agents~=1.0.0rc4",
|
|
13
|
+
"pydantic>=2.0",
|
|
14
|
+
"python-dotenv>=1.1.0",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: Apache Software License",
|
|
20
|
+
"Topic :: Multimedia :: Sound/Audio",
|
|
21
|
+
"Topic :: Multimedia :: Video",
|
|
22
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
23
|
+
"Programming Language :: Python :: 3",
|
|
24
|
+
"Programming Language :: Python :: 3.9",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
[dependency-groups]
|
|
31
|
+
dev = [
|
|
32
|
+
"ipykernel>=6.29.5",
|
|
33
|
+
"livekit-plugins-cartesia>=1.0.0rc4",
|
|
34
|
+
"livekit-plugins-openai>=1.0.0rc4",
|
|
35
|
+
"livekit-plugins-deepgram>=1.0.0rc4",
|
|
36
|
+
"livekit-plugins-silero>=1.0.0rc4",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[[tool.uv.index]]
|
|
40
|
+
name = "tsinghua"
|
|
41
|
+
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
[build-system]
|
|
45
|
+
requires = ["hatchling"]
|
|
46
|
+
build-backend = "hatchling.build"
|
|
47
|
+
|
|
48
|
+
[tool.hatch.version]
|
|
49
|
+
path = "livekit/plugins/volcengine/version.py"
|
|
50
|
+
|
|
51
|
+
[tool.hatch.build.targets.wheel]
|
|
52
|
+
packages = ["livekit"]
|
|
53
|
+
|
|
54
|
+
[tool.hatch.build.targets.sdist]
|
|
55
|
+
include = ["/livekit"]
|