aichat-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aichat_sdk/__init__.py +423 -0
- aichat_sdk/asr/asr_client.py +206 -0
- aichat_sdk/config.py +12 -0
- aichat_sdk/llm/__init__.py +156 -0
- aichat_sdk/llm/mcp/__init__.py +317 -0
- aichat_sdk/llm/mcp/external.py +156 -0
- aichat_sdk/llm/mcp/mcp_end_session.py +10 -0
- aichat_sdk/llm/mcp/mcp_exec.py +32 -0
- aichat_sdk/llm/mcp/mcp_read_skill.py +44 -0
- aichat_sdk/llm/mcp/server.py +8 -0
- aichat_sdk/llm/openai_client.py +253 -0
- aichat_sdk/llm/skills.py +63 -0
- aichat_sdk/llm/system_prompt/__init__.py +49 -0
- aichat_sdk/llm/system_prompt/system.jinja2 +67 -0
- aichat_sdk/llm/type_enum.py +21 -0
- aichat_sdk/llm/utils/__init__.py +0 -0
- aichat_sdk/llm/utils/calendar.py +182 -0
- aichat_sdk/mcp_tool/__init__.py +17 -0
- aichat_sdk/mcp_tool/__main__.py +17 -0
- aichat_sdk/mcp_tool/mcp_joke.py +50 -0
- aichat_sdk/mcp_tool/mcp_play_music.py +67 -0
- aichat_sdk/mcp_tool/mcp_weather.py +51 -0
- aichat_sdk/mcp_tool/mcp_web_search.py +162 -0
- aichat_sdk/mcp_tool/music//344/270/200/350/267/257/345/220/221/345/214/227.jsonl +72 -0
- aichat_sdk/mcp_tool/music//346/231/264/345/244/251.jsonl +126 -0
- aichat_sdk/mcp_tool/server.py +8 -0
- aichat_sdk/tts/bytedance/bidirection.py +218 -0
- aichat_sdk/tts/bytedance/protocols/__init__.py +41 -0
- aichat_sdk/tts/bytedance/protocols/protocols.py +526 -0
- aichat_sdk/tts/bytedancev2/bidirection.py +338 -0
- aichat_sdk/tts/bytedancev2/protocols.py +508 -0
- aichat_sdk/tts/bytedancev3/bidirection.py +373 -0
- aichat_sdk/tts/bytedancev3/protocols/__init__.py +41 -0
- aichat_sdk/tts/bytedancev3/protocols/protocols.py +527 -0
- aichat_sdk/tts/edge/__init__.py +0 -0
- aichat_sdk/tts/edge/bidirection.py +184 -0
- aichat_sdk/tts/edge/main.py +94 -0
- aichat_sdk/utils/event_emitter.py +81 -0
- aichat_sdk/utils/opus_codec.py +136 -0
- aichat_sdk-0.1.0.dist-info/METADATA +83 -0
- aichat_sdk-0.1.0.dist-info/RECORD +42 -0
- aichat_sdk-0.1.0.dist-info/WHEEL +4 -0
aichat_sdk/__init__.py
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from aichat_sdk.asr.asr_client import AsrWebSocketClient
|
|
7
|
+
from aichat_sdk.config import ASR_SERVER_WS_URL
|
|
8
|
+
from aichat_sdk.llm import LlmAgent
|
|
9
|
+
from aichat_sdk.llm.mcp import get_default_chat_tools
|
|
10
|
+
from aichat_sdk.llm.type_enum import AgentInfo, DeviceInfo
|
|
11
|
+
from aichat_sdk.utils.opus_codec import (
|
|
12
|
+
OpusStreamDecoder,
|
|
13
|
+
OpusStreamEncoder,
|
|
14
|
+
PcmStreamResampler,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
__version__ = "0.1.0"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
IDLE_TIMEOUT_SECONDS = 60
|
|
23
|
+
IDLE_FAREWELL_PROMPT = (
|
|
24
|
+
"用户已长时间没有说话,本次会话即将自动结束。"
|
|
25
|
+
"请直接输出一两句简短自然的道别话,可以结合刚才聊到的内容。"
|
|
26
|
+
"只输出纯文本,不要调用任何工具,也不要向用户提问或期待回复。"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_tts_client(tts_engine: str = "bytedance"):
|
|
31
|
+
engine = tts_engine.lower()
|
|
32
|
+
if engine in {"edge", "edge-tts"}:
|
|
33
|
+
from aichat_sdk.tts.edge.bidirection import EdgeTTSClient
|
|
34
|
+
|
|
35
|
+
return EdgeTTSClient()
|
|
36
|
+
if engine in {"bytedance", "v2", "bytedancev2"}:
|
|
37
|
+
from aichat_sdk.tts.bytedancev2.bidirection import BytedanceTTSClient
|
|
38
|
+
|
|
39
|
+
return BytedanceTTSClient(voice_type="zh_female_wanwanxiaohe_moon_bigtts")
|
|
40
|
+
if engine in {"v3", "bytedancev3"}:
|
|
41
|
+
from aichat_sdk.tts.bytedancev3.bidirection import BytedanceTTSClient
|
|
42
|
+
|
|
43
|
+
return BytedanceTTSClient(voice_type="zh_female_vv_uranus_bigtts")
|
|
44
|
+
raise ValueError("未知 tts_engine,支持: bytedance, v2, v3, edge")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ChatBot:
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
tts_engine: str = "bytedance",
|
|
51
|
+
input_sample_rate: int = 24000, # Opus 编码采样率(限 8k/12k/16k/24k/48k),TTS 源 PCM 采样率不同时会自动重采样;
|
|
52
|
+
output_sample_rate: int = 24000, # 宿主解码后拿到的 PCM 采样率(任意),解码器用 create_opus_decoder() 创建。
|
|
53
|
+
asr_server_url: str = ASR_SERVER_WS_URL, # ASR 服务地址
|
|
54
|
+
send_idle_farewell: bool = True, # 空闲超时后是否先发送告别语,为 False 时直接关闭会话。
|
|
55
|
+
user_info: DeviceInfo = DeviceInfo(timezone=8, city="北京"), # 用户信息
|
|
56
|
+
agent_info: AgentInfo = AgentInfo(), # 角色信息
|
|
57
|
+
) -> None:
|
|
58
|
+
"""input_sample_rate:Opus 编码采样率(限 8k/12k/16k/24k/48k),
|
|
59
|
+
TTS 源 PCM 采样率不同时会自动重采样;
|
|
60
|
+
output_sample_rate:宿主解码后拿到的 PCM 采样率(任意),解码器用 create_opus_decoder() 创建。
|
|
61
|
+
send_idle_farewell:空闲超时后是否先发送告别语,为 False 时直接关闭会话。
|
|
62
|
+
"""
|
|
63
|
+
self.tts = get_tts_client(tts_engine)
|
|
64
|
+
self.asr = AsrWebSocketClient(asr_server_url)
|
|
65
|
+
|
|
66
|
+
self.output_sample_rate = output_sample_rate
|
|
67
|
+
self._send_idle_farewell = send_idle_farewell
|
|
68
|
+
self.llm = LlmAgent(user_info=user_info, agent_info=agent_info)
|
|
69
|
+
self._queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
|
70
|
+
self._input_sample_rate = input_sample_rate
|
|
71
|
+
self._opus_encoder = OpusStreamEncoder(input_sample_rate)
|
|
72
|
+
self._pcm_resampler: PcmStreamResampler | None = None
|
|
73
|
+
self._audio_text = ""
|
|
74
|
+
self._audio_terminal_sent = True
|
|
75
|
+
self._tts_session_started = False
|
|
76
|
+
self._chat_task: asyncio.Task | None = None
|
|
77
|
+
self._session_lock = asyncio.Lock()
|
|
78
|
+
self._asr_dispatch_tasks: set[asyncio.Task] = set()
|
|
79
|
+
self._pending_mcp_call: tuple[str, dict[str, Any]] | None = None
|
|
80
|
+
# 宿主从队列取走 audio_finish 时置位,等"真正播完"用
|
|
81
|
+
self._audio_finish_consumed = asyncio.Event()
|
|
82
|
+
|
|
83
|
+
self._is_session_closed = False
|
|
84
|
+
self._idle_task: asyncio.Task | None = None
|
|
85
|
+
|
|
86
|
+
def create_opus_decoder(self) -> OpusStreamDecoder:
|
|
87
|
+
"""给宿主建配套解码器,输出 output_sample_rate 的 PCM,按同样的采样率播放。"""
|
|
88
|
+
return OpusStreamDecoder(self.output_sample_rate)
|
|
89
|
+
|
|
90
|
+
def audio_callback(self, indata, frames, time, status) -> None:
|
|
91
|
+
"""把宿主采集的 16 kHz 单声道 Float32 PCM 发往 ASR 服务。"""
|
|
92
|
+
self.asr.audio_callback(indata, frames, time, status)
|
|
93
|
+
|
|
94
|
+
def _start_idle_timer(self):
|
|
95
|
+
self._cancel_idle_timer()
|
|
96
|
+
if not self._is_session_closed:
|
|
97
|
+
self._idle_task = asyncio.create_task(self._idle_farewell())
|
|
98
|
+
|
|
99
|
+
def _cancel_idle_timer(self):
|
|
100
|
+
if self._idle_task and not self._idle_task.done():
|
|
101
|
+
self._idle_task.cancel()
|
|
102
|
+
self._idle_task = None
|
|
103
|
+
|
|
104
|
+
async def _idle_farewell(self):
|
|
105
|
+
"""超过 IDLE_TIMEOUT_SECONDS 用户无回复,按配置告别或直接结束会话。"""
|
|
106
|
+
await asyncio.sleep(IDLE_TIMEOUT_SECONDS)
|
|
107
|
+
# 先清引用再 send_text,否则 send_text 里的 abort 会把当前任务自己 cancel 掉
|
|
108
|
+
self._idle_task = None
|
|
109
|
+
self._is_session_closed = True
|
|
110
|
+
|
|
111
|
+
if self._send_idle_farewell:
|
|
112
|
+
logger.info(f"[idle] 用户超过 {IDLE_TIMEOUT_SECONDS}s 未回复,推送告别")
|
|
113
|
+
await self.send_text("", response_instructions=IDLE_FAREWELL_PROMPT, role="user")
|
|
114
|
+
# await self.send_text(IDLE_FAREWELL_PROMPT, role="system")
|
|
115
|
+
|
|
116
|
+
else:
|
|
117
|
+
logger.info(f"[idle] 用户超过 {IDLE_TIMEOUT_SECONDS}s 未回复,直接关闭")
|
|
118
|
+
await self._queue.put({"type": "close"})
|
|
119
|
+
|
|
120
|
+
async def receive_item(self):
|
|
121
|
+
item = await self._queue.get()
|
|
122
|
+
if item.get("type") == "lyric":
|
|
123
|
+
logger.info(f"[music] lyric: {item['data']}")
|
|
124
|
+
# 空闲计时以宿主消费(≈播放)节奏为准:取到 audio_finish 即真正播完、开始计时;
|
|
125
|
+
# 音频是瞬间推满队列的(整首歌几秒推完),按推送侧计时会在播放途中误触发。
|
|
126
|
+
elif item.get("type") == "audio_finish":
|
|
127
|
+
self._start_idle_timer()
|
|
128
|
+
self._audio_finish_consumed.set()
|
|
129
|
+
elif item.get("type") == "audio":
|
|
130
|
+
self._cancel_idle_timer()
|
|
131
|
+
return item
|
|
132
|
+
|
|
133
|
+
async def reg_llm(self):
|
|
134
|
+
|
|
135
|
+
@self.llm.on("text")
|
|
136
|
+
async def handle_chat_text(text: str):
|
|
137
|
+
# 会话开着但 10s 没收到文本,服务端就把它作废:之后送的字不再出声,
|
|
138
|
+
# FinishSession 也可能不回(实测 9s 正常、10s 起必挂)。工具调用和
|
|
139
|
+
# LLM 首字慢都能轻松撑过 10s,所以等真的有字了再开会话。
|
|
140
|
+
if not self._tts_session_started:
|
|
141
|
+
self._tts_session_started = True
|
|
142
|
+
await self.tts.start_session()
|
|
143
|
+
await self.tts.send_char(text)
|
|
144
|
+
|
|
145
|
+
@self.llm.on("tool")
|
|
146
|
+
async def handle_chat_tool(payload: dict):
|
|
147
|
+
arguments_str = payload["arguments"]
|
|
148
|
+
if len(arguments_str) > 50:
|
|
149
|
+
arguments_str = arguments_str[:50] + "..."
|
|
150
|
+
logger.info(f"[tool] tool_call: {payload['name']} ({arguments_str})")
|
|
151
|
+
await self._queue.put({"type": "tool", "data": payload})
|
|
152
|
+
|
|
153
|
+
@self.llm.on("before_tool_exec")
|
|
154
|
+
async def handle_chat_before_tool_exec(text: str):
|
|
155
|
+
await self._wait_audio_played(text)
|
|
156
|
+
|
|
157
|
+
@self.llm.on("tool_results")
|
|
158
|
+
async def handle_chat_tool_results(payload: list[dict]):
|
|
159
|
+
payload_str = json.dumps(payload, ensure_ascii=False)
|
|
160
|
+
if len(payload_str) > 100:
|
|
161
|
+
payload_str = payload_str[:100] + "..."
|
|
162
|
+
logger.info(f"[tool] tool_results: {payload_str}")
|
|
163
|
+
await self._queue.put({"type": "tool_results", "data": payload})
|
|
164
|
+
|
|
165
|
+
if any(msg.get("name") == "end_session" for msg in payload):
|
|
166
|
+
self._is_session_closed = True
|
|
167
|
+
|
|
168
|
+
for msg in payload:
|
|
169
|
+
if msg["name"] == "play_music":
|
|
170
|
+
try:
|
|
171
|
+
result = json.loads(msg["content"])
|
|
172
|
+
mcp_name = result.get("mcp")
|
|
173
|
+
song = result.get("song")
|
|
174
|
+
if isinstance(mcp_name, str) and isinstance(song, str):
|
|
175
|
+
self._pending_mcp_call = (mcp_name, {"song": song})
|
|
176
|
+
except (json.JSONDecodeError, KeyError, TypeError):
|
|
177
|
+
pass # 结果里没有 path 说明选歌失败,无需播放
|
|
178
|
+
|
|
179
|
+
@self.llm.on("finish")
|
|
180
|
+
async def handle_chat_finish(text: str):
|
|
181
|
+
self._audio_text = text
|
|
182
|
+
await self.tts.finish_session()
|
|
183
|
+
# 正常实现会在 finish_session 内触发 audio_finish;这里作为协议
|
|
184
|
+
# 兜底,防止第三方 TTS 返回却忘记发终结事件。
|
|
185
|
+
await self._finish_audio_and_followups()
|
|
186
|
+
|
|
187
|
+
async def reg_tts(self):
|
|
188
|
+
|
|
189
|
+
@self.tts.on("audio")
|
|
190
|
+
async def handle_audio(pcm_data: bytes):
|
|
191
|
+
if self._pcm_resampler is not None:
|
|
192
|
+
pcm_data = self._pcm_resampler.resample(pcm_data)
|
|
193
|
+
for packet in self._opus_encoder.encode(pcm_data):
|
|
194
|
+
await self._queue.put({"type": "audio", "data": packet})
|
|
195
|
+
|
|
196
|
+
@self.tts.on("audio_finish")
|
|
197
|
+
async def handle_audio_finish():
|
|
198
|
+
await self._finish_audio_and_followups()
|
|
199
|
+
|
|
200
|
+
@self.tts.on("sentence_start")
|
|
201
|
+
async def handle_sentence_start(text: str):
|
|
202
|
+
logger.info(f"[tts] {text}")
|
|
203
|
+
await self._queue.put({"type": "sentence_start", "data": text})
|
|
204
|
+
|
|
205
|
+
@self.tts.on("sentence_end")
|
|
206
|
+
async def handle_sentence_end(text: str):
|
|
207
|
+
await self._queue.put({"type": "sentence_end", "data": text})
|
|
208
|
+
|
|
209
|
+
async def reg_asr(self):
|
|
210
|
+
@self.asr.on("stt")
|
|
211
|
+
async def handle_stt(payload: dict):
|
|
212
|
+
logger.info(f"[asr] {payload['content']}")
|
|
213
|
+
self.tts.set_last_context_text(payload["content"])
|
|
214
|
+
task = asyncio.create_task(self.send_text(payload["content"], audio_url=payload.get("audio_url")))
|
|
215
|
+
self._asr_dispatch_tasks.add(task)
|
|
216
|
+
task.add_done_callback(self._finish_asr_dispatch)
|
|
217
|
+
|
|
218
|
+
@self.asr.on("error")
|
|
219
|
+
async def handle_asr_error(payload: dict):
|
|
220
|
+
logger.error("[asr] server error: %s", payload)
|
|
221
|
+
|
|
222
|
+
def _finish_asr_dispatch(self, task: asyncio.Task) -> None:
|
|
223
|
+
self._asr_dispatch_tasks.discard(task)
|
|
224
|
+
if task.cancelled():
|
|
225
|
+
return
|
|
226
|
+
error = task.exception()
|
|
227
|
+
if error is not None:
|
|
228
|
+
logger.error(
|
|
229
|
+
"ASR result dispatch failed",
|
|
230
|
+
exc_info=(type(error), error, error.__traceback__),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
async def initialize(self):
|
|
234
|
+
await self.reg_llm()
|
|
235
|
+
await self.reg_tts()
|
|
236
|
+
await self.reg_asr()
|
|
237
|
+
try:
|
|
238
|
+
await self.asr.connect()
|
|
239
|
+
except Exception as e:
|
|
240
|
+
logger.error(f"ASR connect failed: {e}")
|
|
241
|
+
await self.asr.close()
|
|
242
|
+
exit(1)
|
|
243
|
+
try:
|
|
244
|
+
await self.tts.connect()
|
|
245
|
+
except Exception as e:
|
|
246
|
+
logger.error(f"TTS connect failed: {e}")
|
|
247
|
+
await self.tts.close()
|
|
248
|
+
exit(1)
|
|
249
|
+
# 工具列表是懒构建的,不预热的话第一次拉起外部 MCP server(stdio 子进程)
|
|
250
|
+
# 的耗时会算进第一轮回复;构建是阻塞调用,放线程里跑不挡事件循环。
|
|
251
|
+
await asyncio.to_thread(get_default_chat_tools)
|
|
252
|
+
source_sample_rate = getattr(
|
|
253
|
+
self.tts,
|
|
254
|
+
"sample_rate",
|
|
255
|
+
self._input_sample_rate,
|
|
256
|
+
)
|
|
257
|
+
if source_sample_rate != self._input_sample_rate:
|
|
258
|
+
self._pcm_resampler = PcmStreamResampler(
|
|
259
|
+
source_sample_rate,
|
|
260
|
+
self._input_sample_rate,
|
|
261
|
+
)
|
|
262
|
+
self._start_idle_timer()
|
|
263
|
+
|
|
264
|
+
async def close(self):
|
|
265
|
+
self._cancel_idle_timer()
|
|
266
|
+
for task in self._asr_dispatch_tasks:
|
|
267
|
+
task.cancel()
|
|
268
|
+
if self._asr_dispatch_tasks:
|
|
269
|
+
await asyncio.gather(
|
|
270
|
+
*self._asr_dispatch_tasks,
|
|
271
|
+
return_exceptions=True,
|
|
272
|
+
)
|
|
273
|
+
self._asr_dispatch_tasks.clear()
|
|
274
|
+
await self.abort()
|
|
275
|
+
await self.asr.close()
|
|
276
|
+
close = getattr(self.tts, "close", None) or getattr(
|
|
277
|
+
self.tts,
|
|
278
|
+
"disconnect",
|
|
279
|
+
None,
|
|
280
|
+
)
|
|
281
|
+
if close is not None:
|
|
282
|
+
await close()
|
|
283
|
+
|
|
284
|
+
async def _run_session_task(self, coroutine_factory) -> None:
|
|
285
|
+
async with self._session_lock:
|
|
286
|
+
await self._abort_locked()
|
|
287
|
+
task = asyncio.create_task(coroutine_factory())
|
|
288
|
+
self._chat_task = task
|
|
289
|
+
try:
|
|
290
|
+
await task
|
|
291
|
+
except asyncio.CancelledError:
|
|
292
|
+
# 被下一轮会话替换时,只有子任务被 cancel,当前包装任务本身没有
|
|
293
|
+
# cancellation request;调用方主动 cancel 时则清理会话并继续传播。
|
|
294
|
+
if asyncio.current_task().cancelling():
|
|
295
|
+
async with self._session_lock:
|
|
296
|
+
if self._chat_task is task:
|
|
297
|
+
await self._abort_locked(notify_host=True)
|
|
298
|
+
raise
|
|
299
|
+
finally:
|
|
300
|
+
async with self._session_lock:
|
|
301
|
+
if self._chat_task is task:
|
|
302
|
+
self._chat_task = None
|
|
303
|
+
|
|
304
|
+
async def send_text(self, user_text: str, role: str = "user", response_instructions: str = None, audio_url: str | None = None):
|
|
305
|
+
"""audio_url:这句话对应的用户录音地址(ASR 服务给的),只挂在 messages 上供宿主回放,不发给 LLM。"""
|
|
306
|
+
await self._run_session_task(lambda: self._run_chat(user_text, role, response_instructions, audio_url))
|
|
307
|
+
|
|
308
|
+
async def _run_chat(self, user_text: str, role, response_instructions, audio_url: str | None = None):
|
|
309
|
+
self._prepare_audio_session()
|
|
310
|
+
try:
|
|
311
|
+
await self._queue.put({"type": "stt", "data": user_text})
|
|
312
|
+
await self.llm.chat(user_text, role, response_instructions, audio_url=audio_url)
|
|
313
|
+
except asyncio.CancelledError:
|
|
314
|
+
raise
|
|
315
|
+
except Exception:
|
|
316
|
+
logger.exception("chat pipeline failed")
|
|
317
|
+
await self._fail_audio_session()
|
|
318
|
+
raise
|
|
319
|
+
|
|
320
|
+
async def send_tts_text(self, text: str):
|
|
321
|
+
await self._run_session_task(lambda: self._run_tts_text(text))
|
|
322
|
+
|
|
323
|
+
async def _run_tts_text(self, text: str) -> None:
|
|
324
|
+
self._prepare_audio_session()
|
|
325
|
+
try:
|
|
326
|
+
await self.tts.start_session()
|
|
327
|
+
await self.tts.send_char(text)
|
|
328
|
+
self._audio_text = text
|
|
329
|
+
await self.tts.finish_session()
|
|
330
|
+
await self._finish_audio_and_followups()
|
|
331
|
+
except asyncio.CancelledError:
|
|
332
|
+
raise
|
|
333
|
+
except Exception:
|
|
334
|
+
logger.exception("direct TTS pipeline failed")
|
|
335
|
+
await self._fail_audio_session()
|
|
336
|
+
raise
|
|
337
|
+
|
|
338
|
+
async def _wait_audio_played(self, text: str) -> None:
|
|
339
|
+
"""工具执行前等本轮已出声的文本真正播完。
|
|
340
|
+
|
|
341
|
+
先收掉 TTS 会话、推 audio_finish,再等宿主把它取走(音频是瞬间推满队列的,
|
|
342
|
+
只有宿主取到 audio_finish 才算播完)。之后按新一段音频处理:工具返回后的
|
|
343
|
+
回复重新开会话、结束时再推一条 audio_finish。一个字都没吐过就不用等,
|
|
344
|
+
也不能去收一个没开过的会话。
|
|
345
|
+
"""
|
|
346
|
+
if not self._tts_session_started:
|
|
347
|
+
return
|
|
348
|
+
self._audio_text = text
|
|
349
|
+
self._audio_finish_consumed.clear()
|
|
350
|
+
await self.tts.finish_session()
|
|
351
|
+
await self._finish_audio_and_followups()
|
|
352
|
+
await self._audio_finish_consumed.wait()
|
|
353
|
+
# 取走 audio_finish 会起空闲计时,但工具还在跑,不算用户没回复
|
|
354
|
+
self._cancel_idle_timer()
|
|
355
|
+
self._prepare_audio_session()
|
|
356
|
+
|
|
357
|
+
def _prepare_audio_session(self) -> None:
|
|
358
|
+
self._audio_text = ""
|
|
359
|
+
self._audio_terminal_sent = False
|
|
360
|
+
self._tts_session_started = False
|
|
361
|
+
|
|
362
|
+
async def _enqueue_audio_finish(self) -> bool:
|
|
363
|
+
if self._audio_terminal_sent:
|
|
364
|
+
return False
|
|
365
|
+
self._audio_terminal_sent = True
|
|
366
|
+
if self._pcm_resampler is not None:
|
|
367
|
+
pcm = self._pcm_resampler.flush()
|
|
368
|
+
for packet in self._opus_encoder.encode(pcm):
|
|
369
|
+
await self._queue.put({"type": "audio", "data": packet})
|
|
370
|
+
for packet in self._opus_encoder.flush():
|
|
371
|
+
await self._queue.put({"type": "audio", "data": packet})
|
|
372
|
+
await self._queue.put({"type": "audio_finish", "data": self._audio_text})
|
|
373
|
+
return True
|
|
374
|
+
|
|
375
|
+
async def _finish_audio_and_followups(self) -> None:
|
|
376
|
+
if not await self._enqueue_audio_finish():
|
|
377
|
+
return
|
|
378
|
+
if self._is_session_closed:
|
|
379
|
+
await self._queue.put({"type": "close"})
|
|
380
|
+
elif self._pending_mcp_call:
|
|
381
|
+
# 等本轮回复的语音播完再开始放歌,避免歌声盖住回复
|
|
382
|
+
name, args = self._pending_mcp_call
|
|
383
|
+
self._pending_mcp_call = None
|
|
384
|
+
async for data in self.llm.stream_mcp_tool(name, args):
|
|
385
|
+
await self._queue.put(data)
|
|
386
|
+
|
|
387
|
+
async def _fail_audio_session(self) -> None:
|
|
388
|
+
try:
|
|
389
|
+
await self.tts.abort()
|
|
390
|
+
except Exception:
|
|
391
|
+
logger.exception("failed to abort TTS after pipeline error")
|
|
392
|
+
await self._enqueue_audio_finish()
|
|
393
|
+
|
|
394
|
+
async def abort(self):
|
|
395
|
+
async with self._session_lock:
|
|
396
|
+
await self._abort_locked(notify_host=True)
|
|
397
|
+
|
|
398
|
+
async def _abort_locked(self, *, notify_host: bool = False) -> None:
|
|
399
|
+
self._cancel_idle_timer()
|
|
400
|
+
session_was_active = not self._audio_terminal_sent
|
|
401
|
+
task = self._chat_task
|
|
402
|
+
if task and task is not asyncio.current_task() and not task.done():
|
|
403
|
+
task.cancel()
|
|
404
|
+
await asyncio.gather(task, return_exceptions=True)
|
|
405
|
+
|
|
406
|
+
if self._chat_task is task:
|
|
407
|
+
self._chat_task = None
|
|
408
|
+
self._pending_mcp_call = None
|
|
409
|
+
try:
|
|
410
|
+
await self.tts.abort()
|
|
411
|
+
except Exception:
|
|
412
|
+
logger.exception("failed to abort TTS session")
|
|
413
|
+
if self._pcm_resampler is not None:
|
|
414
|
+
self._pcm_resampler.reset()
|
|
415
|
+
self._opus_encoder.reset()
|
|
416
|
+
removed_audio_finish = False
|
|
417
|
+
while not self._queue.empty():
|
|
418
|
+
item = self._queue.get_nowait()
|
|
419
|
+
removed_audio_finish |= item.get("type") == "audio_finish"
|
|
420
|
+
self._audio_text = ""
|
|
421
|
+
self._audio_terminal_sent = True
|
|
422
|
+
if notify_host and (session_was_active or removed_audio_finish):
|
|
423
|
+
await self._queue.put({"type": "audio_finish", "data": ""})
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
from typing import Any
|
|
7
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import websockets
|
|
11
|
+
|
|
12
|
+
from aichat_sdk.utils.event_emitter import AsyncEventEmitter
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger("aichat_sdk")
|
|
15
|
+
|
|
16
|
+
SAMPLE_RATE = 16000
|
|
17
|
+
CHUNK_SIZE_MS = 240
|
|
18
|
+
BLOCK_SIZE = SAMPLE_RATE * CHUNK_SIZE_MS // 1000
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AsrWebSocketClient(AsyncEventEmitter):
|
|
22
|
+
"""Stream microphone PCM to the ASR WebSocket service."""
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
url: str,
|
|
27
|
+
*,
|
|
28
|
+
reconnect_delay: float = 1.0,
|
|
29
|
+
max_audio_queue: int = 64,
|
|
30
|
+
) -> None:
|
|
31
|
+
super().__init__()
|
|
32
|
+
self.url = url
|
|
33
|
+
# 服务端 stt 里的 audio_url 是相对路径(/audio/...),按 ws 地址换成同主机的 http 地址
|
|
34
|
+
parts = urlsplit(url)
|
|
35
|
+
self._http_base = urlunsplit(("https" if parts.scheme == "wss" else "http", parts.netloc, "", "", ""))
|
|
36
|
+
self.reconnect_delay = reconnect_delay
|
|
37
|
+
self._audio_queue: asyncio.Queue[np.ndarray] = asyncio.Queue(maxsize=max_audio_queue)
|
|
38
|
+
self._loop: asyncio.AbstractEventLoop | None = None
|
|
39
|
+
self._websocket: Any | None = None
|
|
40
|
+
self._manager_task: asyncio.Task[None] | None = None
|
|
41
|
+
self._closing = False
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def connected(self) -> bool:
|
|
45
|
+
return self._websocket is not None
|
|
46
|
+
|
|
47
|
+
async def connect(self) -> None:
|
|
48
|
+
"""Connect once, then keep reconnecting in the background if disconnected."""
|
|
49
|
+
if self._manager_task is not None and not self._manager_task.done():
|
|
50
|
+
return
|
|
51
|
+
|
|
52
|
+
self._loop = asyncio.get_running_loop()
|
|
53
|
+
self._closing = False
|
|
54
|
+
websocket = await websockets.connect(self.url, max_size=1024 * 1024)
|
|
55
|
+
self._websocket = websocket
|
|
56
|
+
self._manager_task = asyncio.create_task(
|
|
57
|
+
self._connection_manager(websocket),
|
|
58
|
+
name="asr-websocket-connection",
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
async def close(self) -> None:
|
|
62
|
+
self._closing = True
|
|
63
|
+
task = self._manager_task
|
|
64
|
+
self._manager_task = None
|
|
65
|
+
if task is not None:
|
|
66
|
+
task.cancel()
|
|
67
|
+
await asyncio.gather(task, return_exceptions=True)
|
|
68
|
+
self._websocket = None
|
|
69
|
+
self._clear_audio_queue()
|
|
70
|
+
|
|
71
|
+
def audio_callback(
|
|
72
|
+
self,
|
|
73
|
+
indata: np.ndarray,
|
|
74
|
+
frames: int,
|
|
75
|
+
time: Any,
|
|
76
|
+
status: Any,
|
|
77
|
+
) -> None:
|
|
78
|
+
"""sounddevice callback: copy mono float samples into the asyncio loop."""
|
|
79
|
+
del frames, time
|
|
80
|
+
if status:
|
|
81
|
+
logger.warning("ASR audio input status: %s", status)
|
|
82
|
+
loop = self._loop
|
|
83
|
+
if loop is None or not loop.is_running() or not self.connected:
|
|
84
|
+
return
|
|
85
|
+
|
|
86
|
+
audio = np.asarray(indata)
|
|
87
|
+
if audio.ndim == 2:
|
|
88
|
+
audio = audio[:, 0]
|
|
89
|
+
chunk = np.ascontiguousarray(audio, dtype="<f4").copy()
|
|
90
|
+
loop.call_soon_threadsafe(self._enqueue_audio, chunk)
|
|
91
|
+
|
|
92
|
+
async def send_audio(self, chunk: np.ndarray) -> bool:
|
|
93
|
+
"""Queue one PCM chunk; return False while the service is disconnected."""
|
|
94
|
+
if not self.connected:
|
|
95
|
+
return False
|
|
96
|
+
audio = np.asarray(chunk)
|
|
97
|
+
if audio.ndim == 2:
|
|
98
|
+
audio = audio[:, 0]
|
|
99
|
+
self._enqueue_audio(np.ascontiguousarray(audio, dtype="<f4").copy())
|
|
100
|
+
return True
|
|
101
|
+
|
|
102
|
+
def _enqueue_audio(self, chunk: np.ndarray) -> None:
|
|
103
|
+
if not self.connected:
|
|
104
|
+
return
|
|
105
|
+
if self._audio_queue.full():
|
|
106
|
+
self._audio_queue.get_nowait()
|
|
107
|
+
logger.warning("ASR audio queue full; dropped the oldest chunk")
|
|
108
|
+
self._audio_queue.put_nowait(chunk)
|
|
109
|
+
|
|
110
|
+
def _clear_audio_queue(self) -> None:
|
|
111
|
+
while not self._audio_queue.empty():
|
|
112
|
+
self._audio_queue.get_nowait()
|
|
113
|
+
|
|
114
|
+
async def _connection_manager(self, websocket: Any) -> None:
|
|
115
|
+
try:
|
|
116
|
+
while not self._closing:
|
|
117
|
+
self._websocket = websocket
|
|
118
|
+
logger.info("ASR WebSocket connected: %s", self.url)
|
|
119
|
+
await self.emit("connected")
|
|
120
|
+
try:
|
|
121
|
+
await self._run_connection(websocket)
|
|
122
|
+
except asyncio.CancelledError:
|
|
123
|
+
raise
|
|
124
|
+
except Exception:
|
|
125
|
+
logger.exception("ASR WebSocket connection lost")
|
|
126
|
+
finally:
|
|
127
|
+
self._websocket = None
|
|
128
|
+
try:
|
|
129
|
+
await websocket.close()
|
|
130
|
+
except Exception:
|
|
131
|
+
logger.exception("failed to close ASR WebSocket")
|
|
132
|
+
self._clear_audio_queue()
|
|
133
|
+
|
|
134
|
+
if self._closing:
|
|
135
|
+
break
|
|
136
|
+
await self.emit("disconnected")
|
|
137
|
+
websocket = await self._reconnect()
|
|
138
|
+
finally:
|
|
139
|
+
self._websocket = None
|
|
140
|
+
|
|
141
|
+
async def _reconnect(self) -> Any:
|
|
142
|
+
while not self._closing:
|
|
143
|
+
await asyncio.sleep(self.reconnect_delay)
|
|
144
|
+
try:
|
|
145
|
+
websocket = await websockets.connect(
|
|
146
|
+
self.url,
|
|
147
|
+
max_size=1024 * 1024,
|
|
148
|
+
)
|
|
149
|
+
logger.info("ASR WebSocket reconnected: %s", self.url)
|
|
150
|
+
return websocket
|
|
151
|
+
except asyncio.CancelledError:
|
|
152
|
+
raise
|
|
153
|
+
except Exception:
|
|
154
|
+
logger.exception("ASR WebSocket reconnect failed")
|
|
155
|
+
raise asyncio.CancelledError
|
|
156
|
+
|
|
157
|
+
async def _run_connection(self, websocket: Any) -> None:
|
|
158
|
+
sender = asyncio.create_task(
|
|
159
|
+
self._send_audio(websocket),
|
|
160
|
+
name="asr-websocket-sender",
|
|
161
|
+
)
|
|
162
|
+
receiver = asyncio.create_task(
|
|
163
|
+
self._receive_results(websocket),
|
|
164
|
+
name="asr-websocket-receiver",
|
|
165
|
+
)
|
|
166
|
+
tasks = {sender, receiver}
|
|
167
|
+
try:
|
|
168
|
+
done, _ = await asyncio.wait(
|
|
169
|
+
tasks,
|
|
170
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
171
|
+
)
|
|
172
|
+
for task in done:
|
|
173
|
+
task.result()
|
|
174
|
+
finally:
|
|
175
|
+
for task in tasks:
|
|
176
|
+
task.cancel()
|
|
177
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
178
|
+
|
|
179
|
+
async def _send_audio(self, websocket: Any) -> None:
|
|
180
|
+
while True:
|
|
181
|
+
chunk = await self._audio_queue.get()
|
|
182
|
+
await websocket.send(chunk.tobytes())
|
|
183
|
+
|
|
184
|
+
async def _receive_results(self, websocket: Any) -> None:
|
|
185
|
+
async for raw_message in websocket:
|
|
186
|
+
if not isinstance(raw_message, str):
|
|
187
|
+
logger.warning("ignored binary ASR server message")
|
|
188
|
+
continue
|
|
189
|
+
try:
|
|
190
|
+
message = json.loads(raw_message)
|
|
191
|
+
except json.JSONDecodeError:
|
|
192
|
+
logger.warning("ignored invalid ASR server JSON: %r", raw_message)
|
|
193
|
+
continue
|
|
194
|
+
|
|
195
|
+
message_type = message.get("type")
|
|
196
|
+
if message_type == "stt":
|
|
197
|
+
payload = dict(message)
|
|
198
|
+
payload.pop("type", None)
|
|
199
|
+
audio_url = payload.get("audio_url")
|
|
200
|
+
if isinstance(audio_url, str) and audio_url.startswith("/"):
|
|
201
|
+
payload["audio_url"] = self._http_base + audio_url
|
|
202
|
+
await self.emit("stt", payload)
|
|
203
|
+
elif message_type == "error":
|
|
204
|
+
await self.emit("error", message)
|
|
205
|
+
else:
|
|
206
|
+
logger.warning("ignored unknown ASR server message: %r", message)
|
aichat_sdk/config.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
END_SESSION_TOOL_NAME = "end_session"
|
|
4
|
+
DEFAULT_AGENT_SOUL = """你是一个智能助手,擅长回答各种问题,提供信息,解决问题,并能根据用户需求提供相应的帮助。"""
|
|
5
|
+
|
|
6
|
+
OPENAI_MODEL = os.environ.get("OPENAI_MODEL", "gpt-5.4")
|
|
7
|
+
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "")
|
|
8
|
+
OPENAI_BASE_URL = os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1")
|
|
9
|
+
ASR_SERVER_WS_URL = os.environ.get("ASR_SERVER_WS_URL", "ws://127.0.0.1:8086/ws")
|
|
10
|
+
|
|
11
|
+
BYTEDANCE_TTS_APP_ID = os.environ.get("BYTEDANCE_TTS_APP_ID", "")
|
|
12
|
+
BYTEDANCE_TTS_APP_KEY = os.environ.get("BYTEDANCE_TTS_APP_KEY", "")
|