srtc 0.1.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- srtc/__init__.py +55 -0
- srtc/_ffi.py +242 -0
- srtc/_lib/BUILD_INFO +3 -0
- srtc/_lib/librtc.dll +0 -0
- srtc/_version.py +7 -0
- srtc/audio.py +257 -0
- srtc/channel.py +571 -0
- srtc/errors.py +27 -0
- srtc/logger.py +23 -0
- srtc/pipecat_transport.py +262 -0
- srtc/types.py +315 -0
- srtc/video.py +45 -0
- srtc-0.1.0.dist-info/METADATA +169 -0
- srtc-0.1.0.dist-info/RECORD +15 -0
- srtc-0.1.0.dist-info/WHEEL +4 -0
srtc/__init__.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""
|
|
2
|
+
SRTC Python SDK —— 面向服务端 AI 场景(ASR → LLM → TTS)的实时音视频 SDK
|
|
3
|
+
|
|
4
|
+
底层是 rtc-go 的 C 库(信令、SFU 协商、RTP 收发都在 Go 里),本包负责 asyncio 封装和
|
|
5
|
+
Opus/PCM 转换:收到的是解码好的 PCM,发送时写 PCM 即可。
|
|
6
|
+
|
|
7
|
+
import srtc
|
|
8
|
+
|
|
9
|
+
class Agent(srtc.ChannelHandler):
|
|
10
|
+
def on_user_join(self, user):
|
|
11
|
+
print("加入:", user.uid)
|
|
12
|
+
|
|
13
|
+
async with await srtc.Channel.join(token, handler=Agent(), auto_subscribe_audio=True) as ch:
|
|
14
|
+
tts = await ch.publish_audio(audio_format=srtc.AudioFormat(24000, 1))
|
|
15
|
+
async for frame in ch.audio_frames(): # 16k 单声道 PCM
|
|
16
|
+
text = asr.feed(frame.uid, frame.pcm)
|
|
17
|
+
...
|
|
18
|
+
await tts.write(pcm_24k) # SDK 按实时节奏发送
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from ._version import __version__
|
|
22
|
+
from .audio import AudioFormat, AudioTrack
|
|
23
|
+
from .channel import Channel, ChannelHandler
|
|
24
|
+
from .errors import SdkError
|
|
25
|
+
from .logger import set_log_level
|
|
26
|
+
from .types import (
|
|
27
|
+
MCU_PUBLISHER_UID,
|
|
28
|
+
TRACK_AMCU_ID,
|
|
29
|
+
TRACK_MCU_ID,
|
|
30
|
+
ActiveSpeaker,
|
|
31
|
+
AudioFrame,
|
|
32
|
+
ChannelInfo,
|
|
33
|
+
Codec,
|
|
34
|
+
ConnectionQuality,
|
|
35
|
+
ConnectionState,
|
|
36
|
+
CustomMsg,
|
|
37
|
+
DeviceType,
|
|
38
|
+
DisconnectReason,
|
|
39
|
+
LayerSwitched,
|
|
40
|
+
QualitySample,
|
|
41
|
+
TrackInfo,
|
|
42
|
+
TrackKind,
|
|
43
|
+
UserInfo,
|
|
44
|
+
VideoFrame,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"Channel", "ChannelHandler", "AudioTrack", "AudioFormat",
|
|
49
|
+
"AudioFrame", "VideoFrame",
|
|
50
|
+
"ChannelInfo", "UserInfo", "TrackInfo",
|
|
51
|
+
"ConnectionQuality", "QualitySample", "ActiveSpeaker", "LayerSwitched", "CustomMsg",
|
|
52
|
+
"Codec", "TrackKind", "DeviceType", "ConnectionState", "DisconnectReason",
|
|
53
|
+
"MCU_PUBLISHER_UID", "TRACK_AMCU_ID", "TRACK_MCU_ID",
|
|
54
|
+
"SdkError", "set_log_level", "__version__",
|
|
55
|
+
]
|
srtc/_ffi.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""
|
|
2
|
+
rtc-go C 库(librtc)的 cffi 绑定
|
|
3
|
+
|
|
4
|
+
用 cffi 的 ABI 模式直接 dlopen 动态库,不编译 C 扩展:
|
|
5
|
+
同一个平台的 wheel 对所有 Python 3.x 通用(py3-none-<平台>)。
|
|
6
|
+
|
|
7
|
+
CDEF 是 rtc-go/capi/rtc_sdk.h 的手抄版(cffi 不认 static inline / #ifdef),
|
|
8
|
+
tests/test_ffi.py 会拿随库分发的 librtc.h 校验函数清单,防止两边漂移。
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from cffi import FFI
|
|
18
|
+
|
|
19
|
+
CDEF = r"""
|
|
20
|
+
#define RTC_OK 0
|
|
21
|
+
#define RTC_ERROR -1
|
|
22
|
+
#define RTC_INVALID_PARAM -2
|
|
23
|
+
#define RTC_NOT_CONNECTED -3
|
|
24
|
+
#define RTC_TIMEOUT -4
|
|
25
|
+
|
|
26
|
+
typedef struct {
|
|
27
|
+
char track_id[64];
|
|
28
|
+
char uid[64];
|
|
29
|
+
char desc[64];
|
|
30
|
+
int kind;
|
|
31
|
+
int codec;
|
|
32
|
+
int width;
|
|
33
|
+
int height;
|
|
34
|
+
int fps;
|
|
35
|
+
int angle;
|
|
36
|
+
int bitrate;
|
|
37
|
+
int sample_rate;
|
|
38
|
+
int channel_count;
|
|
39
|
+
const char* props;
|
|
40
|
+
} rtc_track_info_t;
|
|
41
|
+
|
|
42
|
+
typedef struct {
|
|
43
|
+
char uid[64];
|
|
44
|
+
char name[128];
|
|
45
|
+
char device_id[128];
|
|
46
|
+
char version[64];
|
|
47
|
+
char channel[128];
|
|
48
|
+
char sid[128];
|
|
49
|
+
const char* props;
|
|
50
|
+
int device_type;
|
|
51
|
+
int is_audience;
|
|
52
|
+
int64_t join_at;
|
|
53
|
+
int64_t leave_at;
|
|
54
|
+
int64_t updated_at;
|
|
55
|
+
rtc_track_info_t* stream_tracks;
|
|
56
|
+
int stream_track_count;
|
|
57
|
+
} rtc_user_info_t;
|
|
58
|
+
|
|
59
|
+
typedef struct {
|
|
60
|
+
char desc[64];
|
|
61
|
+
int width;
|
|
62
|
+
int height;
|
|
63
|
+
int fps;
|
|
64
|
+
int sample_rate;
|
|
65
|
+
int channel_count;
|
|
66
|
+
int angle;
|
|
67
|
+
int bitrate;
|
|
68
|
+
const char* props;
|
|
69
|
+
} rtc_publish_options_t;
|
|
70
|
+
|
|
71
|
+
typedef void (*rtc_connection_callback)(void* context, int state);
|
|
72
|
+
typedef void (*rtc_user_event_callback)(void* context, const char* uid, int event_type);
|
|
73
|
+
typedef void (*rtc_track_event_callback)(void* context, const char* uid, rtc_track_info_t* track_info, int event_type);
|
|
74
|
+
typedef void (*rtc_track_sample_callback)(void* context, rtc_user_info_t* user_info, rtc_track_info_t* track_info,
|
|
75
|
+
uint8_t* data, int len, int64_t timestamp, int64_t duration);
|
|
76
|
+
|
|
77
|
+
typedef struct {
|
|
78
|
+
char sub_key[128];
|
|
79
|
+
char from_track_id[64];
|
|
80
|
+
char to_track_id[64];
|
|
81
|
+
char reason[32];
|
|
82
|
+
int64_t latency_ms;
|
|
83
|
+
} rtc_layer_switched_t;
|
|
84
|
+
typedef void (*rtc_layer_switched_callback)(void* context, const rtc_layer_switched_t* data);
|
|
85
|
+
|
|
86
|
+
typedef struct {
|
|
87
|
+
double score;
|
|
88
|
+
char level[16];
|
|
89
|
+
double mos;
|
|
90
|
+
double loss;
|
|
91
|
+
double rtt;
|
|
92
|
+
double jitter;
|
|
93
|
+
int64_t packets;
|
|
94
|
+
double bitrate;
|
|
95
|
+
int64_t bytes;
|
|
96
|
+
} rtc_quality_sample_t;
|
|
97
|
+
|
|
98
|
+
typedef struct {
|
|
99
|
+
int64_t ts;
|
|
100
|
+
rtc_quality_sample_t pub;
|
|
101
|
+
rtc_quality_sample_t sub;
|
|
102
|
+
} rtc_connection_quality_t;
|
|
103
|
+
typedef void (*rtc_connection_quality_callback)(void* context, const rtc_connection_quality_t* q);
|
|
104
|
+
|
|
105
|
+
typedef struct {
|
|
106
|
+
char uid[64];
|
|
107
|
+
char track_id[64];
|
|
108
|
+
double level;
|
|
109
|
+
} rtc_active_speaker_t;
|
|
110
|
+
typedef void (*rtc_active_speakers_callback)(void* context, int64_t ts, const rtc_active_speaker_t* speakers, int speakers_count);
|
|
111
|
+
|
|
112
|
+
typedef struct {
|
|
113
|
+
char action[64];
|
|
114
|
+
char sid[128];
|
|
115
|
+
char uid[64];
|
|
116
|
+
char channel[128];
|
|
117
|
+
int is_private;
|
|
118
|
+
const char* content_json;
|
|
119
|
+
} rtc_custom_msg_t;
|
|
120
|
+
typedef void (*rtc_custom_msg_callback)(void* context, const rtc_custom_msg_t* msg);
|
|
121
|
+
|
|
122
|
+
typedef void (*rtc_keyframe_request_callback)(void* context, const char* track_id);
|
|
123
|
+
typedef void (*rtc_disconnected_callback)(void* context, int reason, int code, const char* msg);
|
|
124
|
+
|
|
125
|
+
typedef struct {
|
|
126
|
+
char app_id[64];
|
|
127
|
+
char channel[128];
|
|
128
|
+
const char* props;
|
|
129
|
+
int64_t created_at;
|
|
130
|
+
int64_t updated_at;
|
|
131
|
+
} rtc_channel_info_t;
|
|
132
|
+
|
|
133
|
+
void rtc_set_log_level(int level);
|
|
134
|
+
|
|
135
|
+
void* rtc_create();
|
|
136
|
+
void rtc_destroy(void* handle);
|
|
137
|
+
int rtc_join_channel(void* handle, const char* token);
|
|
138
|
+
int rtc_join_channel_sync(void* handle, const char* token, int timeout_ms);
|
|
139
|
+
int rtc_leave_channel(void* handle);
|
|
140
|
+
|
|
141
|
+
void rtc_set_connection_callback(void* handle, rtc_connection_callback callback, void* context);
|
|
142
|
+
void rtc_set_user_event_callback(void* handle, rtc_user_event_callback callback, void* context);
|
|
143
|
+
void rtc_set_track_event_callback(void* handle, rtc_track_event_callback callback, void* context);
|
|
144
|
+
void rtc_set_track_sample_callback(void* handle, rtc_track_sample_callback callback, void* context);
|
|
145
|
+
void rtc_set_layer_switched_callback(void* handle, rtc_layer_switched_callback callback, void* context);
|
|
146
|
+
void rtc_set_connection_quality_callback(void* handle, rtc_connection_quality_callback callback, void* context);
|
|
147
|
+
void rtc_set_active_speakers_callback(void* handle, rtc_active_speakers_callback callback, void* context);
|
|
148
|
+
void rtc_set_custom_msg_callback(void* handle, rtc_custom_msg_callback callback, void* context);
|
|
149
|
+
void rtc_set_disconnected_callback(void* handle, rtc_disconnected_callback callback, void* context);
|
|
150
|
+
|
|
151
|
+
int rtc_get_last_error(void* handle, char* msg_buf, int buf_len);
|
|
152
|
+
|
|
153
|
+
void rtc_set_auto_subscribe(void* handle, int auto_audio, int auto_video);
|
|
154
|
+
|
|
155
|
+
int rtc_subscribe_audio(void* handle, const char* uid, const char* track_id);
|
|
156
|
+
int rtc_subscribe_video(void* handle, const char* uid, const char* track_id);
|
|
157
|
+
int rtc_unsubscribe(void* handle, const char* uid, const char* track_id);
|
|
158
|
+
int rtc_request_key_frame(void* handle, const char* uid, const char* track_id);
|
|
159
|
+
int rtc_switch_layer(void* handle, const char* pub_uid, const char* track_id, const char* target_track_id);
|
|
160
|
+
int rtc_get_connection_quality(void* handle, rtc_connection_quality_t* out);
|
|
161
|
+
|
|
162
|
+
void* rtc_create_local_track(int codec);
|
|
163
|
+
void rtc_destroy_local_track(void* track_handle);
|
|
164
|
+
int rtc_publish_local_track(void* handle, void* track_handle, rtc_publish_options_t* options);
|
|
165
|
+
int rtc_publish_mcu_video_track(void* handle, void* track_handle, rtc_publish_options_t* options);
|
|
166
|
+
int rtc_publish_mcu_audio_track(void* handle, void* track_handle, rtc_publish_options_t* options);
|
|
167
|
+
int rtc_unpublish_local_track(void* handle, void* track_handle);
|
|
168
|
+
int rtc_write_sample(void* track_handle, uint8_t* data, int length, uint32_t samples);
|
|
169
|
+
void rtc_set_keyframe_request_callback(void* track_handle, rtc_keyframe_request_callback callback, void* context);
|
|
170
|
+
|
|
171
|
+
int rtc_get_user_info(void* handle, const char* uid, rtc_user_info_t* user);
|
|
172
|
+
void rtc_free_user_info(rtc_user_info_t* user);
|
|
173
|
+
int rtc_get_users_info(void* handle, rtc_user_info_t** users, int* count);
|
|
174
|
+
void rtc_free_users_info(rtc_user_info_t* users, int count);
|
|
175
|
+
int rtc_get_local_user_info(void* handle, rtc_user_info_t* user);
|
|
176
|
+
|
|
177
|
+
int rtc_get_channel_info(void* handle, rtc_channel_info_t* info);
|
|
178
|
+
void rtc_free_channel_info(rtc_channel_info_t* info);
|
|
179
|
+
"""
|
|
180
|
+
|
|
181
|
+
# 各平台动态库文件名
|
|
182
|
+
_LIB_NAMES = {"darwin": "librtc.dylib", "win32": "librtc.dll"}
|
|
183
|
+
|
|
184
|
+
# 随包分发的库所在目录(由 scripts/fetch_lib.sh 从 Nexus 拉取后放入)
|
|
185
|
+
LIB_DIR = Path(__file__).parent / "_lib"
|
|
186
|
+
|
|
187
|
+
ffi = FFI()
|
|
188
|
+
ffi.cdef(CDEF)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _lib_path() -> str:
|
|
192
|
+
"""定位 librtc:环境变量 SRTC_LIB_PATH 优先(本地调试 rtc-go 用),否则取包内 _lib/。"""
|
|
193
|
+
override = os.getenv("SRTC_LIB_PATH")
|
|
194
|
+
if override:
|
|
195
|
+
return override
|
|
196
|
+
path = LIB_DIR / _LIB_NAMES.get(sys.platform, "librtc.so")
|
|
197
|
+
if not path.exists():
|
|
198
|
+
raise ImportError(
|
|
199
|
+
f"找不到 rtc-go C 库: {path}。"
|
|
200
|
+
"开发环境请先运行 scripts/fetch_lib.sh,或用 SRTC_LIB_PATH 指定库文件路径"
|
|
201
|
+
)
|
|
202
|
+
return str(path)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
# Go c-shared 库在 dlopen 时就启动 Go runtime(含若干后台线程)。fork 只复制调用线程,
|
|
206
|
+
# 子进程里的 Go runtime 是残缺的,一用就死锁/崩溃。所以:
|
|
207
|
+
# 1. 延迟到第一次真正使用(创建 Channel)时才 dlopen,import srtc 本身不加载,
|
|
208
|
+
# gunicorn --preload 这类"先 import 后 fork"的用法不受影响;
|
|
209
|
+
# 2. 父进程已加载后再 fork 出的子进程,禁止继续使用,给出明确报错而不是挂死。
|
|
210
|
+
_lib = None
|
|
211
|
+
_forked_after_load = False
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _mark_forked_child() -> None:
|
|
215
|
+
global _lib, _forked_after_load
|
|
216
|
+
if _lib is not None:
|
|
217
|
+
_lib = None
|
|
218
|
+
_forked_after_load = True
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
if hasattr(os, "register_at_fork"):
|
|
222
|
+
os.register_at_fork(after_in_child=_mark_forked_child)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def get_lib():
|
|
226
|
+
"""返回已加载的 librtc(首次调用时 dlopen)。"""
|
|
227
|
+
global _lib
|
|
228
|
+
if _forked_after_load:
|
|
229
|
+
raise RuntimeError(
|
|
230
|
+
"当前进程是在 SRTC 已加载后 fork 出来的,Go runtime 在子进程中不可用。"
|
|
231
|
+
"请改用 multiprocessing 的 spawn 启动方式,或确保 fork 前没有创建过 Channel"
|
|
232
|
+
)
|
|
233
|
+
if _lib is None:
|
|
234
|
+
_lib = ffi.dlopen(_lib_path())
|
|
235
|
+
return _lib
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def cstr(buf) -> str:
|
|
239
|
+
"""把 C 定长 char 数组 / char* 转成 Python 字符串(NULL 返回空串)。"""
|
|
240
|
+
if buf == ffi.NULL:
|
|
241
|
+
return ""
|
|
242
|
+
return ffi.string(buf).decode("utf-8", errors="replace")
|
srtc/_lib/BUILD_INFO
ADDED
srtc/_lib/librtc.dll
ADDED
|
Binary file
|
srtc/_version.py
ADDED
srtc/audio.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
"""
|
|
2
|
+
音频编解码与发送节奏
|
|
3
|
+
|
|
4
|
+
C 库收发的是 Opus 编码帧,AI 管线要的是 PCM,本模块负责两头的转换:
|
|
5
|
+
|
|
6
|
+
- OpusReceiver(收):Opus → PCM(S16) → 重采样到入会时指定的格式,并按 RTP 时间戳补 DTX 静音。
|
|
7
|
+
- AudioTrack(发):PCM → 重采样到 48k → 切 20ms 帧 → Opus 编码 → 按实时节奏逐帧 rtc_write_sample。
|
|
8
|
+
|
|
9
|
+
两者都用 PyAV(其 wheel 自带 libopus / libswresample,无需系统 ffmpeg)。
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
from collections import deque
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
|
|
18
|
+
import av
|
|
19
|
+
import numpy as np
|
|
20
|
+
|
|
21
|
+
from ._ffi import ffi
|
|
22
|
+
from .logger import logger
|
|
23
|
+
from .types import AudioFrame
|
|
24
|
+
|
|
25
|
+
# Opus 在 WebRTC 里固定 48kHz 时钟(RTP 时间戳按 48000 递增,与实际声道数/采样率无关)
|
|
26
|
+
OPUS_CLOCK_RATE = 48000
|
|
27
|
+
# 发送端每帧 20ms(WebRTC 默认 ptime)
|
|
28
|
+
FRAME_MS = 20
|
|
29
|
+
FRAME_SAMPLES_48K = OPUS_CLOCK_RATE * FRAME_MS // 1000 # 960
|
|
30
|
+
|
|
31
|
+
# 单次最多补多长的 DTX 静音(秒)。正常 DTX 期间对端约每 400ms 发一个 SID 包,空洞远小于此;
|
|
32
|
+
# 超过说明对端静音/断流了很久,只补这么多,避免一次性吐出几分钟的静音撑爆下游。
|
|
33
|
+
MAX_GAP_FILL_SECONDS = 5.0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _layout(channels: int) -> str:
|
|
37
|
+
return "stereo" if channels == 2 else "mono"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class AudioFormat:
|
|
42
|
+
"""PCM 格式:S16LE 交错排列"""
|
|
43
|
+
sample_rate: int = 16000 # AI 场景最常用:大多数 ASR 模型吃 16k 单声道
|
|
44
|
+
channels: int = 1
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class OpusReceiver:
|
|
48
|
+
"""单条远端音频轨道的解码器
|
|
49
|
+
|
|
50
|
+
只在该轨道的 Go 收流线程里被调用(rtc-go 每条轨道一个 goroutine 串行回调),无需加锁。
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(self, uid: str, track_id: str, out: AudioFormat):
|
|
54
|
+
self.uid = uid
|
|
55
|
+
self.track_id = track_id
|
|
56
|
+
self.out = out
|
|
57
|
+
self._decoder = av.CodecContext.create("libopus", "r")
|
|
58
|
+
self._decoder.sample_rate = OPUS_CLOCK_RATE
|
|
59
|
+
self._decoder.layout = "stereo" # 统一按双声道解,单声道流会被复制成两路,再由重采样器下混
|
|
60
|
+
self._resampler = av.AudioResampler(format="s16", layout=_layout(out.channels), rate=out.sample_rate)
|
|
61
|
+
self._next_rtp_ts: int | None = None # 预期下一包的 RTP 时间戳(48k 时钟),None 表示尚未收到首包
|
|
62
|
+
self._pts = 0 # 已输出的采样数(按输出采样率计),即下一帧的 pts
|
|
63
|
+
|
|
64
|
+
def feed(self, data: bytes, rtp_ts: int) -> list[AudioFrame]:
|
|
65
|
+
"""喂入一个 Opus 包,返回解码出的 PCM 帧(可能先带一段补齐的静音帧)"""
|
|
66
|
+
frames: list[AudioFrame] = []
|
|
67
|
+
|
|
68
|
+
# DTX 空洞:本包时间戳比预期靠后,说明中间对端没发包,补上等长静音
|
|
69
|
+
if self._next_rtp_ts is not None:
|
|
70
|
+
gap = (rtp_ts - self._next_rtp_ts) & 0xFFFFFFFF
|
|
71
|
+
if 0 < gap < 0x80000000: # 只处理"往后跳";回绕/乱序的包差值落在另一半区间
|
|
72
|
+
frames.append(self._silence(min(gap / OPUS_CLOCK_RATE, MAX_GAP_FILL_SECONDS)))
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
decoded = self._decoder.decode(av.Packet(data))
|
|
76
|
+
except av.error.FFmpegError as e:
|
|
77
|
+
logger.debug("opus 解码失败 %s/%s: %s", self.uid, self.track_id, e)
|
|
78
|
+
return frames
|
|
79
|
+
|
|
80
|
+
samples_48k = 0
|
|
81
|
+
for f in decoded:
|
|
82
|
+
samples_48k += f.samples
|
|
83
|
+
for rf in self._resampler.resample(f):
|
|
84
|
+
frames.append(self._emit(rf.to_ndarray().tobytes(), rf.samples))
|
|
85
|
+
self._next_rtp_ts = (rtp_ts + (samples_48k or FRAME_SAMPLES_48K)) & 0xFFFFFFFF
|
|
86
|
+
return frames
|
|
87
|
+
|
|
88
|
+
def _silence(self, seconds: float) -> AudioFrame:
|
|
89
|
+
samples = int(seconds * self.out.sample_rate)
|
|
90
|
+
return self._emit(bytes(samples * self.out.channels * 2), samples, is_silence=True)
|
|
91
|
+
|
|
92
|
+
def _emit(self, pcm: bytes, samples: int, is_silence: bool = False) -> AudioFrame:
|
|
93
|
+
frame = AudioFrame(
|
|
94
|
+
uid=self.uid, track_id=self.track_id, pcm=pcm,
|
|
95
|
+
sample_rate=self.out.sample_rate, channels=self.out.channels,
|
|
96
|
+
samples=samples, pts=self._pts, is_silence=is_silence,
|
|
97
|
+
)
|
|
98
|
+
self._pts += samples
|
|
99
|
+
return frame
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class AudioTrack:
|
|
103
|
+
"""本地发布的音频轨道(如 TTS 输出)
|
|
104
|
+
|
|
105
|
+
用法:await track.write(pcm) 写入任意长度的 PCM,SDK 按实时节奏每 20ms 发一帧;
|
|
106
|
+
用户插话时 track.clear() 立即丢弃未播完的部分。
|
|
107
|
+
|
|
108
|
+
为什么要 SDK 控速:TTS 生成速度远快于实时,一口气写进 C 库会在一瞬间发出几秒的 RTP 包,
|
|
109
|
+
对端抖动缓冲区装不下就会丢,听起来是吞字/加速。
|
|
110
|
+
|
|
111
|
+
空闲时持续发送编码后的静音帧:保持 RTP 时间戳与墙钟同步推进,
|
|
112
|
+
否则长时间停发后时间戳不连续,对端会把新包当成迟到包处理。
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
def __init__(self, handle, fmt: AudioFormat, bitrate: int, max_buffer_seconds: float):
|
|
116
|
+
self._handle = handle # rtc_create_local_track 返回的轨道句柄
|
|
117
|
+
self.format = fmt # write() 接收的 PCM 格式
|
|
118
|
+
self._max_buffer = int(max_buffer_seconds * OPUS_CLOCK_RATE) # 缓冲上限(48k 采样数),超过则 write 等待
|
|
119
|
+
|
|
120
|
+
self._encoder = av.CodecContext.create("libopus", "w")
|
|
121
|
+
self._encoder.sample_rate = OPUS_CLOCK_RATE
|
|
122
|
+
self._encoder.layout = _layout(fmt.channels)
|
|
123
|
+
self._encoder.format = "s16"
|
|
124
|
+
self._encoder.bit_rate = bitrate
|
|
125
|
+
self._encoder.options = {"application": "voip", "frame_duration": str(FRAME_MS)}
|
|
126
|
+
self._encoder.open()
|
|
127
|
+
|
|
128
|
+
# 输入 PCM 重采样到 48k(输入本就是 48k 时也走一遍,保证格式统一)
|
|
129
|
+
self._resampler = av.AudioResampler(format="s16", layout=_layout(fmt.channels), rate=OPUS_CLOCK_RATE)
|
|
130
|
+
|
|
131
|
+
self._chunks: deque[bytes] = deque() # 待发送的 48k PCM
|
|
132
|
+
self._buffered = 0 # _chunks 里的总采样数(每声道)
|
|
133
|
+
self._frame_bytes = FRAME_SAMPLES_48K * fmt.channels * 2
|
|
134
|
+
self._silence = bytes(self._frame_bytes)
|
|
135
|
+
self._pts = 0 # 已送编码器的采样数,作为编码帧的 pts
|
|
136
|
+
|
|
137
|
+
self._space = asyncio.Event() # 缓冲有空位(write 背压用)
|
|
138
|
+
self._space.set()
|
|
139
|
+
self._drained = asyncio.Event() # 缓冲已播空(wait_for_playout 用)
|
|
140
|
+
self._drained.set()
|
|
141
|
+
self._task: asyncio.Task | None = None
|
|
142
|
+
self._closed = False
|
|
143
|
+
|
|
144
|
+
# ---------------- 对外接口 ----------------
|
|
145
|
+
|
|
146
|
+
async def write(self, pcm: bytes | np.ndarray) -> None:
|
|
147
|
+
"""写入 PCM(S16LE 交错,格式为创建轨道时指定的 AudioFormat)
|
|
148
|
+
|
|
149
|
+
缓冲超过 max_buffer_seconds 时等待,形成背压。
|
|
150
|
+
"""
|
|
151
|
+
if self._closed:
|
|
152
|
+
raise RuntimeError("轨道已取消发布")
|
|
153
|
+
data = pcm.astype(np.int16, copy=False).tobytes() if isinstance(pcm, np.ndarray) else bytes(pcm)
|
|
154
|
+
unit = self.format.channels * 2
|
|
155
|
+
if len(data) % unit:
|
|
156
|
+
raise ValueError(f"PCM 字节数必须是 {unit} 的整数倍(S16 × {self.format.channels} 声道)")
|
|
157
|
+
if not data:
|
|
158
|
+
return
|
|
159
|
+
|
|
160
|
+
frame = av.AudioFrame.from_ndarray(
|
|
161
|
+
np.frombuffer(data, dtype=np.int16).reshape(1, -1),
|
|
162
|
+
format="s16", layout=_layout(self.format.channels),
|
|
163
|
+
)
|
|
164
|
+
frame.sample_rate = self.format.sample_rate
|
|
165
|
+
for rf in self._resampler.resample(frame):
|
|
166
|
+
self._push(rf.to_ndarray().tobytes())
|
|
167
|
+
|
|
168
|
+
while self._buffered > self._max_buffer and not self._closed:
|
|
169
|
+
self._space.clear()
|
|
170
|
+
await self._space.wait()
|
|
171
|
+
|
|
172
|
+
def clear(self) -> None:
|
|
173
|
+
"""丢弃所有未发送的音频(用户打断 TTS 时调用)"""
|
|
174
|
+
self._chunks.clear()
|
|
175
|
+
self._buffered = 0
|
|
176
|
+
self._space.set()
|
|
177
|
+
self._drained.set()
|
|
178
|
+
|
|
179
|
+
async def wait_for_playout(self) -> None:
|
|
180
|
+
"""等待已写入的音频全部发送完"""
|
|
181
|
+
await self._drained.wait()
|
|
182
|
+
|
|
183
|
+
@property
|
|
184
|
+
def buffered_seconds(self) -> float:
|
|
185
|
+
"""尚未发送的音频时长(秒)"""
|
|
186
|
+
return self._buffered / OPUS_CLOCK_RATE
|
|
187
|
+
|
|
188
|
+
# ---------------- 内部 ----------------
|
|
189
|
+
|
|
190
|
+
def _push(self, pcm48k: bytes) -> None:
|
|
191
|
+
if not pcm48k:
|
|
192
|
+
return
|
|
193
|
+
self._chunks.append(pcm48k)
|
|
194
|
+
self._buffered += len(pcm48k) // (self.format.channels * 2)
|
|
195
|
+
self._drained.clear()
|
|
196
|
+
|
|
197
|
+
def _pop_frame(self) -> bytes:
|
|
198
|
+
"""取出 20ms 的 PCM,不足部分补零;缓冲为空返回静音帧"""
|
|
199
|
+
if not self._chunks:
|
|
200
|
+
return self._silence
|
|
201
|
+
out = bytearray()
|
|
202
|
+
while self._chunks and len(out) < self._frame_bytes:
|
|
203
|
+
chunk = self._chunks.popleft()
|
|
204
|
+
need = self._frame_bytes - len(out)
|
|
205
|
+
out += chunk[:need]
|
|
206
|
+
if len(chunk) > need:
|
|
207
|
+
self._chunks.appendleft(chunk[need:])
|
|
208
|
+
taken = len(out) // (self.format.channels * 2)
|
|
209
|
+
self._buffered = max(0, self._buffered - taken)
|
|
210
|
+
if len(out) < self._frame_bytes:
|
|
211
|
+
out += bytes(self._frame_bytes - len(out))
|
|
212
|
+
if self._buffered <= self._max_buffer:
|
|
213
|
+
self._space.set()
|
|
214
|
+
if not self._chunks:
|
|
215
|
+
self._drained.set()
|
|
216
|
+
return bytes(out)
|
|
217
|
+
|
|
218
|
+
def _encode(self, pcm: bytes) -> list[bytes]:
|
|
219
|
+
frame = av.AudioFrame.from_ndarray(
|
|
220
|
+
np.frombuffer(pcm, dtype=np.int16).reshape(1, -1),
|
|
221
|
+
format="s16", layout=_layout(self.format.channels),
|
|
222
|
+
)
|
|
223
|
+
frame.sample_rate = OPUS_CLOCK_RATE
|
|
224
|
+
frame.pts = self._pts
|
|
225
|
+
self._pts += FRAME_SAMPLES_48K
|
|
226
|
+
return [bytes(p) for p in self._encoder.encode(frame)]
|
|
227
|
+
|
|
228
|
+
def _start(self, lib) -> None:
|
|
229
|
+
self._task = asyncio.get_running_loop().create_task(self._send_loop(lib))
|
|
230
|
+
|
|
231
|
+
async def _send_loop(self, lib) -> None:
|
|
232
|
+
"""每 20ms 发一帧。按绝对时刻排期,sleep 的误差不会累积;落后太多(进程卡顿)则重新对齐,不补发。"""
|
|
233
|
+
loop = asyncio.get_running_loop()
|
|
234
|
+
interval = FRAME_MS / 1000
|
|
235
|
+
deadline = loop.time()
|
|
236
|
+
while not self._closed:
|
|
237
|
+
for packet in self._encode(self._pop_frame()):
|
|
238
|
+
buf = ffi.from_buffer("uint8_t[]", packet)
|
|
239
|
+
if lib.rtc_write_sample(self._handle, buf, len(packet), FRAME_SAMPLES_48K) != 0:
|
|
240
|
+
logger.debug("rtc_write_sample 失败")
|
|
241
|
+
deadline += interval
|
|
242
|
+
delay = deadline - loop.time()
|
|
243
|
+
if delay < -0.2:
|
|
244
|
+
deadline = loop.time()
|
|
245
|
+
delay = 0
|
|
246
|
+
await asyncio.sleep(max(0.0, delay))
|
|
247
|
+
|
|
248
|
+
async def _stop(self) -> None:
|
|
249
|
+
self._closed = True
|
|
250
|
+
self._space.set()
|
|
251
|
+
self._drained.set()
|
|
252
|
+
if self._task:
|
|
253
|
+
self._task.cancel()
|
|
254
|
+
try:
|
|
255
|
+
await self._task
|
|
256
|
+
except asyncio.CancelledError:
|
|
257
|
+
pass
|