yt-chat-components 1.5.7 → 1.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/public/index.html +36 -8
- package/src/YtChatView/chatWidget/chatWindow/audioPlayer/AudioPlayer.tsx +225 -0
- package/src/YtChatView/chatWidget/chatWindow/audioPlayer/PCMAudioPlayer.js +101 -0
- package/src/YtChatView/chatWidget/chatWindow/audioRecorder/index.tsx +237 -0
- package/src/YtChatView/chatWidget/chatWindow/callInterface/index.tsx +1 -1
- package/src/YtChatView/chatWidget/chatWindow/chatMessage/index.tsx +33 -16
- package/src/YtChatView/chatWidget/chatWindow/index.tsx +97 -7
- package/src/YtChatView/chatWidget/chatWindow/inputMobile/index.module.css +134 -0
- package/src/YtChatView/chatWidget/chatWindow/inputMobile/inputMobile.js +170 -0
- package/src/YtChatView/chatWidget/index.tsx +11 -1
- package/src/YtChatView/components/TabSelector/index.jsx +102 -3
- package/src/YtChatView/components/TabSelector/index.module.css +115 -0
- package/src/YtChatView/logoBtn/index.jsx +1 -1
- package/src/YtChatView/mobileChat/index.jsx +1 -0
- package/src/YtChatView/mobileChatV2/index.jsx +712 -0
- package/src/YtChatView/mobileChatV2/index.module.css +269 -0
- package/src/YtChatView/previewDialog/index.jsx +1 -0
- package/src/YtChatView/previewDialogV2/index.jsx +28 -40
- package/src/index.tsx +3 -2
package/package.json
CHANGED
package/public/index.html
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
<script type="text/javascript" src="https://res.wx.qq.com/open/js/jweixin-1.6.0.js"></script>
|
|
8
8
|
</head>
|
|
9
9
|
<!--<body style="width: 100vw; height: 100vh; position: relative; margin: unset ">-->
|
|
10
|
+
<!--<!–0043a683-cef6-4606-b43a-8d1a2f530bf6 a2806443-0780-49fb-a865-e0d1784f2125–>-->
|
|
10
11
|
<!--<yt-chat-->
|
|
11
12
|
<!-- right="100"-->
|
|
12
13
|
<!-- bottom="100"-->
|
|
@@ -14,7 +15,7 @@
|
|
|
14
15
|
<!-- height="50"-->
|
|
15
16
|
<!-- title="菜鸟驿站"-->
|
|
16
17
|
<!-- icon-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/ccit/user/xc//image/ebfaf4da-c1d9-46fb-a0b1-f159e95cffc2_AI招生咨询小助手.png"-->
|
|
17
|
-
<!-- host-url="
|
|
18
|
+
<!-- host-url="http://localhost:7860"-->
|
|
18
19
|
<!-- user-info='{"id": "123", "name": "John Doe", "code":"2020230608" }'-->
|
|
19
20
|
<!-- app-id="c3148c58-c2a2-45a2-a872-bb6e7fc2dd7e"-->
|
|
20
21
|
<!-- is-show-side-left=true-->
|
|
@@ -29,6 +30,8 @@
|
|
|
29
30
|
<!-- logo-position="fixed"-->
|
|
30
31
|
<!-- is-title-side-icon=false-->
|
|
31
32
|
<!-- is-show-upload-button=true-->
|
|
33
|
+
<!-- is_enable_call=true-->
|
|
34
|
+
<!-- is-show-read-icon=true-->
|
|
32
35
|
<!--/>-->
|
|
33
36
|
<!--</body>-->
|
|
34
37
|
|
|
@@ -106,23 +109,48 @@
|
|
|
106
109
|
<!--/>-->
|
|
107
110
|
<!--</body>-->
|
|
108
111
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
host-url="https://ai-api.yuntu.cn"
|
|
112
|
+
<!--<body style="width: 100vw; height: 100vh; margin:0;">-->
|
|
113
|
+
<!--<yt-page-chat-v2-->
|
|
114
|
+
<!-- host-url="https://ai-api.yuntu.cn"-->
|
|
115
|
+
<!-- sign-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/logo.png"-->
|
|
116
|
+
<!-- app-id="5f48c683-38d8-430e-8479-17730a605821"-->
|
|
117
|
+
<!-- box-style='{"height":"100%","minWidth": "1380px","background":"url(https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/system/portal_bg_blue.png) center top / 100% 100% no-repeat"}'-->
|
|
118
|
+
<!-- is-show-side-right=true-->
|
|
119
|
+
<!-- is-show-side-left=true-->
|
|
120
|
+
<!-- dialog-index="999999999"-->
|
|
121
|
+
<!-- agent-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/kai_yuan/byz/xinge.png"-->
|
|
122
|
+
<!-- agent-name="白城医学高等专科学校招生咨询平台"-->
|
|
123
|
+
<!-- logo-width="27px"-->
|
|
124
|
+
<!-- logo-font-size="20px"-->
|
|
125
|
+
<!-- is-title-side-icon=true-->
|
|
126
|
+
<!-- is-show-upload-button=false-->
|
|
127
|
+
<!-- is-show-read-icon=true-->
|
|
128
|
+
<!-- drop-man-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/ai/ai_man01.png"-->
|
|
129
|
+
<!-- asset-map='{"sendMessageUrl":"https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/sendMessage.png","stopMessageUrl":"https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/stopMessage.png","speakUrl":"https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/speak.png"}'-->
|
|
130
|
+
<!--/>-->
|
|
131
|
+
<!--</body>-->
|
|
132
|
+
|
|
133
|
+
<body style="width: 100vw; height: 100vh; margin:0 ">
|
|
134
|
+
<yt-page-chat-mobile-v2
|
|
135
|
+
host-url="http://192.168.110.135:7860"
|
|
112
136
|
sign-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/logo.png"
|
|
113
137
|
app-id="5f48c683-38d8-430e-8479-17730a605821"
|
|
114
|
-
|
|
138
|
+
agent-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/kai_yuan/byz/xinge.png"
|
|
139
|
+
box-style='{"height":"100%"}'
|
|
115
140
|
is-show-side-right=true
|
|
116
141
|
is-show-side-left=true
|
|
117
142
|
dialog-index="999999999"
|
|
118
|
-
agent-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/kai_yuan/byz/xinge.png"
|
|
119
143
|
agent-name="白城医学高等专科学校招生咨询平台"
|
|
120
|
-
|
|
121
|
-
logo-
|
|
144
|
+
agent-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/ccit/user/xc//image/ebfaf4da-c1d9-46fb-a0b1-f159e95cffc2_AI招生咨询小助手.png"
|
|
145
|
+
logo-width="42px"
|
|
146
|
+
logo-font-size="26px"
|
|
122
147
|
is-title-side-icon=true
|
|
123
148
|
is-show-upload-button=false
|
|
149
|
+
is-show-read-icon=true
|
|
124
150
|
drop-man-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/ai/ai_man01.png"
|
|
125
151
|
asset-map='{"sendMessageUrl":"https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/sendMessage.png","stopMessageUrl":"https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/stopMessage.png","speakUrl":"https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/speak.png"}'
|
|
152
|
+
header-name="白城医学高等专科学校招生咨询平台"
|
|
153
|
+
is-show-mobile-input-area=true
|
|
126
154
|
/>
|
|
127
155
|
</body>
|
|
128
156
|
</html>
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
// @ts-ignore
|
|
2
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
3
|
+
import PCMAudioPlayer from "./PCMAudioPlayer";
|
|
4
|
+
|
|
5
|
+
export class AudioPlayer {
|
|
6
|
+
private websocket: WebSocket | null = null;
|
|
7
|
+
private appkey: string;
|
|
8
|
+
private token: string;
|
|
9
|
+
private isSendSentenceEnd : boolean = false;
|
|
10
|
+
private pcmAudioPlayer: PCMAudioPlayer | null = null;
|
|
11
|
+
private voice : string = 'longxiaochun'; // longxiaochun zhixiaoxia
|
|
12
|
+
|
|
13
|
+
// 回调函数
|
|
14
|
+
onError?: (error: any) => void;
|
|
15
|
+
onReady?: () => void;
|
|
16
|
+
onComplete?: () => void;
|
|
17
|
+
|
|
18
|
+
private taskId: string = '';
|
|
19
|
+
private sentences: string[] = [];
|
|
20
|
+
private isPlaying: boolean = false;
|
|
21
|
+
|
|
22
|
+
constructor(voice: string) {
|
|
23
|
+
// this.appkey = "9S7HL8z0fT80giNH";
|
|
24
|
+
// this.token = 'd20cf27fac634fee88d5d54a12cc36a6';
|
|
25
|
+
|
|
26
|
+
this.appkey = 'DhyCL7mwzachpGjL';
|
|
27
|
+
this.token = 'e3e3d121a15f41b198117a35ad1f33de';
|
|
28
|
+
|
|
29
|
+
this.taskId = uuidv4().replaceAll('-', '');
|
|
30
|
+
if (voice){
|
|
31
|
+
this.voice = voice;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
this.pcmAudioPlayer = new PCMAudioPlayer(16000);
|
|
35
|
+
this.pcmAudioPlayer.onQueueEnd = () => {
|
|
36
|
+
this.isPlaying = false;
|
|
37
|
+
if (this.onComplete){
|
|
38
|
+
this.onComplete();
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
this.connect();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
connect() {
|
|
45
|
+
// cosyvoice 流式语言合成 2元、万字
|
|
46
|
+
const socketUrl = `wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1?token=${this.token}`;
|
|
47
|
+
// 普通语言合成 1元、万字
|
|
48
|
+
// const socketUrl = `wss://nls-gateway.cn-shanghai.aliyuncs.com/ws/v1?token=${this.token}`;
|
|
49
|
+
this.websocket = new WebSocket(socketUrl);
|
|
50
|
+
this.websocket.binaryType = 'arraybuffer';
|
|
51
|
+
this.websocket.onopen = () => {
|
|
52
|
+
console.log('WebSocket 已连接');
|
|
53
|
+
this.sendStartSynthesis();
|
|
54
|
+
this.pcmAudioPlayer?.connect();
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
this.websocket.onmessage = this.handleMessage.bind(this);
|
|
58
|
+
this.websocket.onerror = this.handleError.bind(this);
|
|
59
|
+
this.websocket.onclose = this.handleClose.bind(this);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
disconnect() {
|
|
63
|
+
if (this.websocket) {
|
|
64
|
+
this.websocket.close();
|
|
65
|
+
this.websocket = null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// 建立链接指令
|
|
70
|
+
sendStartSynthesis() {
|
|
71
|
+
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
|
|
72
|
+
console.warn('WebSocket not connected.');
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
const messageId = uuidv4().replaceAll('-', '');
|
|
76
|
+
const startMessage = {
|
|
77
|
+
header: {
|
|
78
|
+
namespace: "FlowingSpeechSynthesizer",
|
|
79
|
+
name: "StartSynthesis",
|
|
80
|
+
task_id: this.taskId,
|
|
81
|
+
message_id: messageId,
|
|
82
|
+
appkey: this.appkey
|
|
83
|
+
},
|
|
84
|
+
payload: {
|
|
85
|
+
enable_subtitle: true,
|
|
86
|
+
format: 'PCM',
|
|
87
|
+
pitch_rate: 0, // 音调
|
|
88
|
+
platform: "javascript",
|
|
89
|
+
sample_rate: "16000",// 输出格式
|
|
90
|
+
speech_rate: 0, // 语速
|
|
91
|
+
voice: this.voice, // 声音模型
|
|
92
|
+
volume: 100 // 音量
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
console.log("发送初始连接消息")
|
|
96
|
+
this.websocket.send(JSON.stringify(startMessage));
|
|
97
|
+
}
|
|
98
|
+
// 终止合成指令
|
|
99
|
+
sendStopSynthesis() {
|
|
100
|
+
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
|
|
101
|
+
console.warn('WebSocket not connected.');
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
const messageId = uuidv4().replaceAll('-', '');
|
|
105
|
+
const stopMessage = {
|
|
106
|
+
header: {
|
|
107
|
+
namespace: "FlowingSpeechSynthesizer",
|
|
108
|
+
name: "StopSynthesis",
|
|
109
|
+
task_id: this.taskId,
|
|
110
|
+
message_id: messageId,
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
this.websocket.send(JSON.stringify(stopMessage));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
sendRunSynthesis(text: string) {
|
|
117
|
+
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
|
|
118
|
+
console.warn('WebSocket not connected.');
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
this.sentences = text.match(/[^。?!\n]+[。?!]?/g) || [text];
|
|
122
|
+
this.sendNextSentence();
|
|
123
|
+
}
|
|
124
|
+
sendNextSentence() {
|
|
125
|
+
if (this.sentences.length === 0 || this.isPlaying) return;
|
|
126
|
+
|
|
127
|
+
const sentence = this.sentences.shift();
|
|
128
|
+
if (!sentence) {
|
|
129
|
+
this.isSendSentenceEnd = true;
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
// console.log("发送句子:", sentence);
|
|
133
|
+
|
|
134
|
+
const message = {
|
|
135
|
+
header: {
|
|
136
|
+
namespace: "FlowingSpeechSynthesizer",
|
|
137
|
+
name: "RunSynthesis",
|
|
138
|
+
task_id: this.taskId,
|
|
139
|
+
message_id: uuidv4().replaceAll('-', ''),
|
|
140
|
+
appkey: this.appkey
|
|
141
|
+
},
|
|
142
|
+
payload: {
|
|
143
|
+
text: sentence
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
// @ts-ignore
|
|
148
|
+
this.websocket.send(JSON.stringify(message));
|
|
149
|
+
|
|
150
|
+
this.sendNextSentence();
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
stop() {
|
|
154
|
+
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
|
155
|
+
const endMessage = {
|
|
156
|
+
header: {
|
|
157
|
+
namespace: "FlowingSpeechSynthesizer",
|
|
158
|
+
name: "StopSynthesis",
|
|
159
|
+
task_id: this.taskId,
|
|
160
|
+
message_id: uuidv4().replaceAll('-', ''),
|
|
161
|
+
appkey: this.appkey
|
|
162
|
+
},
|
|
163
|
+
payload: {}
|
|
164
|
+
};
|
|
165
|
+
this.websocket.send(JSON.stringify(endMessage));
|
|
166
|
+
}
|
|
167
|
+
if (this.onComplete){
|
|
168
|
+
this.onComplete();
|
|
169
|
+
}
|
|
170
|
+
this.pcmAudioPlayer?.stop();
|
|
171
|
+
this.disconnect();
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
private handleMessage(event: MessageEvent) {
|
|
175
|
+
try {
|
|
176
|
+
const data = event.data;
|
|
177
|
+
|
|
178
|
+
if (typeof data === 'string') {
|
|
179
|
+
const message = JSON.parse(data);
|
|
180
|
+
const namespace = message.header?.namespace;
|
|
181
|
+
const name = message.header?.name;
|
|
182
|
+
// console.log("收到消息:", message);
|
|
183
|
+
// console.log("消息类型:", namespace, name);
|
|
184
|
+
|
|
185
|
+
switch (`${namespace}.${name}`) {
|
|
186
|
+
case "FlowingSpeechSynthesizer.SynthesisStarted":
|
|
187
|
+
console.log("语音合成开始");
|
|
188
|
+
if (this.onReady) this.onReady();
|
|
189
|
+
break;
|
|
190
|
+
case "FlowingSpeechSynthesizer.SentenceEnd":
|
|
191
|
+
if (this.isSendSentenceEnd){
|
|
192
|
+
}
|
|
193
|
+
break;
|
|
194
|
+
case "Default.TaskFailed":
|
|
195
|
+
console.error("任务失败:", message.header.status_text);
|
|
196
|
+
if (this.onError) {
|
|
197
|
+
this.onError(message.header.status_text);
|
|
198
|
+
}
|
|
199
|
+
this.disconnect();
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
} else if (data instanceof Blob || data instanceof ArrayBuffer) {
|
|
204
|
+
// 处理音频数据
|
|
205
|
+
console.log('收到音频数据:', data);
|
|
206
|
+
this.pcmAudioPlayer?.pushPCM(data);
|
|
207
|
+
}
|
|
208
|
+
} catch (e) {
|
|
209
|
+
console.error("解析消息失败:", e);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
private handleError(error: Event) {
|
|
214
|
+
console.error("WebSocket 错误:", error);
|
|
215
|
+
if (this.onError) {
|
|
216
|
+
this.onError(error);
|
|
217
|
+
}
|
|
218
|
+
this.disconnect();
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
private handleClose() {
|
|
222
|
+
console.log("WebSocket 已断开");
|
|
223
|
+
this.isPlaying = false;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
class PCMAudioPlayer {
|
|
2
|
+
constructor(sampleRate) {
|
|
3
|
+
this.sampleRate = sampleRate;
|
|
4
|
+
this.audioContext = null;
|
|
5
|
+
this.audioQueue = [];
|
|
6
|
+
this.isPlaying = false;
|
|
7
|
+
this.currentSource = null;
|
|
8
|
+
const bufferThreshold = 2;
|
|
9
|
+
this.onQueueEnd = null;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
connect() {
|
|
13
|
+
if (!this.audioContext) {
|
|
14
|
+
this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
pushPCM(arrayBuffer) {
|
|
19
|
+
this.audioQueue.push(arrayBuffer);
|
|
20
|
+
this._playNextAudio();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* 将arrayBuffer转为audioBuffer
|
|
25
|
+
*/
|
|
26
|
+
_bufferPCMData(pcmData) {
|
|
27
|
+
const sampleRate = this.sampleRate; // 设置为 PCM 数据的采样率
|
|
28
|
+
const length = pcmData.byteLength / 2; // 假设 PCM 数据为 16 位,需除以 2
|
|
29
|
+
const audioBuffer = this.audioContext.createBuffer(1, length, sampleRate);
|
|
30
|
+
const channelData = audioBuffer.getChannelData(0);
|
|
31
|
+
const int16Array = new Int16Array(pcmData); // 将 PCM 数据转换为 Int16Array
|
|
32
|
+
|
|
33
|
+
for (let i = 0; i < length; i++) {
|
|
34
|
+
// 将 16 位 PCM 转换为浮点数 (-1.0 到 1.0)
|
|
35
|
+
channelData[i] = int16Array[i] / 32768; // 16 位数据转换范围
|
|
36
|
+
}
|
|
37
|
+
let audioLength = length/sampleRate*1000;
|
|
38
|
+
console.log(`prepare audio: ${length} samples, ${audioLength} ms`)
|
|
39
|
+
|
|
40
|
+
return audioBuffer;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async _playAudio(arrayBuffer) {
|
|
44
|
+
if (this.audioContext.state === 'suspended') {
|
|
45
|
+
await this.audioContext.resume();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const audioBuffer = this._bufferPCMData(arrayBuffer);
|
|
49
|
+
|
|
50
|
+
this.currentSource = this.audioContext.createBufferSource();
|
|
51
|
+
this.currentSource.buffer = audioBuffer;
|
|
52
|
+
this.currentSource.connect(this.audioContext.destination);
|
|
53
|
+
|
|
54
|
+
this.currentSource.onended = () => {
|
|
55
|
+
console.log('Audio playback ended.');
|
|
56
|
+
this.isPlaying = false;
|
|
57
|
+
this.currentSource = null;
|
|
58
|
+
this._playNextAudio(); // Play the next audio in the queue
|
|
59
|
+
};
|
|
60
|
+
this.currentSource.start();
|
|
61
|
+
this.isPlaying = true;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
_playNextAudio() {
|
|
65
|
+
if (this.audioQueue.length > 0 && !this.isPlaying) {
|
|
66
|
+
// 计算总的字节长度
|
|
67
|
+
const totalLength = this.audioQueue.reduce((acc, buffer) => acc + buffer.byteLength, 0);
|
|
68
|
+
const combinedBuffer = new Uint8Array(totalLength);
|
|
69
|
+
let offset = 0;
|
|
70
|
+
|
|
71
|
+
// 将所有 audioQueue 中的 buffer 拼接到一个新的 Uint8Array 中
|
|
72
|
+
for (const buffer of this.audioQueue) {
|
|
73
|
+
combinedBuffer.set(new Uint8Array(buffer), offset);
|
|
74
|
+
offset += buffer.byteLength;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// 清空 audioQueue,因为我们已经拼接完所有数据
|
|
78
|
+
this.audioQueue = [];
|
|
79
|
+
// 发送拼接的 audio 数据给 playAudio
|
|
80
|
+
this._playAudio(combinedBuffer.buffer);
|
|
81
|
+
}else if (!this.isPlaying && this.audioQueue.length === 0) {
|
|
82
|
+
// 所有音频播放完毕,触发 onQueueEnd
|
|
83
|
+
console.log("音频队列已全部播放完毕");
|
|
84
|
+
if (typeof this.onQueueEnd === 'function') {
|
|
85
|
+
this.onQueueEnd();
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
stop() {
|
|
90
|
+
if (this.currentSource) {
|
|
91
|
+
this.currentSource.stop(); // 停止当前音频播放
|
|
92
|
+
this.currentSource = null; // 清除音频源引用
|
|
93
|
+
this.isPlaying = false; // 更新播放状态
|
|
94
|
+
}
|
|
95
|
+
this.audioQueue = []; // 清空音频队列
|
|
96
|
+
console.log('Playback stopped and queue cleared.');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export default PCMAudioPlayer;
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
// @ts-ignore
|
|
2
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
3
|
+
|
|
4
|
+
export class AudioRecorder {
|
|
5
|
+
// 状态与资源
|
|
6
|
+
websocket: WebSocket | null;
|
|
7
|
+
audioContext: AudioContext | null;
|
|
8
|
+
scriptProcessor: ScriptProcessorNode | null;
|
|
9
|
+
audioInput: MediaStreamAudioSourceNode | null;
|
|
10
|
+
audioStream: MediaStream | null;
|
|
11
|
+
recordingTimeout: any = null;
|
|
12
|
+
timeoutSeconds: number = 60;
|
|
13
|
+
appkey: string;
|
|
14
|
+
token: string;
|
|
15
|
+
onTextUpdate: ((text: string) => void) | null = null;
|
|
16
|
+
onRecordError: ((error: any) => void) | null = null;
|
|
17
|
+
onRecordTimeout: (() => void) | null = null;
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
// 缓存识别结果
|
|
21
|
+
transcriptionCache: {
|
|
22
|
+
[taskId: string]: {
|
|
23
|
+
[index: number]: {
|
|
24
|
+
result: string;
|
|
25
|
+
final: boolean;
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
} = {};
|
|
29
|
+
|
|
30
|
+
finalResults: string[] = [];
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
constructor(timeoutSeconds: number) {
|
|
34
|
+
this.websocket = null;
|
|
35
|
+
this.audioContext = null;
|
|
36
|
+
this.scriptProcessor = null;
|
|
37
|
+
this.audioInput = null;
|
|
38
|
+
this.audioStream = null;
|
|
39
|
+
if (timeoutSeconds){
|
|
40
|
+
this.timeoutSeconds = timeoutSeconds;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
this.appkey = 'DhyCL7mwzachpGjL';
|
|
44
|
+
this.token = 'e3e3d121a15f41b198117a35ad1f33de';
|
|
45
|
+
|
|
46
|
+
this.bindEventHandlers();
|
|
47
|
+
console.log('AudioRecorder init')
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
bindEventHandlers() {
|
|
51
|
+
// 绑定 this 到所有回调函数中
|
|
52
|
+
this.connectWebSocket = this.connectWebSocket.bind(this);
|
|
53
|
+
this.disconnectWebSocket = this.disconnectWebSocket.bind(this);
|
|
54
|
+
this.onOpen = this.onOpen.bind(this);
|
|
55
|
+
this.openRecorder = this.openRecorder.bind(this);
|
|
56
|
+
this.onMessage = this.onMessage.bind(this);
|
|
57
|
+
this.onError = this.onError.bind(this);
|
|
58
|
+
this.onClose = this.onClose.bind(this);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
connectWebSocket() {
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
const socketUrl = `wss://nls-gateway-cn-shanghai.aliyuncs.com/ws/v1?token=${this.token}`;
|
|
65
|
+
|
|
66
|
+
this.websocket = new WebSocket(socketUrl);
|
|
67
|
+
|
|
68
|
+
// 移除旧事件监听器
|
|
69
|
+
this.cleanupWebSocketListeners();
|
|
70
|
+
|
|
71
|
+
this.websocket.onopen = this.onOpen;
|
|
72
|
+
this.websocket.onmessage = this.onMessage;
|
|
73
|
+
// @ts-ignore
|
|
74
|
+
this.websocket.onerror = this.onError;
|
|
75
|
+
this.websocket.onclose = this.onClose;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
cleanupWebSocketListeners() {
|
|
79
|
+
if (this.websocket) {
|
|
80
|
+
this.websocket.onopen = null;
|
|
81
|
+
this.websocket.onmessage = null;
|
|
82
|
+
this.websocket.onerror = null;
|
|
83
|
+
this.websocket.onclose = null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
onOpen() {
|
|
88
|
+
console.log({message: '连接到 WebSocket 服务器'});
|
|
89
|
+
|
|
90
|
+
this.openRecorder().then(r => {
|
|
91
|
+
const startTranscriptionMessage = {
|
|
92
|
+
header: {
|
|
93
|
+
appkey: this.appkey,
|
|
94
|
+
namespace: "SpeechTranscriber",
|
|
95
|
+
name: "StartTranscription",
|
|
96
|
+
task_id: uuidv4().replaceAll('-', ''),
|
|
97
|
+
message_id: uuidv4().replaceAll('-', '')
|
|
98
|
+
},
|
|
99
|
+
payload: {
|
|
100
|
+
format: "pcm",
|
|
101
|
+
sample_rate: 16000,
|
|
102
|
+
enable_intermediate_result: true,
|
|
103
|
+
enable_punctuation_prediction: true,
|
|
104
|
+
enable_inverse_text_normalization: true
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
// @ts-ignore
|
|
109
|
+
this.websocket.send(JSON.stringify(startTranscriptionMessage));
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
onMessage(event: { data: string; }) {
|
|
114
|
+
console.log({message: '服务端: ' + event.data});
|
|
115
|
+
|
|
116
|
+
try {
|
|
117
|
+
const message = JSON.parse(event.data);
|
|
118
|
+
const taskId = message.header?.task_id;
|
|
119
|
+
const index = message.payload?.index;
|
|
120
|
+
const name = message.header?.name;
|
|
121
|
+
const result = message.payload?.result;
|
|
122
|
+
|
|
123
|
+
if (!taskId || index === undefined || !result) return;
|
|
124
|
+
|
|
125
|
+
// 初始化缓存
|
|
126
|
+
if (!this.transcriptionCache[taskId]) {
|
|
127
|
+
this.transcriptionCache[taskId] = {};
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// 缓存当前识别结果
|
|
131
|
+
this.transcriptionCache[taskId][index] = {
|
|
132
|
+
result,
|
|
133
|
+
final: name === "SentenceEnd"
|
|
134
|
+
};
|
|
135
|
+
// 如果是最终结果,则合并所有已排序的句子
|
|
136
|
+
if (message.header?.name === "SentenceEnd" || message.header?.name === "TranscriptionResultChanged") {
|
|
137
|
+
const sentences = Object.keys(this.transcriptionCache[taskId])
|
|
138
|
+
.map(Number)
|
|
139
|
+
.sort((a, b) => a - b)
|
|
140
|
+
.map(i => this.transcriptionCache[taskId][i].result);
|
|
141
|
+
|
|
142
|
+
const fullText = sentences.join(" ");
|
|
143
|
+
this.finalResults.push(fullText);
|
|
144
|
+
// 调用回调函数通知外部更新 UI
|
|
145
|
+
if (this.onTextUpdate) {
|
|
146
|
+
this.onTextUpdate(fullText);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
} catch (e) {
|
|
150
|
+
console.log({message: '解析消息失败: ' + e});
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
onError(event: string) {
|
|
155
|
+
console.log('WebSocket 错误: ' + event);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
onClose() {
|
|
159
|
+
console.log('与 WebSocket 服务器断开');
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
disconnectWebSocket() {
|
|
163
|
+
if (this.websocket) {
|
|
164
|
+
this.websocket.close();
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async openRecorder() {
|
|
169
|
+
try {
|
|
170
|
+
this.audioStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
171
|
+
this.audioContext = new (window.AudioContext || (window as any).webkitAudioContext)({
|
|
172
|
+
sampleRate: 16000
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
this.audioInput = this.audioContext.createMediaStreamSource(this.audioStream);
|
|
176
|
+
this.scriptProcessor = this.audioContext.createScriptProcessor(2048, 1, 1);
|
|
177
|
+
|
|
178
|
+
this.scriptProcessor.onaudioprocess = (event) => {
|
|
179
|
+
const inputData = event.inputBuffer.getChannelData(0);
|
|
180
|
+
const inputData16 = new Int16Array(inputData.length);
|
|
181
|
+
for (let i = 0; i < inputData.length; ++i) {
|
|
182
|
+
inputData16[i] = Math.max(-1, Math.min(1, inputData[i])) * 0x7FFF; // PCM 16-bit
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
|
186
|
+
this.websocket.send(inputData16.buffer);
|
|
187
|
+
console.log('发送音频数据块');
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
this.audioInput.connect(this.scriptProcessor);
|
|
192
|
+
this.scriptProcessor.connect(this.audioContext.destination);
|
|
193
|
+
} catch (e) {
|
|
194
|
+
console.log('录音失败: ' + e);
|
|
195
|
+
if (this.onRecordError){
|
|
196
|
+
this.onRecordError(e);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async startRecorder() {
|
|
202
|
+
this.connectWebSocket();
|
|
203
|
+
|
|
204
|
+
// 设置 60 秒超时自动停止
|
|
205
|
+
this.recordingTimeout = setTimeout(() => {
|
|
206
|
+
console.log('录音已达 60 秒,自动停止');
|
|
207
|
+
this.stopRecorder();
|
|
208
|
+
if (this.onRecordTimeout){
|
|
209
|
+
this.onRecordTimeout();
|
|
210
|
+
}
|
|
211
|
+
}, this.timeoutSeconds * 1000);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
stopRecorder() {
|
|
215
|
+
if (this.recordingTimeout) {
|
|
216
|
+
clearTimeout(this.recordingTimeout);
|
|
217
|
+
this.recordingTimeout = null;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if (this.scriptProcessor) {
|
|
221
|
+
this.scriptProcessor.disconnect();
|
|
222
|
+
this.scriptProcessor = null;
|
|
223
|
+
}
|
|
224
|
+
if (this.audioInput) {
|
|
225
|
+
this.audioInput.disconnect();
|
|
226
|
+
this.audioInput = null;
|
|
227
|
+
}
|
|
228
|
+
if (this.audioStream) {
|
|
229
|
+
this.audioStream.getTracks().forEach(track => track.stop());
|
|
230
|
+
this.audioStream = null;
|
|
231
|
+
}
|
|
232
|
+
if (this.audioContext) {
|
|
233
|
+
this.audioContext.close()?.then(r => {});
|
|
234
|
+
this.audioContext = null;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
@@ -420,7 +420,7 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
|
|
|
420
420
|
// msgIdRef.current = null
|
|
421
421
|
if (message.textResponse && addMessage) {
|
|
422
422
|
// 流式插入消息到聊天记录
|
|
423
|
-
|
|
423
|
+
console.log(message.textResponse)
|
|
424
424
|
// handleMessageContent('token',{
|
|
425
425
|
// chunk: message.textResponse,
|
|
426
426
|
// name:'agent',
|