pi2web 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ANDROID_APP_FEATURES.md +256 -0
- package/LICENSE +21 -0
- package/README.md +194 -0
- package/VOICE_SERVER_SETUP.md +393 -0
- package/dist/agents.js +123 -0
- package/dist/auth.js +31 -0
- package/dist/mail-notifier.js +132 -0
- package/dist/main.js +90 -0
- package/dist/metadata-store.js +139 -0
- package/dist/sdk-backend.js +85 -0
- package/dist/server.js +419 -0
- package/dist/terminals.js +70 -0
- package/dist/voice.js +322 -0
- package/dist/workspaces.js +60 -0
- package/package.json +79 -0
- package/packages/edge-tts-server/README.md +68 -0
- package/packages/edge-tts-server/pyproject.toml +19 -0
- package/packages/edge-tts-server/src/remote_pi_edge_tts/__init__.py +3 -0
- package/packages/edge-tts-server/src/remote_pi_edge_tts/__main__.py +77 -0
- package/packages/protocol/openapi.yaml +38 -0
- package/packages/protocol/websocket.schema.json +17 -0
- package/web/app.js +544 -0
- package/web/icon.svg +19 -0
- package/web/index.html +84 -0
- package/web/styles.css +29 -0
package/dist/voice.js
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
const punctuation = new Set(['。', '!', '?', '!', '?', '.']);
|
|
4
|
+
const closing = new Set(['”', '’', '"', '\'', ')', ')', '】', ']']);
|
|
5
|
+
const endpoint = (base, path) => `${base.replace(/\/$/, '')}${base.replace(/\/$/, '').endsWith('/v1') ? '' : '/v1'}${path}`;
|
|
6
|
+
const authHeaders = (key) => key ? { authorization: `Bearer ${key}` } : {};
|
|
7
|
+
const abortError = (error) => error instanceof Error && error.name === 'AbortError';
|
|
8
|
+
const timeoutError = (error) => error instanceof Error && error.name === 'TimeoutError';
|
|
9
|
+
/** Combines caller cancellation with a hard timeout so a stalled speech server cannot block the announcement queue forever. */
|
|
10
|
+
const deadline = (ms, signal) => signal ? AbortSignal.any([signal, AbortSignal.timeout(ms)]) : AbortSignal.timeout(ms);
|
|
11
|
+
export function assistantText(message) {
|
|
12
|
+
if (typeof message === 'string')
|
|
13
|
+
return message.trim() || undefined;
|
|
14
|
+
if (!message || typeof message !== 'object')
|
|
15
|
+
return;
|
|
16
|
+
const value = message;
|
|
17
|
+
if (value.role && value.role !== 'assistant')
|
|
18
|
+
return;
|
|
19
|
+
if (typeof value.content === 'string')
|
|
20
|
+
return value.content.trim() || undefined;
|
|
21
|
+
if (Array.isArray(value.content)) {
|
|
22
|
+
const text = value.content.filter((part) => part?.type === 'text' && typeof part.text === 'string').map((part) => part.text).join('\n').trim();
|
|
23
|
+
return text || undefined;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export function prepareSummaryInput(text, maxChars = 32000) {
|
|
27
|
+
let cleaned = text
|
|
28
|
+
.replace(/```[\s\S]*?```/g, ' [代码已省略] ')
|
|
29
|
+
.replace(/<details[\s\S]*?<\/details>/gi, ' [详细内容已省略] ')
|
|
30
|
+
.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
|
|
31
|
+
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
|
32
|
+
.replace(/https?:\/\/\S+/g, '链接')
|
|
33
|
+
.replace(/^\s{0,3}#{1,6}\s+/gm, '')
|
|
34
|
+
.replace(/^\s*[-*+]\s+/gm, '')
|
|
35
|
+
.replace(/`([^`]+)`/g, '$1')
|
|
36
|
+
.replace(/[ \t]+/g, ' ')
|
|
37
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
38
|
+
.trim();
|
|
39
|
+
if (cleaned.length <= maxChars)
|
|
40
|
+
return cleaned;
|
|
41
|
+
const head = Math.floor(maxChars * 0.375), tail = maxChars - head;
|
|
42
|
+
cleaned = `${cleaned.slice(0, head)}\n\n[中间内容已省略]\n\n${cleaned.slice(-tail)}`;
|
|
43
|
+
return cleaned;
|
|
44
|
+
}
|
|
45
|
+
function clipSummaryField(text, maxChars) {
|
|
46
|
+
const normalized = text.replace(/[ \t]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
|
|
47
|
+
if (normalized.length <= maxChars)
|
|
48
|
+
return normalized;
|
|
49
|
+
const head = Math.floor(maxChars * 0.375), tail = maxChars - head;
|
|
50
|
+
return `${normalized.slice(0, head)}\n\n[中间内容已省略]\n\n${normalized.slice(-tail)}`;
|
|
51
|
+
}
|
|
52
|
+
export function prepareSummaryContext(input, maxChars = 32000) {
|
|
53
|
+
const promptBudget = Math.max(1, Math.floor(maxChars * 0.4)), outputBudget = Math.max(1, maxChars - promptBudget);
|
|
54
|
+
return JSON.stringify({
|
|
55
|
+
userPrompt: clipSummaryField(input.userPrompt, promptBudget),
|
|
56
|
+
piFinalOutput: prepareSummaryInput(input.finalOutput, outputBudget),
|
|
57
|
+
sessionNeedsName: !input.sessionName?.trim()
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
export function parseSummaryResult(text) {
|
|
61
|
+
const trimmed = text.trim(), candidate = trimmed.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
|
|
62
|
+
let value;
|
|
63
|
+
try {
|
|
64
|
+
value = JSON.parse(candidate);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
const start = candidate.indexOf('{'), end = candidate.lastIndexOf('}');
|
|
68
|
+
if (start < 0 || end <= start)
|
|
69
|
+
throw new Error('Summary endpoint returned invalid JSON');
|
|
70
|
+
try {
|
|
71
|
+
value = JSON.parse(candidate.slice(start, end + 1));
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
throw new Error('Summary endpoint returned invalid JSON');
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
const summary = typeof value?.summary === 'string' ? value.summary.trim() : '';
|
|
78
|
+
if (!summary)
|
|
79
|
+
throw new Error('Summary endpoint returned no summary');
|
|
80
|
+
const rawName = typeof value?.sessionName === 'string' ? value.sessionName.trim() : '';
|
|
81
|
+
const sessionName = rawName.replace(/[\r\n\t]+/g, ' ').replace(/\s{2,}/g, ' ').slice(0, 60).trim();
|
|
82
|
+
return { summary, ...(sessionName ? { sessionName } : {}) };
|
|
83
|
+
}
|
|
84
|
+
export function formatSpokenSummary(summary, sessionName, language = 'zh-CN') {
|
|
85
|
+
const name = sessionName?.replace(/[\r\n\t]+/g, ' ').replace(/\s{2,}/g, ' ').slice(0, 60).trim();
|
|
86
|
+
if (!name)
|
|
87
|
+
return summary;
|
|
88
|
+
return language.toLowerCase().startsWith('zh') ? `会话${name}已完成:${summary}` : `Session ${name} completed: ${summary}`;
|
|
89
|
+
}
|
|
90
|
+
export class SentenceChunker {
|
|
91
|
+
maxChars;
|
|
92
|
+
buffer = '';
|
|
93
|
+
constructor(maxChars = 100) {
|
|
94
|
+
this.maxChars = maxChars;
|
|
95
|
+
}
|
|
96
|
+
add(delta) { this.buffer += delta; return this.consume(false); }
|
|
97
|
+
close() { return this.consume(true); }
|
|
98
|
+
consume(final) {
|
|
99
|
+
const chunks = [];
|
|
100
|
+
while (this.buffer) {
|
|
101
|
+
let cut = -1;
|
|
102
|
+
for (let i = 0; i < this.buffer.length; i++)
|
|
103
|
+
if (punctuation.has(this.buffer[i]) && i >= 3) {
|
|
104
|
+
cut = i + 1;
|
|
105
|
+
while (cut < this.buffer.length && closing.has(this.buffer[cut]))
|
|
106
|
+
cut++;
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
if (cut < 0 && this.buffer.length >= this.maxChars) {
|
|
110
|
+
const range = this.buffer.slice(0, this.maxChars + 1), fallback = Math.max(range.lastIndexOf(','), range.lastIndexOf(','), range.lastIndexOf(' '));
|
|
111
|
+
cut = fallback >= 8 ? fallback + 1 : this.maxChars;
|
|
112
|
+
}
|
|
113
|
+
if (cut < 0) {
|
|
114
|
+
if (!final)
|
|
115
|
+
break;
|
|
116
|
+
cut = this.buffer.length;
|
|
117
|
+
}
|
|
118
|
+
const chunk = this.buffer.slice(0, cut).trim();
|
|
119
|
+
this.buffer = this.buffer.slice(cut);
|
|
120
|
+
if (chunk)
|
|
121
|
+
chunks.push(chunk);
|
|
122
|
+
}
|
|
123
|
+
return chunks;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
async function* sseText(response) {
|
|
127
|
+
if (!response.body)
|
|
128
|
+
throw new Error('Summary endpoint returned no response body');
|
|
129
|
+
const reader = response.body.getReader(), decoder = new TextDecoder();
|
|
130
|
+
let buffer = '';
|
|
131
|
+
while (true) {
|
|
132
|
+
const { done, value } = await reader.read();
|
|
133
|
+
buffer += decoder.decode(value ?? new Uint8Array(), { stream: !done });
|
|
134
|
+
const records = buffer.split(/\r?\n\r?\n/);
|
|
135
|
+
buffer = records.pop() ?? '';
|
|
136
|
+
for (const record of records) {
|
|
137
|
+
for (const line of record.split(/\r?\n/)) {
|
|
138
|
+
if (!line.startsWith('data:'))
|
|
139
|
+
continue;
|
|
140
|
+
const data = line.slice(5).trim();
|
|
141
|
+
if (!data || data === '[DONE]')
|
|
142
|
+
continue;
|
|
143
|
+
let parsed;
|
|
144
|
+
try {
|
|
145
|
+
parsed = JSON.parse(data);
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const content = parsed?.choices?.[0]?.delta?.content;
|
|
151
|
+
if (typeof content === 'string' && content)
|
|
152
|
+
yield content;
|
|
153
|
+
else if (Array.isArray(content))
|
|
154
|
+
for (const part of content)
|
|
155
|
+
if (part?.type === 'text' && typeof part.text === 'string')
|
|
156
|
+
yield part.text;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (done)
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
export class VoiceManager {
|
|
164
|
+
fetcher;
|
|
165
|
+
emitter = new EventEmitter();
|
|
166
|
+
lastAssistant = new Map();
|
|
167
|
+
lastPrompt = new Map();
|
|
168
|
+
active = new Map();
|
|
169
|
+
generations = new Map();
|
|
170
|
+
announcementQueue = Promise.resolve();
|
|
171
|
+
closed = false;
|
|
172
|
+
config;
|
|
173
|
+
constructor(config, fetcher = fetch) {
|
|
174
|
+
this.fetcher = fetcher;
|
|
175
|
+
this.config = { language: 'zh-CN', maxInputChars: 32000, maxOutputTokens: 160, sampleRate: 24000, ttsFormat: 'pcm', requestTimeoutMs: 120000, ...config };
|
|
176
|
+
}
|
|
177
|
+
subscribe(listener) { this.emitter.on('event', listener); return () => this.emitter.off('event', listener); }
|
|
178
|
+
capabilities() { return { tts: true, stt: !!this.config.sttModel }; }
|
|
179
|
+
recordUserPrompt(agentId, prompt) { const value = prompt.trim(); if (value)
|
|
180
|
+
this.lastPrompt.set(agentId, value); }
|
|
181
|
+
emit(agentId, event) { this.emitter.emit('event', agentId, event); }
|
|
182
|
+
handleAgentEvent(agentId, event, context = {}) {
|
|
183
|
+
if (event.type === 'before_agent_start' && typeof event.prompt === 'string') {
|
|
184
|
+
this.lastPrompt.set(agentId, event.prompt);
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
if (event.type === 'message_end') {
|
|
188
|
+
const text = assistantText(event.message);
|
|
189
|
+
if (text)
|
|
190
|
+
this.lastAssistant.set(agentId, text);
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
if (event.type === 'agent_start') {
|
|
194
|
+
if (typeof event.message === 'string')
|
|
195
|
+
this.recordUserPrompt(agentId, event.message);
|
|
196
|
+
this.cancel(agentId);
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
if (event.type === 'agent_settled') {
|
|
200
|
+
const finalOutput = this.lastAssistant.get(agentId), userPrompt = this.lastPrompt.get(agentId) ?? '';
|
|
201
|
+
this.lastAssistant.delete(agentId);
|
|
202
|
+
this.lastPrompt.delete(agentId);
|
|
203
|
+
if (finalOutput)
|
|
204
|
+
void this.announce(agentId, { userPrompt, finalOutput, ...context });
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
cancel(agentId) { this.generations.set(agentId, (this.generations.get(agentId) ?? 0) + 1); const controller = this.active.get(agentId); if (controller) {
|
|
208
|
+
controller.abort();
|
|
209
|
+
this.active.delete(agentId);
|
|
210
|
+
this.emit(agentId, { type: 'voice_cancelled' });
|
|
211
|
+
} }
|
|
212
|
+
close() { this.closed = true; for (const id of [...new Set([...this.active.keys(), ...this.generations.keys()])])
|
|
213
|
+
this.cancel(id); this.emitter.removeAllListeners(); }
|
|
214
|
+
async transcribe(audio, mimeType = 'audio/webm', filename = 'recording.webm') {
|
|
215
|
+
if (!this.config.sttModel)
|
|
216
|
+
throw Object.assign(new Error('Voice STT model is not configured'), { code: 'VOICE_STT_DISABLED' });
|
|
217
|
+
const copy = new Uint8Array(audio.byteLength);
|
|
218
|
+
copy.set(audio);
|
|
219
|
+
const form = new FormData();
|
|
220
|
+
form.append('file', new Blob([copy.buffer], { type: mimeType }), filename);
|
|
221
|
+
form.append('model', this.config.sttModel);
|
|
222
|
+
form.append('response_format', 'json');
|
|
223
|
+
const response = await this.fetcher(endpoint(this.config.speechBaseUrl, '/audio/transcriptions'), { method: 'POST', signal: deadline(this.config.requestTimeoutMs), headers: authHeaders(this.config.speechApiKey), body: form }).catch(error => { throw timeoutError(error) ? Object.assign(new Error(`Speech transcription timed out after ${this.config.requestTimeoutMs} ms`), { code: 'VOICE_STT_FAILED' }) : error; });
|
|
224
|
+
if (!response.ok)
|
|
225
|
+
throw Object.assign(new Error(`Speech transcription failed (${response.status}): ${(await response.text()).slice(0, 500)}`), { code: 'VOICE_STT_FAILED' });
|
|
226
|
+
const value = await response.json();
|
|
227
|
+
const text = typeof value === 'string' ? value : value?.text;
|
|
228
|
+
if (typeof text !== 'string')
|
|
229
|
+
throw Object.assign(new Error('Speech transcription returned no text'), { code: 'VOICE_STT_FAILED' });
|
|
230
|
+
return { text: text.trim(), language: value?.language };
|
|
231
|
+
}
|
|
232
|
+
async announce(agentId, value) {
|
|
233
|
+
const input = typeof value === 'string' ? { userPrompt: '', finalOutput: value } : value;
|
|
234
|
+
this.cancel(agentId);
|
|
235
|
+
const generation = this.generations.get(agentId);
|
|
236
|
+
const job = this.announcementQueue.then(() => this.closed || this.generations.get(agentId) !== generation ? undefined : this.runAnnouncement(agentId, input, generation));
|
|
237
|
+
this.announcementQueue = job.then(() => undefined, () => undefined);
|
|
238
|
+
return job;
|
|
239
|
+
}
|
|
240
|
+
async runAnnouncement(agentId, input, generation) {
|
|
241
|
+
const controller = new AbortController(), playbackId = randomUUID();
|
|
242
|
+
this.active.set(agentId, controller);
|
|
243
|
+
this.emit(agentId, { type: 'voice_start', playbackId, encoding: this.config.ttsFormat === 'mp3' ? 'mp3' : 'pcm_s16le', sampleRate: this.config.sampleRate, channels: 1 });
|
|
244
|
+
let rawResult = '', spokenSummary = '';
|
|
245
|
+
try {
|
|
246
|
+
const summaryInput = prepareSummaryContext(input, this.config.maxInputChars);
|
|
247
|
+
const response = await this.fetcher(endpoint(this.config.summaryBaseUrl, '/chat/completions'), { method: 'POST', signal: deadline(this.config.requestTimeoutMs, controller.signal), headers: { 'content-type': 'application/json', ...authHeaders(this.config.summaryApiKey) }, body: JSON.stringify({ model: this.config.summaryModel, stream: true, temperature: 0.2, max_tokens: this.config.maxOutputTokens, messages: [{ role: 'system', content: this.summaryPrompt() }, { role: 'user', content: summaryInput }] }) });
|
|
248
|
+
if (!response.ok)
|
|
249
|
+
throw new Error(`Summary request failed (${response.status}): ${(await response.text()).slice(0, 500)}`);
|
|
250
|
+
for await (const delta of sseText(response))
|
|
251
|
+
rawResult += delta;
|
|
252
|
+
const result = parseSummaryResult(rawResult), sessionName = input.sessionName?.trim() || result.sessionName;
|
|
253
|
+
spokenSummary = formatSpokenSummary(result.summary, sessionName, this.config.language);
|
|
254
|
+
if (!input.sessionName?.trim() && result.sessionName && input.setSessionName) {
|
|
255
|
+
try {
|
|
256
|
+
await input.setSessionName(result.sessionName);
|
|
257
|
+
}
|
|
258
|
+
catch (error) {
|
|
259
|
+
this.emit(agentId, { type: 'voice_session_name_error', playbackId, message: error.message });
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
this.emit(agentId, { type: 'voice_summary_delta', playbackId, text: spokenSummary });
|
|
263
|
+
const chunker = new SentenceChunker();
|
|
264
|
+
for (const sentence of [...chunker.add(spokenSummary), ...chunker.close()])
|
|
265
|
+
await this.speak(agentId, playbackId, sentence, controller.signal);
|
|
266
|
+
if (this.active.get(agentId) === controller)
|
|
267
|
+
this.emit(agentId, { type: 'voice_end', playbackId, summary: spokenSummary, sessionName: result.sessionName });
|
|
268
|
+
}
|
|
269
|
+
catch (error) {
|
|
270
|
+
if (!abortError(error) && this.active.get(agentId) === controller) {
|
|
271
|
+
this.emit(agentId, { type: 'voice_error', playbackId, message: error.message });
|
|
272
|
+
const fallback = this.config.language.toLowerCase().startsWith('zh') ? 'Agent 已完成,等待输入。' : 'The agent has finished and is ready for input.';
|
|
273
|
+
if (!spokenSummary)
|
|
274
|
+
try {
|
|
275
|
+
this.emit(agentId, { type: 'voice_summary_delta', playbackId, text: fallback });
|
|
276
|
+
await this.speak(agentId, playbackId, fallback, controller.signal);
|
|
277
|
+
this.emit(agentId, { type: 'voice_end', playbackId, summary: fallback });
|
|
278
|
+
}
|
|
279
|
+
catch { /* The text notification remains available when TTS also fails. */ }
|
|
280
|
+
else
|
|
281
|
+
this.emit(agentId, { type: 'voice_end', playbackId, summary: spokenSummary, partial: true });
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
finally {
|
|
285
|
+
if (this.active.get(agentId) === controller)
|
|
286
|
+
this.active.delete(agentId);
|
|
287
|
+
if (this.generations.get(agentId) !== generation)
|
|
288
|
+
controller.abort();
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
summaryPrompt() { const language = this.config.language.toLowerCase().startsWith('zh') ? '简体中文' : '与用户输入相同的主要语言'; return `你负责把一次 Pi 编程任务压缩成准确、自然、可直接朗读的完成通知,并在需要时为 Session 命名。\n\n输入是一个 JSON 对象:\n- userPrompt:用户本轮真正想完成的任务。\n- piFinalOutput:Pi 最后一次回复,包含完成情况、改动、验证结果和后续事项。\n- sessionNeedsName:只有为 true 时才需要生成 Session 名称。\n\n先结合 userPrompt 判断目标,再以 piFinalOutput 为事实依据总结。不要把用户的要求误说成已经完成;没有明确证据时不要声称测试通过或任务成功。摘要使用${language},写 2 到 4 个短句,最多 120 个中文字符或 70 个英文单词。优先交代:是否完成、最重要的结果或修改、测试/验证结果、失败原因,以及用户必须采取的下一步。省略代码、命令、URL、完整路径、哈希、冗长文件清单、日志和 Markdown;不要使用标题、列表、“总结如下”等套话。\n\n如果 sessionNeedsName 为 true,请根据 userPrompt 生成一个具体、简短、便于检索的名称:中文建议 6 到 18 个字,英文建议 3 到 8 个词;使用任务主题或目标,不写“新会话”“任务总结”等空泛名称,不带句号、引号、Emoji 或路径。如果 sessionNeedsName 为 false,sessionName 必须为 null。\n\n只输出一个合法 JSON 对象,不要输出 Markdown 代码块、解释或任何额外文字。有名称时使用 {"summary":"适合直接朗读的摘要","sessionName":"简短名称"};无需命名时使用 {"summary":"适合直接朗读的摘要","sessionName":null}。summary 必须是 JSON 字符串,sessionName 必须是 JSON 字符串或 null。`; }
|
|
292
|
+
async speak(agentId, playbackId, text, signal) {
|
|
293
|
+
const response = await this.fetcher(endpoint(this.config.speechBaseUrl, '/audio/speech'), { method: 'POST', signal: deadline(this.config.requestTimeoutMs, signal), headers: { 'content-type': 'application/json', ...authHeaders(this.config.speechApiKey) }, body: JSON.stringify({ model: this.config.ttsModel, voice: this.config.ttsVoice, input: text, response_format: this.config.ttsFormat, sample_rate: this.config.sampleRate, stream_format: 'audio' }) });
|
|
294
|
+
if (!response.ok)
|
|
295
|
+
throw new Error(`Speech synthesis failed (${response.status}): ${(await response.text()).slice(0, 500)}`);
|
|
296
|
+
if (this.config.ttsFormat === 'mp3') {
|
|
297
|
+
const audio = Buffer.from(await response.arrayBuffer());
|
|
298
|
+
if (!audio.length)
|
|
299
|
+
throw new Error('Speech synthesis returned no audio');
|
|
300
|
+
this.emit(agentId, { type: 'voice_audio_chunk', playbackId, encoding: 'mp3', audio: audio.toString('base64') });
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
if (!response.body)
|
|
304
|
+
throw new Error('Speech synthesis returned no audio');
|
|
305
|
+
const reader = response.body.getReader();
|
|
306
|
+
let pending;
|
|
307
|
+
while (true) {
|
|
308
|
+
const { done, value } = await reader.read();
|
|
309
|
+
if (done)
|
|
310
|
+
break;
|
|
311
|
+
if (!value?.length)
|
|
312
|
+
continue;
|
|
313
|
+
const bytes = pending ? Buffer.concat([pending, Buffer.from(value)]) : Buffer.from(value);
|
|
314
|
+
const evenLength = bytes.length & ~1;
|
|
315
|
+
if (evenLength)
|
|
316
|
+
this.emit(agentId, { type: 'voice_audio_chunk', playbackId, sampleRate: this.config.sampleRate, audio: bytes.subarray(0, evenLength).toString('base64') });
|
|
317
|
+
pending = evenLength < bytes.length ? Buffer.from(bytes.subarray(evenLength)) : undefined;
|
|
318
|
+
}
|
|
319
|
+
if (pending)
|
|
320
|
+
throw new Error('Speech synthesis returned an incomplete PCM16 sample');
|
|
321
|
+
}
|
|
322
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { statSync } from 'node:fs';
|
|
3
|
+
import { lstat, open, realpath, readdir, stat } from 'node:fs/promises';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
export class WorkspaceStore {
|
|
6
|
+
items;
|
|
7
|
+
constructor(items = []) {
|
|
8
|
+
this.items = items;
|
|
9
|
+
}
|
|
10
|
+
list() { return this.items.map(x => ({ ...x })); }
|
|
11
|
+
replace(items) { this.items = items.map(item => ({ ...item, rootPath: path.resolve(item.rootPath) })); }
|
|
12
|
+
add(label, rootPath) { const root = path.resolve(rootPath); let rootStat; try {
|
|
13
|
+
rootStat = statSync(root);
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
if (error.code === 'ENOENT' || error.code === 'ENOTDIR')
|
|
17
|
+
throw Object.assign(new Error('Workspace root does not exist'), { code: 'WORKSPACE_ROOT_NOT_FOUND' });
|
|
18
|
+
throw error;
|
|
19
|
+
} if (!rootStat.isDirectory())
|
|
20
|
+
throw Object.assign(new Error('Workspace root is not a directory'), { code: 'WORKSPACE_ROOT_NOT_DIRECTORY' }); const key = process.platform === 'win32' ? root.toLowerCase() : root; if (this.items.some(item => (process.platform === 'win32' ? item.rootPath.toLowerCase() : item.rootPath) === key))
|
|
21
|
+
throw Object.assign(new Error('Workspace root already exists'), { code: 'WORKSPACE_ALREADY_EXISTS' }); const item = { id: randomUUID(), label, rootPath: root, createdAt: new Date().toISOString() }; this.items.push(item); return item; }
|
|
22
|
+
get(id) { return this.items.find(x => x.id === id); }
|
|
23
|
+
async resolve(ws, relative = '.') { const candidate = path.resolve(ws.rootPath, relative); const root = await realpath(ws.rootPath); let actual; try {
|
|
24
|
+
actual = await realpath(candidate);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
throw Object.assign(new Error('Path does not exist'), { code: 'PATH_NOT_FOUND' });
|
|
28
|
+
} if (actual !== root && !actual.startsWith(root + path.sep))
|
|
29
|
+
throw Object.assign(new Error('Path is outside the workspace root'), { code: 'WORKSPACE_PATH_OUTSIDE_ROOT' }); return actual; }
|
|
30
|
+
async tree(id, relative = '.') { const ws = this.get(id); if (!ws)
|
|
31
|
+
throw Object.assign(new Error('Workspace not found'), { code: 'WORKSPACE_NOT_FOUND' }); const dir = await this.resolve(ws, relative); if (!(await stat(dir)).isDirectory())
|
|
32
|
+
throw new Error('Not a directory'); const entries = await readdir(dir, { withFileTypes: true }); return Promise.all(entries.sort((a, b) => Number(b.isDirectory()) - Number(a.isDirectory()) || a.name.localeCompare(b.name)).map(async (e) => { const p = path.join(dir, e.name); const s = await lstat(p); return { name: e.name, type: e.isDirectory() ? 'directory' : 'file', size: s.size, modifiedAt: s.mtime.toISOString() }; })); }
|
|
33
|
+
async stat(id, relative = '.') { const ws = this.get(id); if (!ws)
|
|
34
|
+
throw Object.assign(new Error('Workspace not found'), { code: 'WORKSPACE_NOT_FOUND' }); const p = await this.resolve(ws, relative); const s = await lstat(p); return { path: relative, type: s.isDirectory() ? 'directory' : s.isFile() ? 'file' : 'other', size: s.size, modifiedAt: s.mtime.toISOString() }; }
|
|
35
|
+
async file(id, relative, offset = 0, limit = 1024 * 1024) {
|
|
36
|
+
const ws = this.get(id);
|
|
37
|
+
if (!ws)
|
|
38
|
+
throw Object.assign(new Error('Workspace not found'), { code: 'WORKSPACE_NOT_FOUND' });
|
|
39
|
+
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1)
|
|
40
|
+
throw Object.assign(new Error('Invalid file range'), { code: 'INVALID_REQUEST' });
|
|
41
|
+
if (limit > 1024 * 1024)
|
|
42
|
+
throw Object.assign(new Error('File limit exceeded'), { code: 'FILE_TOO_LARGE' });
|
|
43
|
+
const p = await this.resolve(ws, relative), s = await stat(p);
|
|
44
|
+
if (!s.isFile())
|
|
45
|
+
throw new Error('Not a file');
|
|
46
|
+
if (s.size > 10 * 1024 * 1024)
|
|
47
|
+
throw Object.assign(new Error('File limit exceeded'), { code: 'FILE_TOO_LARGE' });
|
|
48
|
+
const length = Math.min(limit, Math.max(0, s.size - offset)), handle = await open(p, 'r');
|
|
49
|
+
let chunk;
|
|
50
|
+
try {
|
|
51
|
+
const buffer = Buffer.alloc(length), result = await handle.read(buffer, 0, length, offset);
|
|
52
|
+
chunk = buffer.subarray(0, result.bytesRead);
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
await handle.close();
|
|
56
|
+
}
|
|
57
|
+
const binary = chunk.includes(0);
|
|
58
|
+
return { path: relative, size: s.size, modifiedAt: s.mtime.toISOString(), binary, content: binary ? undefined : chunk.toString('utf8'), offset, limit: chunk.length };
|
|
59
|
+
}
|
|
60
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "pi2web",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Secure remote web and Android client server for Pi coding agent",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"author": "sxwxs",
|
|
7
|
+
"license": "MIT",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/sxwxs/pi2web.git"
|
|
11
|
+
},
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/sxwxs/pi2web/issues"
|
|
14
|
+
},
|
|
15
|
+
"homepage": "https://github.com/sxwxs/pi2web#readme",
|
|
16
|
+
"bin": {
|
|
17
|
+
"pi2web": "dist/main.js"
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"dist/",
|
|
21
|
+
"web/",
|
|
22
|
+
"packages/protocol/",
|
|
23
|
+
"packages/edge-tts-server/src/remote_pi_edge_tts/",
|
|
24
|
+
"packages/edge-tts-server/pyproject.toml",
|
|
25
|
+
"packages/edge-tts-server/README.md",
|
|
26
|
+
"README.md",
|
|
27
|
+
"VOICE_SERVER_SETUP.md",
|
|
28
|
+
"ANDROID_APP_FEATURES.md",
|
|
29
|
+
"LICENSE"
|
|
30
|
+
],
|
|
31
|
+
"publishConfig": {
|
|
32
|
+
"access": "public",
|
|
33
|
+
"registry": "https://registry.npmjs.org/"
|
|
34
|
+
},
|
|
35
|
+
"scripts": {
|
|
36
|
+
"clean": "rm -rf dist",
|
|
37
|
+
"build": "npm run clean && tsc -p tsconfig.json",
|
|
38
|
+
"check": "npm run build",
|
|
39
|
+
"test": "vitest run",
|
|
40
|
+
"dev": "tsx src/main.ts",
|
|
41
|
+
"start": "node dist/main.js",
|
|
42
|
+
"prepack": "npm run build",
|
|
43
|
+
"prepublishOnly": "npm run build && npm test"
|
|
44
|
+
},
|
|
45
|
+
"engines": {
|
|
46
|
+
"node": ">=20.10"
|
|
47
|
+
},
|
|
48
|
+
"dependencies": {
|
|
49
|
+
"@earendil-works/pi-coding-agent": "0.80.6",
|
|
50
|
+
"@xterm/addon-fit": "^0.11.0",
|
|
51
|
+
"@xterm/xterm": "^6.0.0",
|
|
52
|
+
"better-sqlite3": "^12.11.1",
|
|
53
|
+
"dompurify": "^3.4.12",
|
|
54
|
+
"marked": "^18.0.6",
|
|
55
|
+
"node-pty": "^1.1.0",
|
|
56
|
+
"ws": "^8.18.0"
|
|
57
|
+
},
|
|
58
|
+
"devDependencies": {
|
|
59
|
+
"@types/better-sqlite3": "^7.6.13",
|
|
60
|
+
"@types/node": "^22.0.0",
|
|
61
|
+
"@types/ws": "^8.5.0",
|
|
62
|
+
"tsx": "^4.19.0",
|
|
63
|
+
"typescript": "^5.9.3",
|
|
64
|
+
"vitest": "^3.0.0"
|
|
65
|
+
},
|
|
66
|
+
"overrides": {
|
|
67
|
+
"vite": "7.3.5"
|
|
68
|
+
},
|
|
69
|
+
"keywords": [
|
|
70
|
+
"pi",
|
|
71
|
+
"pi-coding-agent",
|
|
72
|
+
"coding-agent",
|
|
73
|
+
"remote",
|
|
74
|
+
"remote-control",
|
|
75
|
+
"web-ui",
|
|
76
|
+
"terminal",
|
|
77
|
+
"websocket"
|
|
78
|
+
]
|
|
79
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Native Edge TTS server for Remote Pi
|
|
2
|
+
|
|
3
|
+
这是一个很薄的 Python 服务:
|
|
4
|
+
|
|
5
|
+
- `edge-tts` 连接 Microsoft Edge 在线语音服务并生成 MP3。
|
|
6
|
+
- `aiohttp` 提供 OpenAI-compatible `POST /v1/audio/speech` HTTP 接口。
|
|
7
|
+
- 不需要 Docker、FFmpeg,也不提供语音识别。
|
|
8
|
+
- 默认只监听 `127.0.0.1:5050`。
|
|
9
|
+
|
|
10
|
+
## 安装和启动
|
|
11
|
+
|
|
12
|
+
要求 Python 3.10 或更高版本。
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
cd packages/edge-tts-server
|
|
16
|
+
python -m venv .venv
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Windows PowerShell:
|
|
20
|
+
|
|
21
|
+
```powershell
|
|
22
|
+
.venv\Scripts\Activate.ps1
|
|
23
|
+
python -m pip install -e .
|
|
24
|
+
remote-pi-edge-tts
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Linux/macOS:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
source .venv/bin/activate
|
|
31
|
+
python -m pip install -e .
|
|
32
|
+
remote-pi-edge-tts
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
可以指定监听地址和默认音色:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
remote-pi-edge-tts --host 127.0.0.1 --port 5050 --voice zh-CN-XiaoxiaoNeural
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## 测试
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
curl http://127.0.0.1:5050/v1/audio/speech \
|
|
45
|
+
-H "Content-Type: application/json" \
|
|
46
|
+
-d '{"model":"edge-tts","voice":"zh-CN-XiaoxiaoNeural","input":"你好,这是 Edge TTS 测试。","response_format":"mp3"}' \
|
|
47
|
+
--output edge-tts-test.mp3
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## 启动 Remote Pi
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
remote-pi \
|
|
54
|
+
--voice-base-url http://127.0.0.1:5050/v1 \
|
|
55
|
+
--voice-tts-model edge-tts \
|
|
56
|
+
--voice-tts-voice zh-CN-XiaoxiaoNeural \
|
|
57
|
+
--voice-tts-format mp3 \
|
|
58
|
+
--voice-summary-base-url https://llm.example.com/v1 \
|
|
59
|
+
--voice-summary-model your-summary-model \
|
|
60
|
+
--voice-summary-api-key-env SUMMARY_API_KEY \
|
|
61
|
+
--voice-language zh-CN
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
不要传 `--voice-stt-model`。Web UI 会提供 Agent 摘要播报,但隐藏麦克风按钮。
|
|
65
|
+
|
|
66
|
+
常用中文音色包括 `zh-CN-XiaoxiaoNeural`、`zh-CN-YunxiNeural`、`zh-CN-YunyangNeural` 和 `zh-CN-YunjianNeural`。
|
|
67
|
+
|
|
68
|
+
Edge TTS 使用非官方消费者服务,建议仅在本机或可信私有网络中使用。
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "remote-pi-edge-tts-server"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Small OpenAI-compatible Edge TTS server for Remote Pi"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"aiohttp>=3.9,<4",
|
|
12
|
+
"edge-tts>=7.2,<8",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[project.scripts]
|
|
16
|
+
remote-pi-edge-tts = "remote_pi_edge_tts.__main__:main"
|
|
17
|
+
|
|
18
|
+
[tool.setuptools.packages.find]
|
|
19
|
+
where = ["src"]
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import asyncio
|
|
5
|
+
import os
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import edge_tts
|
|
9
|
+
from aiohttp import web
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def error(message: str, status: int = 400) -> web.Response:
|
|
13
|
+
return web.json_response({"error": {"message": message}}, status=status)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def create_app(default_voice: str, max_chars: int) -> web.Application:
|
|
17
|
+
app = web.Application(client_max_size=256 * 1024)
|
|
18
|
+
app["default_voice"] = default_voice
|
|
19
|
+
app["max_chars"] = max_chars
|
|
20
|
+
|
|
21
|
+
async def health(_: web.Request) -> web.Response:
|
|
22
|
+
return web.json_response({"status": "ok", "service": "remote-pi-edge-tts"})
|
|
23
|
+
|
|
24
|
+
async def voices(_: web.Request) -> web.Response:
|
|
25
|
+
values = await edge_tts.list_voices()
|
|
26
|
+
return web.json_response({"data": values})
|
|
27
|
+
|
|
28
|
+
async def speech(request: web.Request) -> web.StreamResponse:
|
|
29
|
+
try:
|
|
30
|
+
body: dict[str, Any] = await request.json()
|
|
31
|
+
except Exception:
|
|
32
|
+
return error("Request body must be valid JSON")
|
|
33
|
+
text = body.get("input")
|
|
34
|
+
if not isinstance(text, str) or not text.strip():
|
|
35
|
+
return error("input must be a non-empty string")
|
|
36
|
+
if len(text) > request.app["max_chars"]:
|
|
37
|
+
return error(f"input exceeds {request.app['max_chars']} characters", 413)
|
|
38
|
+
response_format = str(body.get("response_format", "mp3")).lower()
|
|
39
|
+
if response_format not in {"mp3", "mpeg"}:
|
|
40
|
+
return error("Only response_format=mp3 is supported")
|
|
41
|
+
voice = str(body.get("voice") or request.app["default_voice"])
|
|
42
|
+
rate = str(body.get("rate", "+0%"))
|
|
43
|
+
volume = str(body.get("volume", "+0%"))
|
|
44
|
+
pitch = str(body.get("pitch", "+0Hz"))
|
|
45
|
+
try:
|
|
46
|
+
communicate = edge_tts.Communicate(text.strip(), voice, rate=rate, volume=volume, pitch=pitch)
|
|
47
|
+
chunks: list[bytes] = []
|
|
48
|
+
async for chunk in communicate.stream():
|
|
49
|
+
if chunk["type"] == "audio":
|
|
50
|
+
chunks.append(chunk["data"])
|
|
51
|
+
audio = b"".join(chunks)
|
|
52
|
+
if not audio:
|
|
53
|
+
return error("Edge TTS returned no audio", 502)
|
|
54
|
+
return web.Response(body=audio, headers={"Content-Type": "audio/mpeg", "Cache-Control": "no-store"})
|
|
55
|
+
except asyncio.CancelledError:
|
|
56
|
+
raise
|
|
57
|
+
except Exception as exc:
|
|
58
|
+
return error(f"Edge TTS failed: {exc}", 502)
|
|
59
|
+
|
|
60
|
+
app.router.add_get("/health", health)
|
|
61
|
+
app.router.add_get("/v1/voices", voices)
|
|
62
|
+
app.router.add_post("/v1/audio/speech", speech)
|
|
63
|
+
return app
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def main() -> None:
|
|
67
|
+
parser = argparse.ArgumentParser(description="OpenAI-compatible Edge TTS server for Remote Pi")
|
|
68
|
+
parser.add_argument("--host", default=os.environ.get("EDGE_TTS_HOST", "127.0.0.1"))
|
|
69
|
+
parser.add_argument("--port", type=int, default=int(os.environ.get("EDGE_TTS_PORT", "5050")))
|
|
70
|
+
parser.add_argument("--voice", default=os.environ.get("EDGE_TTS_VOICE", "zh-CN-XiaoxiaoNeural"))
|
|
71
|
+
parser.add_argument("--max-chars", type=int, default=int(os.environ.get("EDGE_TTS_MAX_CHARS", "5000")))
|
|
72
|
+
args = parser.parse_args()
|
|
73
|
+
web.run_app(create_app(args.voice, args.max_chars), host=args.host, port=args.port)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
if __name__ == "__main__":
|
|
77
|
+
main()
|