pi2web 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/dist/agents.js +183 -41
- package/dist/http/agent-routes.js +28 -2
- package/dist/http/utils.js +5 -1
- package/dist/http/web-assets.js +1 -0
- package/dist/metadata-store.js +14 -1
- package/dist/sdk-backend.js +28 -0
- package/dist/server.js +52 -2
- package/dist/session-performance.js +137 -0
- package/dist/sotto-stt.js +217 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/packages/protocol/openapi.yaml +18 -0
- package/web/app.js +602 -66
- package/web/collab.js +18 -4
- package/web/index.html +4 -2
- package/web/styles.css +7 -2
- package/web/voice-pcm.js +36 -0
package/dist/server.js
CHANGED
|
@@ -8,6 +8,7 @@ import { AgentManager } from './agents.js';
|
|
|
8
8
|
import { createSdkBackend } from './sdk-backend.js';
|
|
9
9
|
import { TerminalManager } from './terminals.js';
|
|
10
10
|
import { MetadataStore } from './metadata-store.js';
|
|
11
|
+
import { SottoSttService } from './sotto-stt.js';
|
|
11
12
|
import { CollabHub } from './collab/hub.js';
|
|
12
13
|
import { CollabStore } from './collab/store.js';
|
|
13
14
|
import { CollabRouter } from './collab/routes.js';
|
|
@@ -54,6 +55,7 @@ export class RemotePiServer {
|
|
|
54
55
|
sessionIndexJobs = new Map();
|
|
55
56
|
agentIndexJobs = new Map();
|
|
56
57
|
voice;
|
|
58
|
+
sottoStt;
|
|
57
59
|
sessionNamer;
|
|
58
60
|
mailNotifier;
|
|
59
61
|
loginLimiter = new LoginLimiter();
|
|
@@ -64,6 +66,7 @@ export class RemotePiServer {
|
|
|
64
66
|
this.dataDir = options.dataDir ?? defaultDataDir();
|
|
65
67
|
this.auth = new AuthStore(this.dataDir);
|
|
66
68
|
this.metadata = new MetadataStore(this.dataDir);
|
|
69
|
+
this.sottoStt = new SottoSttService(this.metadata);
|
|
67
70
|
this.templates = new AgentTemplateStore(path.join(this.dataDir, AGENT_TEMPLATES_DIR));
|
|
68
71
|
this.workspaces = options.workspaces ?? new WorkspaceStore();
|
|
69
72
|
this.voice = options.voice;
|
|
@@ -115,6 +118,7 @@ export class RemotePiServer {
|
|
|
115
118
|
console.warn('WARNING: --allow-insecure-no-auth disables loopback bind and Host/Origin restrictions. Anyone who can reach this server, including through a web page, can access workspaces and control agents and terminals without a pairing code. Use only on a trusted, access-controlled network.');
|
|
116
119
|
const first = this.noAuth ? {} : await this.auth.init();
|
|
117
120
|
this.metadata.init();
|
|
121
|
+
this.sottoStt.load();
|
|
118
122
|
this.collab.init();
|
|
119
123
|
this.unsubscribeCollab = this.collab.subscribe(event => this.collabNotification(event));
|
|
120
124
|
this.collabStallTimer = setInterval(() => {
|
|
@@ -343,7 +347,10 @@ export class RemotePiServer {
|
|
|
343
347
|
.map(x => x.trim())
|
|
344
348
|
.find(x => x.startsWith('access-token.'))
|
|
345
349
|
?.slice(13);
|
|
346
|
-
const pathname = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`).pathname,
|
|
350
|
+
const pathname = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`).pathname, voiceMatch = pathname.match(/^\/api\/v1\/agents\/([^/]+)\/transcribe\/ws$/), isTranscription = pathname === '/api/v1/transcribe/ws', isSocket = pathname === '/api/v1/ws' ||
|
|
351
|
+
/^\/api\/v1\/terminals\/[^/]+\/ws$/.test(pathname) ||
|
|
352
|
+
!!voiceMatch ||
|
|
353
|
+
isTranscription;
|
|
347
354
|
if (!isSocket) {
|
|
348
355
|
socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
|
|
349
356
|
socket.destroy();
|
|
@@ -374,6 +381,24 @@ export class RemotePiServer {
|
|
|
374
381
|
return;
|
|
375
382
|
}
|
|
376
383
|
this.loginLimiter.clear(key);
|
|
384
|
+
if (voiceMatch || isTranscription) {
|
|
385
|
+
try {
|
|
386
|
+
if (voiceMatch)
|
|
387
|
+
this.agents.get(decodeURIComponent(voiceMatch[1]));
|
|
388
|
+
this.sottoStt.streamUrl();
|
|
389
|
+
}
|
|
390
|
+
catch {
|
|
391
|
+
socket.write('HTTP/1.1 404 Not Found\r\n\r\n');
|
|
392
|
+
socket.destroy();
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
this.wss.handleUpgrade(req, socket, head, ws => {
|
|
396
|
+
this.clients.add(ws);
|
|
397
|
+
ws.on('close', () => this.clients.delete(ws));
|
|
398
|
+
this.sottoStt.stream(ws);
|
|
399
|
+
});
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
377
402
|
this.wss.handleUpgrade(req, socket, head, ws => this.wss.emit('connection', ws, req));
|
|
378
403
|
}
|
|
379
404
|
clientKey(req) {
|
|
@@ -575,12 +600,36 @@ export class RemotePiServer {
|
|
|
575
600
|
terminalCount: this.terminals.list().length,
|
|
576
601
|
webSocketConnections: this.clients.size,
|
|
577
602
|
voiceEnabled: !!this.voice,
|
|
578
|
-
voiceCapabilities:
|
|
603
|
+
voiceCapabilities: {
|
|
604
|
+
tts: this.voice?.capabilities().tts ?? false,
|
|
605
|
+
stt: (this.voice?.capabilities().stt ?? false) || this.sottoStt.getSettings().enabled
|
|
606
|
+
},
|
|
607
|
+
sottoStt: this.sottoStt.getSettings(),
|
|
579
608
|
sessionNamingEnabled: !!this.sessionNamer,
|
|
580
609
|
mailNotificationsConfigured: !!this.mailNotifier,
|
|
581
610
|
mailNotificationsEnabled: this.mailNotifier?.getSettings().enabled ?? false
|
|
582
611
|
}
|
|
583
612
|
});
|
|
613
|
+
if (u.pathname === '/api/v1/sotto-stt') {
|
|
614
|
+
if (req.method === 'GET')
|
|
615
|
+
return json(res, 200, { data: this.sottoStt.getSettings() });
|
|
616
|
+
if (req.method === 'PUT')
|
|
617
|
+
return json(res, 200, { data: await this.sottoStt.update(await readJsonBody(req)) });
|
|
618
|
+
}
|
|
619
|
+
if (req.method === 'POST' && u.pathname === '/api/v1/transcribe') {
|
|
620
|
+
const mime = String(req.headers['content-type'] ?? 'audio/webm').split(';')[0];
|
|
621
|
+
if (this.sottoStt.getSettings().enabled) {
|
|
622
|
+
if (mime !== 'application/octet-stream')
|
|
623
|
+
throw Object.assign(new Error('Sotto requires 16 kHz float32 PCM'), { code: 'INVALID_REQUEST' });
|
|
624
|
+
return json(res, 200, { data: await this.sottoStt.transcribe(await readBinaryBody(req)) });
|
|
625
|
+
}
|
|
626
|
+
if (!this.voice)
|
|
627
|
+
throw Object.assign(new Error('Voice service is disabled'), { code: 'VOICE_DISABLED' });
|
|
628
|
+
const filename = String(req.headers['x-audio-filename'] ?? 'recording.webm');
|
|
629
|
+
return json(res, 200, {
|
|
630
|
+
data: await this.voice.transcribe(await readBinaryBody(req), mime, filename)
|
|
631
|
+
});
|
|
632
|
+
}
|
|
584
633
|
if (await handleMailSettingsRoutes({ mailNotifier: this.mailNotifier, metadata: this.metadata }, req, res, u))
|
|
585
634
|
return;
|
|
586
635
|
const workspaceContext = {
|
|
@@ -599,6 +648,7 @@ export class RemotePiServer {
|
|
|
599
648
|
metadata: this.metadata,
|
|
600
649
|
workspaces: this.workspaces,
|
|
601
650
|
voice: this.voice,
|
|
651
|
+
sottoStt: this.sottoStt,
|
|
602
652
|
sessionNamer: this.sessionNamer,
|
|
603
653
|
ensureSessionIndex: (workspace, cwd, force) => this.ensureSessionIndex(workspace, cwd, force),
|
|
604
654
|
indexAgentSession: (agentId) => this.indexAgentSession(agentId),
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
export const PERFORMANCE_ENTRY = 'pi2web.performance.v1';
|
|
2
|
+
/** Wall-clock spans, collected on the host running Pi. Parallel tool spans deliberately overlap. */
|
|
3
|
+
export class SessionPerformanceTracker {
|
|
4
|
+
save;
|
|
5
|
+
now;
|
|
6
|
+
run = null;
|
|
7
|
+
model = null;
|
|
8
|
+
tools = new Map();
|
|
9
|
+
other = null;
|
|
10
|
+
constructor(save, now = Date.now) {
|
|
11
|
+
this.save = save;
|
|
12
|
+
this.now = now;
|
|
13
|
+
}
|
|
14
|
+
finish(span, at) {
|
|
15
|
+
if (span)
|
|
16
|
+
span.end = at;
|
|
17
|
+
}
|
|
18
|
+
span(kind, at, name) {
|
|
19
|
+
const span = { kind, start: at, end: at };
|
|
20
|
+
if (name)
|
|
21
|
+
span.name = name;
|
|
22
|
+
this.run.spans.push(span);
|
|
23
|
+
return span;
|
|
24
|
+
}
|
|
25
|
+
apply(event) {
|
|
26
|
+
const at = this.now(), e = event;
|
|
27
|
+
if (event.type === 'agent_start' && !this.run)
|
|
28
|
+
this.run = { start: at, spans: [] };
|
|
29
|
+
if (!this.run)
|
|
30
|
+
return;
|
|
31
|
+
switch (event.type) {
|
|
32
|
+
case 'turn_start':
|
|
33
|
+
if (this.other?.kind === 'retry') {
|
|
34
|
+
this.finish(this.other, at);
|
|
35
|
+
this.other = null;
|
|
36
|
+
}
|
|
37
|
+
this.finish(this.model, at);
|
|
38
|
+
this.model = this.span('model', at);
|
|
39
|
+
break;
|
|
40
|
+
case 'message_update':
|
|
41
|
+
if (this.model && this.model.firstResponse === undefined)
|
|
42
|
+
this.model.firstResponse = at;
|
|
43
|
+
break;
|
|
44
|
+
case 'message_end':
|
|
45
|
+
if (e.message?.role === 'assistant') {
|
|
46
|
+
this.finish(this.model, at);
|
|
47
|
+
this.model = null;
|
|
48
|
+
}
|
|
49
|
+
break;
|
|
50
|
+
case 'tool_execution_start':
|
|
51
|
+
if (e.parentToolCallId)
|
|
52
|
+
break; // nested tools are included in their parent's wall-clock span
|
|
53
|
+
if (this.model) {
|
|
54
|
+
this.finish(this.model, at);
|
|
55
|
+
this.model = null;
|
|
56
|
+
}
|
|
57
|
+
if (e.toolCallId) {
|
|
58
|
+
const span = this.span('tool', at, String(e.toolName ?? 'tool'));
|
|
59
|
+
span.toolCallId = String(e.toolCallId);
|
|
60
|
+
this.tools.set(span.toolCallId, span);
|
|
61
|
+
}
|
|
62
|
+
break;
|
|
63
|
+
case 'tool_execution_end': {
|
|
64
|
+
const span = this.tools.get(String(e.toolCallId));
|
|
65
|
+
if (span) {
|
|
66
|
+
this.finish(span, at);
|
|
67
|
+
this.tools.delete(String(e.toolCallId));
|
|
68
|
+
}
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
case 'compaction_start':
|
|
72
|
+
case 'auto_retry_start':
|
|
73
|
+
this.finish(this.other, at);
|
|
74
|
+
this.other = this.span(event.type === 'compaction_start' ? 'compaction' : 'retry', at);
|
|
75
|
+
break;
|
|
76
|
+
case 'compaction_end':
|
|
77
|
+
case 'auto_retry_end':
|
|
78
|
+
this.finish(this.other, at);
|
|
79
|
+
this.other = null;
|
|
80
|
+
if (event.type === 'auto_retry_end' && e.finalError === 'Retry cancelled')
|
|
81
|
+
this.run.incomplete = true;
|
|
82
|
+
break;
|
|
83
|
+
case 'agent_end':
|
|
84
|
+
this.finish(this.model, at);
|
|
85
|
+
this.model = null;
|
|
86
|
+
if (e.aborted)
|
|
87
|
+
this.run.incomplete = true;
|
|
88
|
+
break;
|
|
89
|
+
case 'agent_settled':
|
|
90
|
+
// Unlike agent_end, this also follows post-run compaction or a cancelled retry delay.
|
|
91
|
+
this.finish(this.model, at);
|
|
92
|
+
this.finish(this.other, at);
|
|
93
|
+
for (const span of this.tools.values()) {
|
|
94
|
+
this.finish(span, at);
|
|
95
|
+
span.incomplete = true;
|
|
96
|
+
}
|
|
97
|
+
this.save({
|
|
98
|
+
start: this.run.start,
|
|
99
|
+
end: at,
|
|
100
|
+
spans: this.run.spans,
|
|
101
|
+
...(this.run.incomplete ? { incomplete: true } : {})
|
|
102
|
+
});
|
|
103
|
+
this.run = null;
|
|
104
|
+
this.model = this.other = null;
|
|
105
|
+
this.tools.clear();
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
/** Current branch only. Earlier sessions have tool timestamps but no model-call spans. */
|
|
110
|
+
export function sessionPerformance(entries) {
|
|
111
|
+
const runs = [];
|
|
112
|
+
const legacyTools = [];
|
|
113
|
+
for (const entry of entries) {
|
|
114
|
+
const e = entry;
|
|
115
|
+
if (e.type === 'custom' &&
|
|
116
|
+
e.customType === PERFORMANCE_ENTRY &&
|
|
117
|
+
e.data?.spans &&
|
|
118
|
+
Number.isFinite(e.data.start) &&
|
|
119
|
+
Number.isFinite(e.data.end)) {
|
|
120
|
+
runs.push(e.data);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
for (const entry of entries) {
|
|
124
|
+
const e = entry;
|
|
125
|
+
if (e.type !== 'message' || e.message?.role !== 'toolResult')
|
|
126
|
+
continue;
|
|
127
|
+
const duration = Number(e.message.durationMs);
|
|
128
|
+
const end = Number(e.message.timestamp) || Date.parse(e.timestamp ?? '');
|
|
129
|
+
if (!Number.isFinite(duration) ||
|
|
130
|
+
duration < 0 ||
|
|
131
|
+
!Number.isFinite(end) ||
|
|
132
|
+
runs.some(run => end >= run.start && end <= run.end))
|
|
133
|
+
continue;
|
|
134
|
+
legacyTools.push({ kind: 'tool', start: end - duration, end, name: String(e.message.toolName ?? 'tool') });
|
|
135
|
+
}
|
|
136
|
+
return { runs, legacyTools };
|
|
137
|
+
}
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { WebSocket } from 'ws';
|
|
2
|
+
export const DEFAULT_SOTTO_ENDPOINT = 'http://127.0.0.1:8766';
|
|
3
|
+
const invalid = (message, code = 'INVALID_SOTTO_SETTINGS') => Object.assign(new Error(message), { code });
|
|
4
|
+
/** No arbitrary URLs: the Sotto Web API is local-only, and this server is the authenticated proxy. */
|
|
5
|
+
export function validateSottoEndpoint(value) {
|
|
6
|
+
if (typeof value !== 'string')
|
|
7
|
+
throw invalid('Sotto endpoint must be a loopback HTTP URL');
|
|
8
|
+
let url;
|
|
9
|
+
try {
|
|
10
|
+
url = new URL(value);
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
throw invalid('Sotto endpoint must be a valid URL');
|
|
14
|
+
}
|
|
15
|
+
if (url.protocol !== 'http:' ||
|
|
16
|
+
url.hostname !== '127.0.0.1' ||
|
|
17
|
+
!url.port ||
|
|
18
|
+
url.username ||
|
|
19
|
+
url.password ||
|
|
20
|
+
(url.pathname !== '/' && url.pathname !== '') ||
|
|
21
|
+
url.search ||
|
|
22
|
+
url.hash)
|
|
23
|
+
throw invalid('Sotto endpoint must be http://127.0.0.1:<port> with no path or credentials');
|
|
24
|
+
return url.origin;
|
|
25
|
+
}
|
|
26
|
+
/** An authenticated browser socket relays only audio and start/stop, never a Sotto control API. */
|
|
27
|
+
function proxySottoStream(client, url) {
|
|
28
|
+
const upstream = new WebSocket(url, { handshakeTimeout: 5000, maxPayload: 65536 });
|
|
29
|
+
let started = false, finished = false, closed = false;
|
|
30
|
+
let timer;
|
|
31
|
+
const close = () => {
|
|
32
|
+
closed = true;
|
|
33
|
+
clearTimeout(timer);
|
|
34
|
+
if (upstream.readyState === WebSocket.OPEN)
|
|
35
|
+
upstream.close();
|
|
36
|
+
else if (upstream.readyState === WebSocket.CONNECTING)
|
|
37
|
+
upstream.terminate();
|
|
38
|
+
};
|
|
39
|
+
const fail = (message, code = 1011) => {
|
|
40
|
+
if (closed)
|
|
41
|
+
return;
|
|
42
|
+
if (client.readyState === WebSocket.OPEN) {
|
|
43
|
+
client.send(JSON.stringify({ type: 'error', message }));
|
|
44
|
+
client.close(code);
|
|
45
|
+
}
|
|
46
|
+
close();
|
|
47
|
+
};
|
|
48
|
+
// Silence still contains PCM. Only a stalled capture/connection times out, not a long dictation.
|
|
49
|
+
const touch = () => {
|
|
50
|
+
clearTimeout(timer);
|
|
51
|
+
timer = setTimeout(() => fail('Audio stream stalled: no microphone frames received for 30 seconds'), 30000);
|
|
52
|
+
};
|
|
53
|
+
touch();
|
|
54
|
+
client.on('close', close);
|
|
55
|
+
client.on('error', close);
|
|
56
|
+
upstream.on('error', () => fail('Sotto streaming service is unavailable'));
|
|
57
|
+
upstream.on('close', code => {
|
|
58
|
+
if (closed)
|
|
59
|
+
return;
|
|
60
|
+
if (!finished || code !== 1000)
|
|
61
|
+
fail('Sotto streaming connection closed unexpectedly');
|
|
62
|
+
else {
|
|
63
|
+
clearTimeout(timer);
|
|
64
|
+
if (client.readyState === WebSocket.OPEN)
|
|
65
|
+
client.close(1000);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
upstream.on('message', (data, binary) => {
|
|
69
|
+
if (closed || binary || client.readyState !== WebSocket.OPEN)
|
|
70
|
+
return;
|
|
71
|
+
if (client.bufferedAmount > 1_000_000) {
|
|
72
|
+
fail('Transcription receiver is too slow', 1009);
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
client.send(data.toString());
|
|
76
|
+
});
|
|
77
|
+
client.on('message', (data, binary) => {
|
|
78
|
+
if (closed || finished || client.readyState !== WebSocket.OPEN)
|
|
79
|
+
return;
|
|
80
|
+
const length = data.byteLength;
|
|
81
|
+
if (upstream.readyState !== WebSocket.OPEN) {
|
|
82
|
+
fail('Sotto streaming service is not ready');
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
if (!binary) {
|
|
86
|
+
try {
|
|
87
|
+
const value = JSON.parse(data.toString());
|
|
88
|
+
if (!started &&
|
|
89
|
+
value.type === 'start' &&
|
|
90
|
+
value.format === 'float32' &&
|
|
91
|
+
Number.isInteger(value.sample_rate) &&
|
|
92
|
+
value.sample_rate >= 16000 &&
|
|
93
|
+
value.sample_rate <= 96000) {
|
|
94
|
+
started = true;
|
|
95
|
+
upstream.send(JSON.stringify({ type: 'start', format: 'float32', sample_rate: value.sample_rate }));
|
|
96
|
+
touch();
|
|
97
|
+
}
|
|
98
|
+
else if (started && value.type === 'stop') {
|
|
99
|
+
finished = true;
|
|
100
|
+
upstream.send(JSON.stringify({ type: 'stop' }));
|
|
101
|
+
}
|
|
102
|
+
else
|
|
103
|
+
fail('Invalid audio protocol', 1008);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
fail('Invalid audio protocol', 1008);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
else if (started &&
|
|
110
|
+
length > 0 &&
|
|
111
|
+
length % 4 === 0 &&
|
|
112
|
+
length <= 32768 &&
|
|
113
|
+
upstream.bufferedAmount < 1_000_000) {
|
|
114
|
+
upstream.send(data, { binary: true });
|
|
115
|
+
touch();
|
|
116
|
+
}
|
|
117
|
+
else
|
|
118
|
+
fail('Invalid audio frame or Sotto cannot keep up with microphone audio', 1009);
|
|
119
|
+
});
|
|
120
|
+
return fail;
|
|
121
|
+
}
|
|
122
|
+
export class SottoSttService {
|
|
123
|
+
metadata;
|
|
124
|
+
fetcher;
|
|
125
|
+
streams = new Set();
|
|
126
|
+
settings = { enabled: false, endpoint: DEFAULT_SOTTO_ENDPOINT };
|
|
127
|
+
/** Bumped by every update, so a slow enable probe cannot overwrite a newer choice. */
|
|
128
|
+
updates = 0;
|
|
129
|
+
constructor(metadata, fetcher = fetch) {
|
|
130
|
+
this.metadata = metadata;
|
|
131
|
+
this.fetcher = fetcher;
|
|
132
|
+
}
|
|
133
|
+
load() {
|
|
134
|
+
const saved = this.metadata.getSetting('sottoStt');
|
|
135
|
+
if (saved && typeof saved.enabled === 'boolean') {
|
|
136
|
+
try {
|
|
137
|
+
this.settings = { enabled: saved.enabled, endpoint: validateSottoEndpoint(saved.endpoint) };
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
/* Corrupt or obsolete settings must not enable an untrusted destination. */
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
getSettings() {
|
|
145
|
+
return { ...this.settings };
|
|
146
|
+
}
|
|
147
|
+
streamUrl() {
|
|
148
|
+
if (!this.settings.enabled)
|
|
149
|
+
throw invalid('Sotto transcription is disabled', 'SOTTO_DISABLED');
|
|
150
|
+
const port = Number(new URL(this.settings.endpoint).port) + 2;
|
|
151
|
+
if (port > 65535)
|
|
152
|
+
throw invalid('Invalid Sotto streaming port');
|
|
153
|
+
return `ws://127.0.0.1:${port}/asr`;
|
|
154
|
+
}
|
|
155
|
+
stream(client) {
|
|
156
|
+
const stop = proxySottoStream(client, this.streamUrl());
|
|
157
|
+
this.streams.add(stop);
|
|
158
|
+
client.once('close', () => this.streams.delete(stop));
|
|
159
|
+
}
|
|
160
|
+
async request(endpoint, path, init, timeoutMs) {
|
|
161
|
+
const response = await this.fetcher(`${endpoint}${path}`, {
|
|
162
|
+
...init,
|
|
163
|
+
redirect: 'error',
|
|
164
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
165
|
+
}).catch(error => {
|
|
166
|
+
throw invalid(`Cannot reach local Sotto: ${error.message}`, 'SOTTO_UNAVAILABLE');
|
|
167
|
+
});
|
|
168
|
+
if (!response.ok) {
|
|
169
|
+
const message = (await response.json().catch(() => ({})));
|
|
170
|
+
throw invalid(message.error || `Sotto returned HTTP ${response.status}`, 'SOTTO_UNAVAILABLE');
|
|
171
|
+
}
|
|
172
|
+
return response;
|
|
173
|
+
}
|
|
174
|
+
async update(input) {
|
|
175
|
+
if (!input || typeof input !== 'object')
|
|
176
|
+
throw invalid('enabled and endpoint are required');
|
|
177
|
+
const { enabled, endpoint } = input;
|
|
178
|
+
if (typeof enabled !== 'boolean')
|
|
179
|
+
throw invalid('enabled must be a boolean');
|
|
180
|
+
const next = { enabled, endpoint: validateSottoEndpoint(endpoint) };
|
|
181
|
+
const update = ++this.updates;
|
|
182
|
+
if (enabled) {
|
|
183
|
+
const capabilities = await (await this.request(next.endpoint, '/api/capabilities', { method: 'GET' }, 4000)).json();
|
|
184
|
+
if (capabilities?.stt !== true ||
|
|
185
|
+
capabilities?.format !== 'pcm-f32le' ||
|
|
186
|
+
capabilities?.sample_rate !== 16000 ||
|
|
187
|
+
capabilities?.stream_port !== Number(new URL(next.endpoint).port) + 2)
|
|
188
|
+
throw invalid('Sotto does not support live PCM transcription', 'SOTTO_UNAVAILABLE');
|
|
189
|
+
if (update !== this.updates)
|
|
190
|
+
throw invalid('Sotto settings were changed by a newer request', 'SOTTO_SETTINGS_SUPERSEDED');
|
|
191
|
+
}
|
|
192
|
+
this.metadata.setSetting('sottoStt', next);
|
|
193
|
+
const changed = !next.enabled || next.endpoint !== this.settings.endpoint;
|
|
194
|
+
this.settings = next;
|
|
195
|
+
if (changed) {
|
|
196
|
+
for (const stop of this.streams)
|
|
197
|
+
stop('Sotto 语音输入已关闭或服务地址已更改');
|
|
198
|
+
this.streams.clear();
|
|
199
|
+
}
|
|
200
|
+
return this.getSettings();
|
|
201
|
+
}
|
|
202
|
+
async transcribe(audio) {
|
|
203
|
+
if (!this.settings.enabled)
|
|
204
|
+
throw invalid('Sotto transcription is disabled', 'SOTTO_DISABLED');
|
|
205
|
+
if (!audio.length || audio.length % 4 || audio.length > 16000 * 4 * 60)
|
|
206
|
+
throw invalid('Expected at most 60 seconds of 16 kHz float32 PCM');
|
|
207
|
+
const response = await this.request(this.settings.endpoint, '/api/transcribe', {
|
|
208
|
+
method: 'POST',
|
|
209
|
+
headers: { 'content-type': 'application/octet-stream' },
|
|
210
|
+
body: new Uint8Array(audio)
|
|
211
|
+
}, 120000);
|
|
212
|
+
const body = (await response.json());
|
|
213
|
+
if (typeof body.text !== 'string')
|
|
214
|
+
throw invalid('Invalid response from Sotto', 'SOTTO_UNAVAILABLE');
|
|
215
|
+
return { text: body.text };
|
|
216
|
+
}
|
|
217
|
+
}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi2web",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.8",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "pi2web",
|
|
9
|
-
"version": "0.2.
|
|
9
|
+
"version": "0.2.8",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@earendil-works/chord": "1.1.0",
|
package/package.json
CHANGED
|
@@ -21,8 +21,26 @@ paths:
|
|
|
21
21
|
post: {operationId: createAgent, requestBody: {$ref: '#/components/requestBodies/Json'}, responses: {'201': {$ref: '#/components/responses/Success'}}}
|
|
22
22
|
/agents/{id}/timeline:
|
|
23
23
|
get: {operationId: agentTimeline, description: 'Older history as display items (see timelineItem in websocket.schema.json): about `limit` items before item `before`, never splitting a tool call from its result. Data {sessionId, items, total, start, hasMore}. Viewers get the newest page from the WebSocket focus snapshot. 404 when `before` is not on the current branch.', parameters: [{$ref: '#/components/parameters/Id'}, {name: limit, in: query, schema: {type: integer, minimum: 1, maximum: 500, default: 50}}, {name: before, in: query, schema: {type: string}}], responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
24
|
+
/agents/{id}/performance:
|
|
25
|
+
get: {operationId: agentPerformance, description: 'Current branch wall-clock model and tool spans recorded by recent runs; older sessions return only available tool durations. Data {sessionId, runs, legacyTools}. Parallel tools overlap.', parameters: [{$ref: '#/components/parameters/Id'}], responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
24
26
|
/agents/{id}/sessions/{sessionId}/items/{itemId}:
|
|
25
27
|
get: {operationId: agentTimelineItem, description: 'Full body of one timeline item, data {id, body}. Items never change, so it is served with Cache-Control private, max-age=31536000, immutable. 404 when the Agent is no longer on that session or branch.', parameters: [{$ref: '#/components/parameters/Id'}, {name: sessionId, in: path, required: true, schema: {type: string}}, {name: itemId, in: path, required: true, schema: {type: string}}], responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
28
|
+
/agents/{id}/system-prompt:
|
|
29
|
+
get: {operationId: getAgentSystemPrompt, parameters: [{$ref: '#/components/parameters/Id'}], responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
30
|
+
post: {operationId: changeAgentSystemPrompt, description: 'Only before the first conversation message; body {templateId?: string}, empty ID selects the Pi default.', parameters: [{$ref: '#/components/parameters/Id'}], requestBody: {$ref: '#/components/requestBodies/Json'}, responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
31
|
+
/sotto-stt:
|
|
32
|
+
get: {operationId: getSottoSttSettings, description: 'Backend-wide Sotto transcription setting; disabled by default, endpoint defaults to http://127.0.0.1:8766.', responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
33
|
+
put: {operationId: setSottoSttSettings, description: 'Body {enabled: boolean, endpoint: http://127.0.0.1:<port>}; probes local Sotto capabilities before enabling. Requires full access token.', requestBody: {$ref: '#/components/requestBodies/Json'}, responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
34
|
+
/transcribe:
|
|
35
|
+
post: {operationId: transcribeDraft, description: 'Backend-wide authenticated transcription, not tied to an Agent. Audio format follows the configured Sotto or STT service; returns {data:{text}}.', responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
36
|
+
/transcribe/ws:
|
|
37
|
+
get: {operationId: streamSottoTranscription, description: 'Authenticated backend-wide Sotto transcription WebSocket; start/PCM/stop protocol matches the Agent route.', responses: {'101': {description: 'WebSocket upgrade'}}}
|
|
38
|
+
/agents/{id}/transcribe:
|
|
39
|
+
post: {operationId: transcribeAgentDraft, description: 'When Sotto is enabled, body is 16 kHz mono float32 little-endian PCM (application/octet-stream; max 60 s); otherwise uses configured OpenAI-compatible STT audio. Returns {data:{text}} without submitting to the Agent.', parameters: [{$ref: '#/components/parameters/Id'}], responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
40
|
+
/agents/{id}/transcribe/ws:
|
|
41
|
+
get: {operationId: streamSottoDraft, description: 'Authenticated WebSocket: start {type:start,format:float32,sample_rate}, binary mono PCM chunks, stop {type:stop}; forwards Sotto ready/partial/final/error messages. Only when Sotto is enabled; never submits to Agent.', parameters: [{$ref: '#/components/parameters/Id'}], responses: {'101': {description: 'WebSocket upgrade'}}}
|
|
42
|
+
/agents/{id}/queue/move:
|
|
43
|
+
post: {operationId: moveQueuedMessage, description: 'Move one unconsumed message to the other queue during a run. Body {from: steering|followUp, index: integer, expected: string}; rejects stale entries.', parameters: [{$ref: '#/components/parameters/Id'}], requestBody: {$ref: '#/components/requestBodies/Json'}, responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
26
44
|
/agents/{id}/{action}:
|
|
27
45
|
get: {operationId: getAgentResource, parameters: [{$ref: '#/components/parameters/Id'}, {name: action, in: path, required: true, schema: {enum: [state,capabilities,session]}}], responses: {'200': {$ref: '#/components/responses/Success'}}}
|
|
28
46
|
post: {operationId: agentCommand, parameters: [{$ref: '#/components/parameters/Id'}, {name: action, in: path, required: true, schema: {enum: [prompt,steer,follow-up,abort,compact,model,thinking,session-name,navigate,fork,extension-response]}}], requestBody: {$ref: '#/components/requestBodies/Json'}, responses: {'200': {$ref: '#/components/responses/Success'}}}
|