switchroom 0.16.29 → 0.16.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +101 -84
- package/dist/auth-broker/index.js +100 -83
- package/dist/cli/autoaccept-poll.js +8 -8
- package/dist/cli/drive-write-pretool.mjs +10 -10
- package/dist/cli/notion-write-pretool.mjs +102 -85
- package/dist/cli/skill-validate-pretool.mjs +91 -91
- package/dist/cli/switchroom.js +62124 -60406
- package/dist/cli/ui/index.html +877 -214
- package/dist/host-control/main.js +1000 -256
- package/dist/vault/approvals/kernel-server.js +141 -121
- package/dist/vault/broker/server.js +163 -143
- package/examples/minimal.yaml +1 -1
- package/examples/switchroom.yaml +1 -1
- package/package.json +3 -2
- package/profiles/_shared/agent-self-service.md.hbs +7 -2
- package/profiles/_shared/reply-discipline.md.hbs +9 -0
- package/skills/switchroom-status/SKILL.md +1 -1
- package/telegram-plugin/auth-snapshot-format.ts +173 -67
- package/telegram-plugin/auto-fallback-fleet.ts +3 -6
- package/telegram-plugin/bridge/bridge.ts +2 -1
- package/telegram-plugin/card-format.ts +59 -3
- package/telegram-plugin/credits-watch.ts +4 -7
- package/telegram-plugin/dist/bridge/bridge.js +132 -114
- package/telegram-plugin/dist/gateway/gateway.js +4456 -1591
- package/telegram-plugin/dist/server.js +180 -163
- package/telegram-plugin/format.ts +551 -20
- package/telegram-plugin/gateway/approval-card.ts +7 -14
- package/telegram-plugin/gateway/approvals-commands.ts +6 -9
- package/telegram-plugin/gateway/auth-command.ts +35 -38
- package/telegram-plugin/gateway/boot-card.ts +7 -2
- package/telegram-plugin/gateway/chat-id-fallback.ts +46 -0
- package/telegram-plugin/gateway/config-approval-handler.ts +6 -9
- package/telegram-plugin/gateway/diff-preview-card.ts +3 -6
- package/telegram-plugin/gateway/gateway.ts +1524 -129
- package/telegram-plugin/gateway/ipc-protocol.ts +66 -2
- package/telegram-plugin/gateway/ipc-server.ts +91 -1
- package/telegram-plugin/gateway/linear-activity.ts +2 -5
- package/telegram-plugin/gateway/model-command.ts +173 -19
- package/telegram-plugin/gateway/obligation-turn-end.ts +27 -0
- package/telegram-plugin/gateway/permission-card-store.ts +104 -0
- package/telegram-plugin/gateway/permission-timeout.ts +25 -6
- package/telegram-plugin/gateway/status-pin-store.ts +302 -0
- package/telegram-plugin/gateway/turns-jsonl-rotate.ts +30 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +12 -1
- package/telegram-plugin/gateway/vault-grant-inbound-builders.ts +35 -0
- package/telegram-plugin/gateway/vault-request-access-card.ts +61 -0
- package/telegram-plugin/history.ts +17 -7
- package/telegram-plugin/hooks/tool-label-pretool.d.mts +12 -0
- package/telegram-plugin/hooks/tool-label-pretool.mjs +54 -16
- package/telegram-plugin/idle-footer.ts +2 -2
- package/telegram-plugin/issues-card.ts +12 -7
- package/telegram-plugin/model-unavailable.ts +3 -6
- package/telegram-plugin/operator-events.ts +4 -6
- package/telegram-plugin/package.json +1 -1
- package/telegram-plugin/quota-check.ts +2 -2
- package/telegram-plugin/quota-watch.ts +7 -10
- package/telegram-plugin/server.ts +3 -1
- package/telegram-plugin/session-tail.ts +47 -1
- package/telegram-plugin/status-pin-driver.ts +102 -0
- package/telegram-plugin/status-pin.ts +76 -0
- package/telegram-plugin/stream-reply-handler.ts +33 -2
- package/telegram-plugin/subagent-watcher.ts +6 -3
- package/telegram-plugin/tests/always-allow-grant.test.ts +34 -2
- package/telegram-plugin/tests/auth-command-format2.test.ts +6 -2
- package/telegram-plugin/tests/auth-command-vernacular.test.ts +18 -0
- package/telegram-plugin/tests/auth-snapshot-format.test.ts +204 -45
- package/telegram-plugin/tests/card-format.test.ts +79 -0
- package/telegram-plugin/tests/chat-id-fallback.test.ts +74 -0
- package/telegram-plugin/tests/claude-code-event-contract.test.ts +151 -0
- package/telegram-plugin/tests/codespan-escaping-golden.test.ts +166 -0
- package/telegram-plugin/tests/credits-watch.test.ts +18 -0
- package/telegram-plugin/tests/format-consistency.test.ts +223 -0
- package/telegram-plugin/tests/formatting-parse-regression.test.ts +272 -0
- package/telegram-plugin/tests/formatting-torture-set.ts +218 -0
- package/telegram-plugin/tests/history.test.ts +38 -2
- package/telegram-plugin/tests/idle-footer.test.ts +53 -9
- package/telegram-plugin/tests/ipc-server-validate-rollout-status.test.ts +66 -0
- package/telegram-plugin/tests/issues-card.test.ts +24 -0
- package/telegram-plugin/tests/model-command.test.ts +213 -47
- package/telegram-plugin/tests/obligation-turn-end.test.ts +76 -0
- package/telegram-plugin/tests/paragraph-normalizer.test.ts +384 -3
- package/telegram-plugin/tests/permission-card-single-edit.test.ts +66 -0
- package/telegram-plugin/tests/permission-card-store.test.ts +83 -0
- package/telegram-plugin/tests/permission-timeout.test.ts +37 -6
- package/telegram-plugin/tests/permission-verdict-resume-guard.test.ts +23 -6
- package/telegram-plugin/tests/quota-check.test.ts +9 -2
- package/telegram-plugin/tests/quota-watch.test.ts +7 -7
- package/telegram-plugin/tests/rich-markdown-oracle.ts +469 -0
- package/telegram-plugin/tests/rollout-status-wiring.test.ts +81 -0
- package/telegram-plugin/tests/session-tail.test.ts +91 -0
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +294 -0
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +210 -0
- package/telegram-plugin/tests/status-pin-store.test.ts +450 -0
- package/telegram-plugin/tests/status-pin.test.ts +202 -0
- package/telegram-plugin/tests/status-vocabulary-unification.test.ts +125 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +39 -0
- package/telegram-plugin/tests/telegram-format.test.ts +121 -8
- package/telegram-plugin/tests/text-voice-scrub.test.ts +142 -22
- package/telegram-plugin/tests/tool-activity-summary.test.ts +88 -25
- package/telegram-plugin/tests/tts-normalize.test.ts +242 -0
- package/telegram-plugin/tests/turns-jsonl-rotate.test.ts +39 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +12 -0
- package/telegram-plugin/tests/vault-grant-inbound-builders.test.ts +54 -0
- package/telegram-plugin/tests/vault-request-access-card.test.ts +94 -0
- package/telegram-plugin/tests/vault-request-access-tool.test.ts +24 -0
- package/telegram-plugin/tests/voice-normalize-text.test.ts +256 -0
- package/telegram-plugin/tests/voice-ondemand.test.ts +299 -0
- package/telegram-plugin/tests/voice-out-one-send.test.ts +153 -0
- package/telegram-plugin/tests/voice-presynth.test.ts +437 -0
- package/telegram-plugin/tests/voice-synthesize-sidecar.test.ts +352 -0
- package/telegram-plugin/tests/voice-transcribe-sidecar.test.ts +332 -0
- package/telegram-plugin/tests/voice-transcribe.test.ts +188 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +77 -10
- package/telegram-plugin/text-voice-scrub.ts +68 -18
- package/telegram-plugin/tool-activity-summary.ts +31 -116
- package/telegram-plugin/tts-normalize.ts +377 -0
- package/telegram-plugin/uat/driver.ts +474 -17
- package/telegram-plugin/uat/scenarios/jtbd-model-litellm-sr-dm.test.ts +34 -14
- package/telegram-plugin/uat/scenarios/jtbd-multipart-render-dm.test.ts +169 -0
- package/telegram-plugin/uat/scenarios/jtbd-narration-intent-dm.test.ts +134 -0
- package/telegram-plugin/uat/scenarios/jtbd-rich-formatting-render-dm.test.ts +254 -0
- package/telegram-plugin/uat/scenarios/jtbd-status-phase-transitions-dm.test.ts +109 -0
- package/telegram-plugin/uat/uat-driver.test.ts +297 -0
- package/telegram-plugin/voice-normalize-text.ts +340 -0
- package/telegram-plugin/voice-ondemand.ts +289 -0
- package/telegram-plugin/voice-presynth.ts +242 -0
- package/telegram-plugin/voice-synthesize-sidecar.ts +259 -0
- package/telegram-plugin/voice-synthesize.ts +128 -0
- package/telegram-plugin/voice-transcribe-sidecar.ts +176 -0
- package/telegram-plugin/worker-activity-feed.ts +22 -5
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spoken-reply synthesis (TTS) via the LOCAL voice sidecar (voice PR-C2).
|
|
3
|
+
*
|
|
4
|
+
* The mirror of `voice-transcribe-sidecar.ts` for the OUTBOUND direction.
|
|
5
|
+
* Where that module POSTs audio bytes to `/stt` and gets text back, this
|
|
6
|
+
* one POSTs the agent's reply text to `/tts` (docker/voice-sidecar/server.py,
|
|
7
|
+
* Kokoro on a local GPU/CPU) and gets OGG/Opus bytes back — ready to hand
|
|
8
|
+
* straight to Telegram `sendVoice`. Used when the host's voice-engine
|
|
9
|
+
* verdict is `local` (no third-party TTS key, vision #3 + #4).
|
|
10
|
+
*
|
|
11
|
+
* Reachability: the gateway runs inside the agent container
|
|
12
|
+
* (`network_mode: host`), so the sidecar's published `0.0.0.0:18900` is
|
|
13
|
+
* reachable at loopback `127.0.0.1:18900` (compose maps host 18900 → the
|
|
14
|
+
* sidecar's in-container 8126). See src/agents/compose.ts
|
|
15
|
+
* (VOICE_SIDECAR_HOST_PORT) for the published-port contract.
|
|
16
|
+
*
|
|
17
|
+
* Contract MIRRORS `transcribeViaSidecar`:
|
|
18
|
+
* - takes the text + a shared-secret token as args (no env reads, no
|
|
19
|
+
* file IO) so it's unit-testable against a mocked fetch;
|
|
20
|
+
* - never throws — every failure becomes a structured
|
|
21
|
+
* `{ ok: false, reason, detail }`, and the gateway caller falls back
|
|
22
|
+
* to a text-only reply on any non-ok outcome (voice is best-effort).
|
|
23
|
+
*
|
|
24
|
+
* The sidecar's /tts response shape (server.py):
|
|
25
|
+
* 200 audio/ogg (OGG/Opus, mono — raw bytes, NOT JSON)
|
|
26
|
+
* 4xx/5xx { ok: false, reason, detail }
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/** Default loopback base URL for the sidecar (host-network agent). */
|
|
30
|
+
export const DEFAULT_SIDECAR_BASE_URL = 'http://127.0.0.1:18900'
|
|
31
|
+
|
|
32
|
+
/** Defence-in-depth ceiling on a SINGLE /tts request.
|
|
33
|
+
*
|
|
34
|
+
* The sidecar now accepts arbitrarily long text and chunks + concatenates
|
|
35
|
+
* internally on the GPU, returning a single ogg/opus file (one voice note
|
|
36
|
+
* per response). This client cap is therefore only a sanity ceiling against
|
|
37
|
+
* a pathological reply, not a per-note splitter — the gateway sends the
|
|
38
|
+
* whole normalized reply in one call. Kept comfortably above any realistic
|
|
39
|
+
* chat reply. (Contract with docker/voice-sidecar/server.py: endpoint path,
|
|
40
|
+
* X-Voice-Token header, and {text, voice?, format?} JSON shape UNCHANGED.) */
|
|
41
|
+
export const SIDECAR_TTS_MAX_CHARS = 100_000
|
|
42
|
+
|
|
43
|
+
/** TTS playback-speed bounds (mirrors the sidecar clamp in server.py). */
|
|
44
|
+
export const SIDECAR_TTS_SPEED_MIN = 0.5
|
|
45
|
+
export const SIDECAR_TTS_SPEED_MAX = 2.0
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Coerce an incoming speed to a float in [SIDECAR_TTS_SPEED_MIN,
|
|
49
|
+
* SIDECAR_TTS_SPEED_MAX]. Non-finite / non-numeric input falls back to
|
|
50
|
+
* `fallback` (default 1.0). Out-of-range numbers are clamped to the nearest
|
|
51
|
+
* bound. Mirrors the sidecar's `_clamp_speed` so the wire value is already
|
|
52
|
+
* valid before it leaves the gateway.
|
|
53
|
+
*/
|
|
54
|
+
export function clampTtsSpeed(value: unknown, fallback = 1.0): number {
|
|
55
|
+
// Only a real, finite number is a speed. null/undefined/boolean/string all
|
|
56
|
+
// fall back (Number(null) is 0 and Number('') is 0 — both would otherwise
|
|
57
|
+
// masquerade as a valid slow speed).
|
|
58
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) return fallback
|
|
59
|
+
return Math.max(SIDECAR_TTS_SPEED_MIN, Math.min(SIDECAR_TTS_SPEED_MAX, value))
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Split already-stripped plain text into sequential TTS chunks, each ≤
|
|
64
|
+
* `chunkChars`, preferring sentence then word boundaries so a voice note
|
|
65
|
+
* never cuts mid-sentence. A run with no boundary that fits (e.g. one
|
|
66
|
+
* enormous URL) is hard-sliced at the cap so a chunk can't exceed the
|
|
67
|
+
* engine limit. PR-C2 (long-reply voice chunking — the full answer is
|
|
68
|
+
* spoken across several notes instead of truncated to text-only).
|
|
69
|
+
*/
|
|
70
|
+
export function chunkTtsText(text: string, chunkChars: number): string[] {
|
|
71
|
+
const cap = Math.max(1, chunkChars)
|
|
72
|
+
if (text.length <= cap) {
|
|
73
|
+
const t = text.trim()
|
|
74
|
+
return t.length > 0 ? [t] : []
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Sentence-ish units: keep the terminator with the sentence. Falls back
|
|
78
|
+
// to the whole string when there's no punctuation to split on.
|
|
79
|
+
const sentences = text.match(/[^.!?]+[.!?]+(?:\s|$)|[^.!?]+$/g) ?? [text]
|
|
80
|
+
const chunks: string[] = []
|
|
81
|
+
let buf = ''
|
|
82
|
+
|
|
83
|
+
const flush = (): void => {
|
|
84
|
+
const t = buf.trim()
|
|
85
|
+
if (t.length > 0) chunks.push(t)
|
|
86
|
+
buf = ''
|
|
87
|
+
}
|
|
88
|
+
const hardSlice = (s: string): void => {
|
|
89
|
+
// No boundary fits the cap — split on word boundaries, then hard-cut
|
|
90
|
+
// any single token still over the cap.
|
|
91
|
+
let rest = s
|
|
92
|
+
while (rest.length > cap) {
|
|
93
|
+
let cut = rest.lastIndexOf(' ', cap)
|
|
94
|
+
if (cut <= 0) cut = cap
|
|
95
|
+
chunks.push(rest.slice(0, cut).trim())
|
|
96
|
+
rest = rest.slice(cut).trimStart()
|
|
97
|
+
}
|
|
98
|
+
buf = rest
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
for (const sentence of sentences) {
|
|
102
|
+
if (sentence.length > cap) {
|
|
103
|
+
// The sentence itself overflows — flush what we have, then slice it.
|
|
104
|
+
flush()
|
|
105
|
+
hardSlice(sentence)
|
|
106
|
+
continue
|
|
107
|
+
}
|
|
108
|
+
if (buf.length + sentence.length > cap) flush()
|
|
109
|
+
buf += sentence
|
|
110
|
+
}
|
|
111
|
+
flush()
|
|
112
|
+
return chunks
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** A successful synthesis: the OGG/Opus voice-note bytes plus sidecar meta. */
|
|
116
|
+
export interface SynthesizeResult {
|
|
117
|
+
ok: true
|
|
118
|
+
/** OGG/Opus (mono) bytes — hand directly to Telegram sendVoice. */
|
|
119
|
+
audio: Uint8Array
|
|
120
|
+
/** Wall-clock (or sidecar-reported) synthesis time in ms. */
|
|
121
|
+
durationMs: number
|
|
122
|
+
/** Audio length in seconds, if the sidecar reported it. */
|
|
123
|
+
audioSeconds: number | null
|
|
124
|
+
/** The voice actually used (echoed by the sidecar). */
|
|
125
|
+
voice: string | null
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export interface SynthesizeFailure {
|
|
129
|
+
ok: false
|
|
130
|
+
reason: string
|
|
131
|
+
detail?: string
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export type SynthesizeOutcome = SynthesizeResult | SynthesizeFailure
|
|
135
|
+
|
|
136
|
+
export interface SidecarSynthesizeArgs {
|
|
137
|
+
/** Shared secret sent in the X-Voice-Token header (vault:voice/sidecar-token). */
|
|
138
|
+
token: string
|
|
139
|
+
/** Plain text to speak. Strip markdown before calling — Kokoro reads it literally. */
|
|
140
|
+
text: string
|
|
141
|
+
/** Optional engine-specific voice id; the sidecar has its own default. */
|
|
142
|
+
voice?: string
|
|
143
|
+
/** Optional playback speed. Clamped to [0.5, 2.0] before send; when unset
|
|
144
|
+
* the sidecar applies its own 1.0 default. */
|
|
145
|
+
speed?: number
|
|
146
|
+
/** Base URL for the sidecar. Defaults to DEFAULT_SIDECAR_BASE_URL. */
|
|
147
|
+
baseUrl?: string
|
|
148
|
+
/** Per-call timeout in ms. Default 60s — Kokoro on CPU can run a few
|
|
149
|
+
* seconds for a long memo; the sidecar's own per-request timeout is 60s. */
|
|
150
|
+
timeoutMs?: number
|
|
151
|
+
/** Override fetch for tests. */
|
|
152
|
+
fetchImpl?: typeof fetch
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Synthesize a single reply against the local sidecar. Resolves to a
|
|
157
|
+
* SynthesizeOutcome — the caller branches on `.ok`. Never throws.
|
|
158
|
+
*
|
|
159
|
+
* Reason codes mirror the STT path where they overlap:
|
|
160
|
+
* - 'no-token' — empty shared secret
|
|
161
|
+
* - 'empty-text' — empty/whitespace text
|
|
162
|
+
* - 'text-too-long' — exceeds SIDECAR_TTS_MAX_CHARS
|
|
163
|
+
* - 'http-<n>' — non-2xx HTTP status code
|
|
164
|
+
* - 'fetch-failed' — network error before any response
|
|
165
|
+
* - 'empty-audio' — 2xx but no body bytes
|
|
166
|
+
* - 'timeout' — exceeded the per-call timeout
|
|
167
|
+
*/
|
|
168
|
+
export async function synthesizeViaSidecar(
|
|
169
|
+
args: SidecarSynthesizeArgs,
|
|
170
|
+
): Promise<SynthesizeOutcome> {
|
|
171
|
+
if (!args.token || args.token.length === 0) {
|
|
172
|
+
return { ok: false, reason: 'no-token' }
|
|
173
|
+
}
|
|
174
|
+
const text = args.text?.trim() ?? ''
|
|
175
|
+
if (text.length === 0) {
|
|
176
|
+
return { ok: false, reason: 'empty-text' }
|
|
177
|
+
}
|
|
178
|
+
if (text.length > SIDECAR_TTS_MAX_CHARS) {
|
|
179
|
+
return {
|
|
180
|
+
ok: false,
|
|
181
|
+
reason: 'text-too-long',
|
|
182
|
+
detail: `${text.length} chars (limit ${SIDECAR_TTS_MAX_CHARS})`,
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const fetchFn = args.fetchImpl ?? fetch
|
|
187
|
+
const baseUrl = (args.baseUrl ?? DEFAULT_SIDECAR_BASE_URL).replace(/\/$/, '')
|
|
188
|
+
const timeoutMs = args.timeoutMs ?? 60_000
|
|
189
|
+
const controller = new AbortController()
|
|
190
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
|
191
|
+
|
|
192
|
+
// JSON body: server.py reads `text` (str) + optional `voice` + `speed` +
|
|
193
|
+
// `format`.
|
|
194
|
+
const body: { text: string; voice?: string; speed?: number; format: 'ogg' } = {
|
|
195
|
+
text,
|
|
196
|
+
format: 'ogg',
|
|
197
|
+
}
|
|
198
|
+
if (args.voice) body.voice = args.voice
|
|
199
|
+
if (args.speed !== undefined) body.speed = clampTtsSpeed(args.speed)
|
|
200
|
+
|
|
201
|
+
const startedAt = Date.now()
|
|
202
|
+
let res: Response
|
|
203
|
+
try {
|
|
204
|
+
res = await fetchFn(`${baseUrl}/tts`, {
|
|
205
|
+
method: 'POST',
|
|
206
|
+
headers: {
|
|
207
|
+
'X-Voice-Token': args.token,
|
|
208
|
+
'Content-Type': 'application/json',
|
|
209
|
+
},
|
|
210
|
+
body: JSON.stringify(body),
|
|
211
|
+
signal: controller.signal,
|
|
212
|
+
})
|
|
213
|
+
} catch (err) {
|
|
214
|
+
clearTimeout(timer)
|
|
215
|
+
const msg = err instanceof Error ? err.message : String(err)
|
|
216
|
+
if (msg.includes('aborted') || /timeout/i.test(msg)) {
|
|
217
|
+
return { ok: false, reason: 'timeout', detail: msg }
|
|
218
|
+
}
|
|
219
|
+
return { ok: false, reason: 'fetch-failed', detail: msg }
|
|
220
|
+
}
|
|
221
|
+
clearTimeout(timer)
|
|
222
|
+
|
|
223
|
+
if (!res.ok) {
|
|
224
|
+
// Error bodies are JSON ({ ok:false, reason, detail }); echo the reason.
|
|
225
|
+
const errText = await res.text().catch(() => '')
|
|
226
|
+
let reason = `http-${res.status}`
|
|
227
|
+
let detail: string | undefined = errText.slice(0, 200)
|
|
228
|
+
try {
|
|
229
|
+
const parsed = JSON.parse(errText) as { reason?: unknown; detail?: unknown }
|
|
230
|
+
if (typeof parsed.reason === 'string') reason = parsed.reason
|
|
231
|
+
if (typeof parsed.detail === 'string') detail = parsed.detail.slice(0, 200)
|
|
232
|
+
} catch {
|
|
233
|
+
// not JSON — keep the http-<n> reason + raw text detail
|
|
234
|
+
}
|
|
235
|
+
return { ok: false, reason, detail }
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const buf = new Uint8Array(await res.arrayBuffer())
|
|
239
|
+
if (buf.length === 0) {
|
|
240
|
+
return { ok: false, reason: 'empty-audio' }
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Sidecar echoes synthesis meta as response headers (server.py). A
|
|
244
|
+
// MISSING header reads as null — guard before Number() (Number(null) is 0,
|
|
245
|
+
// which would masquerade as a real 0-second clip).
|
|
246
|
+
const rawDuration = res.headers.get('X-Voice-Duration-Ms')
|
|
247
|
+
const rawAudioSec = res.headers.get('X-Voice-Audio-Seconds')
|
|
248
|
+
const hdrDuration = rawDuration != null ? Number(rawDuration) : NaN
|
|
249
|
+
const hdrAudioSec = rawAudioSec != null ? Number(rawAudioSec) : NaN
|
|
250
|
+
return {
|
|
251
|
+
ok: true,
|
|
252
|
+
audio: buf,
|
|
253
|
+
durationMs: Number.isFinite(hdrDuration) && hdrDuration > 0
|
|
254
|
+
? hdrDuration
|
|
255
|
+
: Date.now() - startedAt,
|
|
256
|
+
audioSeconds: Number.isFinite(hdrAudioSec) ? hdrAudioSec : null,
|
|
257
|
+
voice: res.headers.get('X-Voice-Voice'),
|
|
258
|
+
}
|
|
259
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spoken-reply synthesis (TTS) via the OpenAI cloud API (voice PR-C2).
|
|
3
|
+
*
|
|
4
|
+
* The cloud twin of `voice-synthesize-sidecar.ts` (local Kokoro sidecar),
|
|
5
|
+
* and the OUTBOUND mirror of `voice-transcribe.ts` (OpenAI Whisper STT).
|
|
6
|
+
* Used only when the operator opts an agent into `voice_out.engine: openai`
|
|
7
|
+
* — an honest-exception third-party path gated on an `api_key` vault ref
|
|
8
|
+
* (the local sidecar is the subscription-honest default).
|
|
9
|
+
*
|
|
10
|
+
* Pure: takes the text + resolved API key as args (no env reads, no file
|
|
11
|
+
* IO) so it's unit-testable against a mocked fetch. Never throws — every
|
|
12
|
+
* failure becomes a structured `{ ok: false, reason, detail }` and the
|
|
13
|
+
* gateway caller falls back to a text-only reply (voice is best-effort).
|
|
14
|
+
*
|
|
15
|
+
* Requests OGG/Opus directly (response_format: 'opus') so the bytes hand
|
|
16
|
+
* straight to Telegram sendVoice with no transcode — same shape the local
|
|
17
|
+
* sidecar returns.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const OPENAI_TTS_URL = 'https://api.openai.com/v1/audio/speech'
|
|
21
|
+
|
|
22
|
+
/** Default model + voice when the operator doesn't pin one. */
|
|
23
|
+
export const DEFAULT_OPENAI_TTS_MODEL = 'gpt-4o-mini-tts'
|
|
24
|
+
export const DEFAULT_OPENAI_TTS_VOICE = 'alloy'
|
|
25
|
+
|
|
26
|
+
export interface OpenAiSynthesizeResult {
|
|
27
|
+
ok: true
|
|
28
|
+
/** OGG/Opus bytes — hand directly to Telegram sendVoice. */
|
|
29
|
+
audio: Uint8Array
|
|
30
|
+
/** Wall-clock synthesis time in ms. */
|
|
31
|
+
durationMs: number
|
|
32
|
+
/** The voice actually requested. */
|
|
33
|
+
voice: string
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface OpenAiSynthesizeError {
|
|
37
|
+
ok: false
|
|
38
|
+
reason: string
|
|
39
|
+
detail?: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type OpenAiSynthesizeOutcome = OpenAiSynthesizeResult | OpenAiSynthesizeError
|
|
43
|
+
|
|
44
|
+
export interface OpenAiSynthesizeArgs {
|
|
45
|
+
/** Resolved OpenAI API key (already materialized from the vault). */
|
|
46
|
+
apiKey: string
|
|
47
|
+
/** Plain text to speak. Strip markdown before calling. */
|
|
48
|
+
text: string
|
|
49
|
+
/** Voice id (e.g. 'alloy', 'nova'). Defaults to DEFAULT_OPENAI_TTS_VOICE. */
|
|
50
|
+
voice?: string
|
|
51
|
+
/** Model. Defaults to DEFAULT_OPENAI_TTS_MODEL. */
|
|
52
|
+
model?: string
|
|
53
|
+
/** Per-call timeout in ms. Default 60s. */
|
|
54
|
+
timeoutMs?: number
|
|
55
|
+
/** Override fetch for tests. */
|
|
56
|
+
fetchImpl?: typeof fetch
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Synthesize a single reply via OpenAI TTS. Resolves to an outcome — the
|
|
61
|
+
* caller branches on `.ok`. Never throws.
|
|
62
|
+
*
|
|
63
|
+
* Reason codes:
|
|
64
|
+
* - 'no-api-key' — empty key
|
|
65
|
+
* - 'empty-text' — empty/whitespace text
|
|
66
|
+
* - 'http-<n>' — non-2xx HTTP status code
|
|
67
|
+
* - 'fetch-failed'— network error before any response
|
|
68
|
+
* - 'empty-audio' — 2xx but no body bytes
|
|
69
|
+
* - 'timeout' — exceeded the per-call timeout
|
|
70
|
+
*/
|
|
71
|
+
export async function synthesizeViaOpenAi(
|
|
72
|
+
args: OpenAiSynthesizeArgs,
|
|
73
|
+
): Promise<OpenAiSynthesizeOutcome> {
|
|
74
|
+
if (!args.apiKey || args.apiKey.length === 0) {
|
|
75
|
+
return { ok: false, reason: 'no-api-key' }
|
|
76
|
+
}
|
|
77
|
+
const text = args.text?.trim() ?? ''
|
|
78
|
+
if (text.length === 0) {
|
|
79
|
+
return { ok: false, reason: 'empty-text' }
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const fetchFn = args.fetchImpl ?? fetch
|
|
83
|
+
const voice = args.voice && args.voice.length > 0 ? args.voice : DEFAULT_OPENAI_TTS_VOICE
|
|
84
|
+
const model = args.model && args.model.length > 0 ? args.model : DEFAULT_OPENAI_TTS_MODEL
|
|
85
|
+
const timeoutMs = args.timeoutMs ?? 60_000
|
|
86
|
+
const controller = new AbortController()
|
|
87
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
|
88
|
+
|
|
89
|
+
const startedAt = Date.now()
|
|
90
|
+
let res: Response
|
|
91
|
+
try {
|
|
92
|
+
res = await fetchFn(OPENAI_TTS_URL, {
|
|
93
|
+
method: 'POST',
|
|
94
|
+
headers: {
|
|
95
|
+
Authorization: `Bearer ${args.apiKey}`,
|
|
96
|
+
'Content-Type': 'application/json',
|
|
97
|
+
},
|
|
98
|
+
// response_format 'opus' → OGG/Opus, ready for Telegram sendVoice.
|
|
99
|
+
body: JSON.stringify({ model, voice, input: text, response_format: 'opus' }),
|
|
100
|
+
signal: controller.signal,
|
|
101
|
+
})
|
|
102
|
+
} catch (err) {
|
|
103
|
+
clearTimeout(timer)
|
|
104
|
+
const msg = err instanceof Error ? err.message : String(err)
|
|
105
|
+
if (msg.includes('aborted') || /timeout/i.test(msg)) {
|
|
106
|
+
return { ok: false, reason: 'timeout', detail: msg }
|
|
107
|
+
}
|
|
108
|
+
return { ok: false, reason: 'fetch-failed', detail: msg }
|
|
109
|
+
}
|
|
110
|
+
clearTimeout(timer)
|
|
111
|
+
|
|
112
|
+
if (!res.ok) {
|
|
113
|
+
const body = await res.text().catch(() => '')
|
|
114
|
+
return { ok: false, reason: `http-${res.status}`, detail: body.slice(0, 200) }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const buf = new Uint8Array(await res.arrayBuffer())
|
|
118
|
+
if (buf.length === 0) {
|
|
119
|
+
return { ok: false, reason: 'empty-audio' }
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
ok: true,
|
|
124
|
+
audio: buf,
|
|
125
|
+
durationMs: Date.now() - startedAt,
|
|
126
|
+
voice,
|
|
127
|
+
}
|
|
128
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Voice-message transcription via the LOCAL GPU STT sidecar (voice PR-B2).
|
|
3
|
+
*
|
|
4
|
+
* The cloud twin of this module is `voice-transcribe.ts` (OpenAI Whisper).
|
|
5
|
+
* This one POSTs the downloaded audio bytes to the in-fleet
|
|
6
|
+
* `voice-sidecar` service (docker/voice-sidecar/server.py, faster-whisper
|
|
7
|
+
* on a local GPU) when the host's voice-engine verdict is `local` — no
|
|
8
|
+
* third-party STT key (vision #3, subscription-honest) and no cloud
|
|
9
|
+
* dependency (vision #4, always-available).
|
|
10
|
+
*
|
|
11
|
+
* Reachability: the gateway runs inside the agent container, which is
|
|
12
|
+
* `network_mode: host`, so the sidecar's published `0.0.0.0:18900` is
|
|
13
|
+
* reachable at loopback `127.0.0.1:18900` (compose maps host 18900 → the
|
|
14
|
+
* sidecar's in-container 8126). See src/agents/compose.ts
|
|
15
|
+
* (VOICE_SIDECAR_HOST_PORT) for the published-port contract.
|
|
16
|
+
*
|
|
17
|
+
* Contract MIRRORS `transcribeViaWhisper`:
|
|
18
|
+
* - takes audio bytes + a shared-secret token as args (no env reads,
|
|
19
|
+
* no file IO) so it's unit-testable against a mocked fetch;
|
|
20
|
+
* - never throws — every failure becomes a structured
|
|
21
|
+
* `{ ok: false, reason, detail }`, and the gateway caller falls back
|
|
22
|
+
* to the legacy "(voice message)" envelope on any non-ok outcome.
|
|
23
|
+
*
|
|
24
|
+
* The sidecar's /stt response shape (server.py):
|
|
25
|
+
* 200 { ok: true, text, language?, durationMs, audioSeconds }
|
|
26
|
+
* 4xx/5xx { ok: false, reason, detail }
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import type {
|
|
30
|
+
TranscribeOutcome,
|
|
31
|
+
TranscribeResult,
|
|
32
|
+
} from './voice-transcribe.js'
|
|
33
|
+
|
|
34
|
+
/** Default loopback base URL for the sidecar (host-network agent). */
|
|
35
|
+
export const DEFAULT_SIDECAR_BASE_URL = 'http://127.0.0.1:18900'
|
|
36
|
+
|
|
37
|
+
/** The sidecar caps inputs at 25MB before transcode (server.py MAX_BYTES);
|
|
38
|
+
* reject before the round-trip to match the cloud path's pre-flight. */
|
|
39
|
+
export const SIDECAR_MAX_BYTES = 25 * 1024 * 1024
|
|
40
|
+
|
|
41
|
+
export interface SidecarTranscribeArgs {
|
|
42
|
+
/** Shared secret sent in the X-Voice-Token header (vault:voice/sidecar-token). */
|
|
43
|
+
token: string
|
|
44
|
+
audio: Uint8Array
|
|
45
|
+
/** Filename hint for the multipart body (matches the actual content). */
|
|
46
|
+
filename: string
|
|
47
|
+
/** Optional ISO-639-1 language hint to skip detection. */
|
|
48
|
+
language?: string
|
|
49
|
+
/** Base URL for the sidecar. Defaults to DEFAULT_SIDECAR_BASE_URL. */
|
|
50
|
+
baseUrl?: string
|
|
51
|
+
/** Per-call timeout in ms. Default 90s — local whisper can run longer
|
|
52
|
+
* than the cloud API on a cold model / long memo (the sidecar's own
|
|
53
|
+
* per-request timeout is 60s by default; we give the HTTP leg slack). */
|
|
54
|
+
timeoutMs?: number
|
|
55
|
+
/** Override fetch for tests. */
|
|
56
|
+
fetchImpl?: typeof fetch
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Run a single transcription against the local sidecar. Resolves to a
|
|
61
|
+
* TranscribeOutcome — the caller branches on `.ok`. Never throws.
|
|
62
|
+
*
|
|
63
|
+
* Reason codes mirror the cloud path where they overlap:
|
|
64
|
+
* - 'no-token' — empty shared secret (mirrors 'no-api-key')
|
|
65
|
+
* - 'audio-too-short' — empty audio
|
|
66
|
+
* - 'audio-too-large' — exceeds SIDECAR_MAX_BYTES
|
|
67
|
+
* - 'http-<n>' — non-2xx HTTP status code
|
|
68
|
+
* - 'sidecar-error' — 200/2xx but body said { ok: false }
|
|
69
|
+
* - 'fetch-failed' — network error before any response
|
|
70
|
+
* - 'malformed-response' — 2xx but body wasn't the expected JSON
|
|
71
|
+
* - 'timeout' — exceeded the per-call timeout
|
|
72
|
+
*/
|
|
73
|
+
export async function transcribeViaSidecar(
|
|
74
|
+
args: SidecarTranscribeArgs,
|
|
75
|
+
): Promise<TranscribeOutcome> {
|
|
76
|
+
if (!args.token || args.token.length === 0) {
|
|
77
|
+
return { ok: false, reason: 'no-token' }
|
|
78
|
+
}
|
|
79
|
+
if (args.audio.length === 0) {
|
|
80
|
+
return { ok: false, reason: 'audio-too-short' }
|
|
81
|
+
}
|
|
82
|
+
if (args.audio.length > SIDECAR_MAX_BYTES) {
|
|
83
|
+
return {
|
|
84
|
+
ok: false,
|
|
85
|
+
reason: 'audio-too-large',
|
|
86
|
+
detail: `${args.audio.length} bytes (limit ${SIDECAR_MAX_BYTES})`,
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const fetchFn = args.fetchImpl ?? fetch
|
|
91
|
+
const baseUrl = (args.baseUrl ?? DEFAULT_SIDECAR_BASE_URL).replace(/\/$/, '')
|
|
92
|
+
const timeoutMs = args.timeoutMs ?? 90_000
|
|
93
|
+
const controller = new AbortController()
|
|
94
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
|
95
|
+
|
|
96
|
+
// Multipart: server.py reads `audio` (bytes) + optional `language`.
|
|
97
|
+
const form = new FormData()
|
|
98
|
+
const blob = new Blob([args.audio as unknown as BlobPart])
|
|
99
|
+
form.append('audio', blob, args.filename)
|
|
100
|
+
if (args.language) form.append('language', args.language)
|
|
101
|
+
|
|
102
|
+
const startedAt = Date.now()
|
|
103
|
+
let res: Response
|
|
104
|
+
try {
|
|
105
|
+
res = await fetchFn(`${baseUrl}/stt`, {
|
|
106
|
+
method: 'POST',
|
|
107
|
+
headers: { 'X-Voice-Token': args.token },
|
|
108
|
+
body: form,
|
|
109
|
+
signal: controller.signal,
|
|
110
|
+
})
|
|
111
|
+
} catch (err) {
|
|
112
|
+
clearTimeout(timer)
|
|
113
|
+
const msg = err instanceof Error ? err.message : String(err)
|
|
114
|
+
if (msg.includes('aborted') || /timeout/i.test(msg)) {
|
|
115
|
+
return { ok: false, reason: 'timeout', detail: msg }
|
|
116
|
+
}
|
|
117
|
+
return { ok: false, reason: 'fetch-failed', detail: msg }
|
|
118
|
+
}
|
|
119
|
+
clearTimeout(timer)
|
|
120
|
+
|
|
121
|
+
if (!res.ok) {
|
|
122
|
+
const body = await res.text().catch(() => '')
|
|
123
|
+
return { ok: false, reason: `http-${res.status}`, detail: body.slice(0, 200) }
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
let parsed: {
|
|
127
|
+
ok?: unknown
|
|
128
|
+
text?: unknown
|
|
129
|
+
language?: unknown
|
|
130
|
+
durationMs?: unknown
|
|
131
|
+
audioSeconds?: unknown
|
|
132
|
+
reason?: unknown
|
|
133
|
+
detail?: unknown
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
parsed = (await res.json()) as typeof parsed
|
|
137
|
+
} catch (err) {
|
|
138
|
+
return {
|
|
139
|
+
ok: false,
|
|
140
|
+
reason: 'malformed-response',
|
|
141
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// A 2xx with { ok: false } (defence in depth — the sidecar returns
|
|
146
|
+
// non-2xx on errors, but a proxy could rewrite the status).
|
|
147
|
+
if (parsed.ok === false) {
|
|
148
|
+
return {
|
|
149
|
+
ok: false,
|
|
150
|
+
reason: typeof parsed.reason === 'string' ? parsed.reason : 'sidecar-error',
|
|
151
|
+
detail: typeof parsed.detail === 'string' ? parsed.detail.slice(0, 200) : undefined,
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if (typeof parsed.text !== 'string') {
|
|
156
|
+
return {
|
|
157
|
+
ok: false,
|
|
158
|
+
reason: 'malformed-response',
|
|
159
|
+
detail: 'response missing `text` field',
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const out: TranscribeResult = {
|
|
164
|
+
ok: true,
|
|
165
|
+
text: parsed.text,
|
|
166
|
+
language: typeof parsed.language === 'string' ? parsed.language : undefined,
|
|
167
|
+
// Prefer the sidecar's own durationMs; fall back to our wall-clock.
|
|
168
|
+
durationMs:
|
|
169
|
+
typeof parsed.durationMs === 'number'
|
|
170
|
+
? parsed.durationMs
|
|
171
|
+
: Date.now() - startedAt,
|
|
172
|
+
audioSeconds:
|
|
173
|
+
typeof parsed.audioSeconds === 'number' ? parsed.audioSeconds : null,
|
|
174
|
+
}
|
|
175
|
+
return out
|
|
176
|
+
}
|
|
@@ -98,13 +98,13 @@ const DESC_MAX = 80
|
|
|
98
98
|
* Layout (running):
|
|
99
99
|
* 🛠 <b>Worker</b> · <i>{description}</i>
|
|
100
100
|
* <i>{elapsed} · {n} tools</i>
|
|
101
|
-
* <i>✓ {earlier step}</i>
|
|
101
|
+
* <s><i>✓ {earlier step}</i></s>
|
|
102
102
|
* <b>→ {newest step}</b>
|
|
103
103
|
*
|
|
104
104
|
* Layout (finished): the feed renders all-done, then a rule + cleaned result:
|
|
105
105
|
* 🛠 <b>Worker</b> · <i>{description}</i>
|
|
106
106
|
* <i>done · {n} tools · {elapsed}</i>
|
|
107
|
-
* <i>✓ {step}</i>
|
|
107
|
+
* <s><i>✓ {step}</i></s>
|
|
108
108
|
* ─────
|
|
109
109
|
* ✅ <i>{cleaned result paragraph}</i>
|
|
110
110
|
*/
|
|
@@ -158,8 +158,10 @@ export function renderWorkerActivity(v: WorkerActivityView, liveSuffix = ''): st
|
|
|
158
158
|
return `🛠 **Worker** · _starting…_`
|
|
159
159
|
}
|
|
160
160
|
if (!finished && steps.length === 0) {
|
|
161
|
-
// Header-only running render → append the starting placeholder
|
|
162
|
-
|
|
161
|
+
// Header-only running render → append the starting placeholder with a GFM
|
|
162
|
+
// hard break (` \n`) so it stacks under the header instead of collapsing
|
|
163
|
+
// onto the header line in the rich-message renderer (matches stackCardLines).
|
|
164
|
+
return `${card} \n_starting…_`
|
|
163
165
|
}
|
|
164
166
|
return card
|
|
165
167
|
}
|
|
@@ -246,6 +248,10 @@ function extractRetryAfterSecs(err: unknown): number | null {
|
|
|
246
248
|
export interface WorkerActivityFeed {
|
|
247
249
|
/** True if a message is currently posted for this worker. */
|
|
248
250
|
has(agentId: string): boolean
|
|
251
|
+
/** The Telegram message_id currently posted for this worker, or null if
|
|
252
|
+
* none is posted (never painted, or dropped after a stale-edit re-post).
|
|
253
|
+
* Lets the gateway pin the EXISTING `🛠 Worker` message (status-pin). */
|
|
254
|
+
messageIdOf(agentId: string): number | null
|
|
249
255
|
/** Push a running-state cue. Returns the serialized op for tests. */
|
|
250
256
|
update(
|
|
251
257
|
agentId: string,
|
|
@@ -423,7 +429,15 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
423
429
|
const liveElapsed = h.dispatchAtMs != null ? now - h.dispatchAtMs : h.lastView.elapsedMs
|
|
424
430
|
const liveSuffix = ' · ' + formatFeedElapsed(liveElapsed)
|
|
425
431
|
// Re-render THROUGH the chain + doUpdate path — never editMessageText directly.
|
|
426
|
-
|
|
432
|
+
//
|
|
433
|
+
// CLOCK-ANCHOR PARITY: refresh the view's elapsedMs to the SAME
|
|
434
|
+
// `liveElapsed` the step suffix shows. The header renders
|
|
435
|
+
// `view.elapsedMs`; passing the stale lastView froze the header at the
|
|
436
|
+
// last watcher event while the `· Ns` suffix kept ticking, so the
|
|
437
|
+
// current step's timer could read MORE than the card's master elapsed
|
|
438
|
+
// (Ken-observed defect). Both numbers now derive from one anchor
|
|
439
|
+
// (dispatchAtMs) at one `now`, so header elapsed >= step suffix always.
|
|
440
|
+
const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
|
|
427
441
|
h.chain = h.chain
|
|
428
442
|
.then(() => doUpdate(h, view, liveSuffix))
|
|
429
443
|
.catch((err) => {
|
|
@@ -440,6 +454,9 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
440
454
|
has(agentId) {
|
|
441
455
|
return handles.get(agentId)?.messageId != null
|
|
442
456
|
},
|
|
457
|
+
messageIdOf(agentId) {
|
|
458
|
+
return handles.get(agentId)?.messageId ?? null
|
|
459
|
+
},
|
|
443
460
|
get size() {
|
|
444
461
|
return handles.size
|
|
445
462
|
},
|