pi-voicekit 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +341 -0
  3. package/extensions/voice/config.ts +395 -0
  4. package/extensions/voice/deepgram.ts +33 -0
  5. package/extensions/voice/device.ts +382 -0
  6. package/extensions/voice/hold-to-talk.ts +69 -0
  7. package/extensions/voice/local.ts +1143 -0
  8. package/extensions/voice/model-download.ts +636 -0
  9. package/extensions/voice/onboarding.ts +739 -0
  10. package/extensions/voice/release-controller.ts +55 -0
  11. package/extensions/voice/settings-panel.ts +1602 -0
  12. package/extensions/voice/sherpa-engine.ts +464 -0
  13. package/extensions/voice/sherpa-loader.ts +143 -0
  14. package/extensions/voice/sherpa-onnx-node.d.ts +4 -0
  15. package/extensions/voice/speak.ts +430 -0
  16. package/extensions/voice/tts-deepgram.ts +454 -0
  17. package/extensions/voice/tts-engine.ts +653 -0
  18. package/extensions/voice/tts-install-progress.ts +257 -0
  19. package/extensions/voice/tts-local-models.ts +1255 -0
  20. package/extensions/voice/tts-onboarding-overlay.ts +186 -0
  21. package/extensions/voice/tts-onboarding.ts +87 -0
  22. package/extensions/voice/tts-playback-indicator.ts +127 -0
  23. package/extensions/voice/tts-playback.ts +675 -0
  24. package/extensions/voice/tts-text-filter.ts +404 -0
  25. package/extensions/voice/ui-aura.ts +272 -0
  26. package/extensions/voice/ui-help-overlay.ts +161 -0
  27. package/extensions/voice/ui-icons.ts +124 -0
  28. package/extensions/voice/ui-locale-labels.ts +110 -0
  29. package/extensions/voice/ui-picker.ts +209 -0
  30. package/extensions/voice/ui-render-ticker.ts +171 -0
  31. package/extensions/voice/ui-widget-base.ts +219 -0
  32. package/extensions/voice/ui-width.ts +112 -0
  33. package/extensions/voice.ts +3644 -0
  34. package/package.json +75 -0
@@ -0,0 +1,3644 @@
1
+ /**
2
+ * pi-voice — Enterprise-grade voice STT for Pi CLI.
3
+ *
4
+ * Architecture (modeled after Claude Code's voice pipeline):
5
+ *
6
+ * STATE MACHINE
7
+ * ─────────────
8
+ * idle → warmup → recording → finalizing → idle
9
+ * ↑ │
10
+ * └─────────┘ (rapid re-press recovery)
11
+ *
12
+ * warmup: User holds SPACE for ≥ HOLD_THRESHOLD_MS (1200ms).
13
+ * A "keep holding…" hint with countdown is shown. If released before
14
+ * the threshold, a normal space character is typed (or "hold longer" hint shown).
15
+ *
16
+ * recording: SoX captures PCM → Deepgram WebSocket streaming.
17
+ * Live interim + final transcripts update the widget.
18
+ * Release SPACE (or press again in toggle mode) → stop.
19
+ *
20
+ * finalizing: CloseStream sent to Deepgram. Waiting for final
21
+ * transcript. Safety timeout auto-completes.
22
+ *
23
+ * HOLD-TO-TALK DETECTION
24
+ * ──────────────────────
25
+ * Two paths depending on terminal capabilities:
26
+ *
27
+ * A) Kitty protocol (Ghostty on Linux, Kitty, WezTerm):
28
+ * True key-down/repeat/release events available.
29
+ * First SPACE press → enter warmup immediately (show countdown).
30
+ * Released < 300ms → tap → type a space.
31
+ * Released 300ms–2s → show "hold longer" hint.
32
+ * Held ≥ 1.2s → activate recording.
33
+ * True release event stops recording.
34
+ *
35
+ * B) Non-Kitty (macOS Terminal, Ghostty on macOS):
36
+ * No key-release event. Holding sends rapid press events (~30-90ms apart).
37
+ * First SPACE press → record time, start release-detect timer (500ms).
38
+ * No more presses within 500ms → TAP → type a space.
39
+ * Rapid presses detected → user is HOLDING.
40
+ * After REPEAT_CONFIRM_COUNT (6) rapid presses → enter warmup.
41
+ * After HOLD_THRESHOLD_MS (1200ms) from first press → activate recording.
42
+ * Gap > RELEASE_DETECT_MS (500ms) after RECORDING_GRACE_MS (800ms) → stop.
43
+ *
44
+ * ENTERPRISE FALLBACKS
45
+ * ────────────────────
46
+ * • Session corruption guard: new recording request during
47
+ * finalizing automatically cancels the stale session first.
48
+ * • Stale transcript cleanup: any prior transcript is cleared
49
+ * before new recording begins.
50
+ * • Silence vs. no-speech: distinguishes "mic captured silence"
51
+ * from "no speech detected" with distinct user messages.
52
+ *
53
+ * Activation:
54
+ * - Hold SPACE (≥1200ms) → release to finalize
55
+ * - Configurable shortcut (default Ctrl+Shift+V) → toggle start/stop (always works)
56
+
57
+ *
58
+ * Config in ~/.pi/agent/settings.json under "voice": { ... }
59
+ */
60
+
61
+ import type { ExtensionAPI, ExtensionContext, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
62
+ import { isKeyRelease, isKeyRepeat, matchesKey, Key, type KeyId } from "@earendil-works/pi-tui";
63
+
64
+ import { spawn, spawnSync, type ChildProcess } from "node:child_process";
65
+ import * as fs from "node:fs";
66
+ import * as os from "node:os";
67
+ import * as path from "node:path";
68
+ import {
69
+ DEFAULT_CONFIG,
70
+ getSessionStartPersistedConfig,
71
+ loadConfigWithSource,
72
+ loadGlobalToggleShortcut,
73
+ saveConfig,
74
+ type VoiceConfig,
75
+ type VoiceSettingsScope,
76
+ } from "./voice/config";
77
+ import {
78
+ finalizeOnboardingConfig,
79
+ runVoiceOnboarding,
80
+ pickLanguage,
81
+ languageDisplayName,
82
+ modelForLanguage,
83
+ } from "./voice/onboarding";
84
+ import { makeWidgetRegistry, type WidgetRegistry } from "./voice/ui-widget-base";
85
+ import { makeRenderTicker, type RenderTicker } from "./voice/ui-render-ticker";
86
+ import { TtsInstallProgressWidget } from "./voice/tts-install-progress";
87
+ import { TtsPlaybackIndicator } from "./voice/tts-playback-indicator";
88
+ import { buildDeepgramWsUrl, resolveDeepgramApiKey, SAMPLE_RATE, CHANNELS } from "./voice/deepgram";
89
+ import {
90
+ startLocalSession,
91
+ stopLocalSession,
92
+ abortLocalSession,
93
+ checkLocalServer,
94
+ LOCAL_MODELS,
95
+ DEFAULT_LOCAL_ENDPOINT,
96
+ getLanguagesForLocalModel,
97
+ isLanguageSupportedByModel,
98
+ localLanguageDisplayName,
99
+ type LocalSession,
100
+ } from "./voice/local";
101
+ import { shouldArmReleaseDetectOnRepeat, decideRecordingStartTimer } from "./voice/hold-to-talk";
102
+ import { GapTimer, type TimerPort } from "./voice/release-controller";
103
+
104
+ /** Adapter for the real event loop — lets GapTimer run under the real setTimeout. */
105
+ const realTimerPort: TimerPort = {
106
+ once(fn, ms) {
107
+ const id = setTimeout(fn, ms);
108
+ return { cancel: () => clearTimeout(id) };
109
+ },
110
+ };
111
+
112
+ // ─── Types ───────────────────────────────────────────────────────────────────
113
+
114
+ /**
115
+ * Voice state machine — strict transitions only:
116
+ * idle → warmup → recording → finalizing → idle
117
+ * warmup → idle (released before threshold)
118
+ * recording → idle (on error)
119
+ * finalizing → idle (on completion or timeout)
120
+ */
121
+ type VoiceState = "idle" | "warmup" | "recording" | "finalizing";
122
+
123
+ // ─── Constants ───────────────────────────────────────────────────────────────
124
+
125
+ const KEEPALIVE_INTERVAL_MS = 8000;
126
+ const MAX_RECORDING_SECS = 120;
127
+ const STREAM_FINALIZE_TIMEOUT_MS = 2500;
128
+
129
+ // Hold-to-talk timing — Apple-style deliberate hold detection
130
+ // The goal: typing normally should NEVER accidentally trigger voice.
131
+ // Only a clearly intentional long press activates it.
132
+ // v7.1.3 — hold threshold is configurable via `voice.holdThresholdMs`.
133
+ // Default lowered from 1200ms → 700ms for snappier hold-to-talk activation.
134
+ // Apple Caps Lock uses ~1s; modern push-to-talk apps tend to use 500-800ms.
135
+ // 700ms strikes a balance: too short risks accidental activation while typing
136
+ // (a single tap on the spacebar at the end of a word is ~80ms), too long
137
+ // feels laggy. Users can dial higher via /voice-hold-delay or settings.json.
138
+ const HOLD_THRESHOLD_DEFAULT_MS = 700;
139
+ const RELEASE_DETECT_MS = 500; // Gap in key-repeat that means "released" (non-Kitty)
140
+ // macOS default InitialKeyRepeat is ~375ms, so 500ms
141
+ // ensures the first repeat arrives before we decide "tap"
142
+ const REPEAT_CONFIRM_COUNT = 6; // Need this many rapid repeat presses to confirm "holding"
143
+ // At ~30ms repeat rate, 6 presses ≈ 180ms of continuous holding
144
+ // This filters out brief pauses while typing
145
+ const REPEAT_CONFIRM_MS = 700; // Max gap between presses to count as rapid repeat
146
+ // macOS initial key-repeat delay is ~417-583ms depending on settings
147
+ // Must be > macOS InitialKeyRepeat (~375ms)
148
+ const RECORDING_GRACE_MS = 800; // After recording starts, ignore release for this long
149
+ // Covers async gap from holdActivationTimer → startVoiceRecording
150
+ const RELEASE_DETECT_RECORDING_MS = 250; // During active recording, gap before we consider
151
+ // the key released (non-Kitty only). macOS Terminal
152
+ // key repeat fires every ~30-50ms. 250ms gap = released.
153
+ const TYPING_COOLDOWN_MS = 400; // If ANY non-space key was pressed within this window,
154
+ // ignore space holds (user is typing, not activating voice)
155
+ const TAIL_RECORDING_MS = 1500; // Keep recording for 1.5s after space release to catch
156
+ // trailing words. If user re-presses space within this
157
+ // window, cancel the delayed stop and keep recording.
158
+ const CORRUPTION_GUARD_MS = 200; // Min gap between stop and restart
159
+
160
+ // Debug logging — set PI_VOICE_DEBUG=1 to enable
161
+ const VOICE_DEBUG = !!process.env.PI_VOICE_DEBUG;
162
+ const VOICE_LOG_FILE = path.join(os.tmpdir(), "pi-voice-debug.log");
163
+
164
+ // ─── Audio level tracking (module scope so streaming can access) ──────
165
+ let audioLevel = 0;
166
+ let audioLevelSmoothed = 0;
167
+
168
+ function updateAudioLevel(chunk: Buffer) {
169
+ const len = chunk.length;
170
+ if (len < 2) return;
171
+ const samples = len >> 1;
172
+ let sum = 0;
173
+ // Use Int16Array view when alignment permits (2-byte aligned), else fall back
174
+ if ((chunk.byteOffset & 1) === 0) {
175
+ const view = new Int16Array(chunk.buffer, chunk.byteOffset, samples);
176
+ for (let i = 0; i < view.length; i++) {
177
+ sum += view[i] * view[i];
178
+ }
179
+ } else {
180
+ for (let i = 0; i < len - 1; i += 2) {
181
+ const s = chunk.readInt16LE(i);
182
+ sum += s * s;
183
+ }
184
+ }
185
+ const rms = Math.sqrt(sum / samples);
186
+ // Lower ceiling (2500) so normal speech hits 0.5-0.9 instead of 0.1-0.3
187
+ // Power curve (^0.6) boosts quiet sounds for more visible reactivity
188
+ audioLevel = Math.min(1, Math.pow(Math.min(rms / 2500, 1), 0.6));
189
+ // Faster attack (0.35 old), slower decay — snappy peaks, smooth falloff
190
+ audioLevelSmoothed =
191
+ audioLevel > audioLevelSmoothed
192
+ ? audioLevelSmoothed * 0.35 + audioLevel * 0.65
193
+ : audioLevelSmoothed * 0.75 + audioLevel * 0.25;
194
+ // Shared state for other extensions (e.g. pi-pompom mouth animation).
195
+ // Using a namespaced globalThis object instead of pi.events because
196
+ // audio levels update at ~60Hz — event emission would be wasteful.
197
+ const shared = ((globalThis as any).__piListen ??= {});
198
+ shared.audioLevel = audioLevelSmoothed;
199
+ }
200
+
201
+ function voiceDebug(...args: unknown[]) {
202
+ if (!VOICE_DEBUG) return;
203
+ const ts = new Date().toISOString().split("T")[1];
204
+ const line = `[voice ${ts}] ${args.map((a) => (typeof a === "object" ? JSON.stringify(a) : String(a))).join(" ")}\n`;
205
+ try {
206
+ fs.appendFileSync(VOICE_LOG_FILE, line);
207
+ } catch {}
208
+ process.stderr.write(line);
209
+ }
210
+
211
+ // Cache command existence checks — avoid sync spawnSync on every recording start
212
+ const _cmdExistsCache = new Map<string, boolean>();
213
+ function commandExists(cmd: string): boolean {
214
+ const cached = _cmdExistsCache.get(cmd);
215
+ if (cached !== undefined) return cached;
216
+ const which = process.platform === "win32" ? "where" : "which";
217
+ const result = spawnSync(which, [cmd], { stdio: "pipe", timeout: 3000 }).status === 0;
218
+ _cmdExistsCache.set(cmd, result);
219
+ return result;
220
+ }
221
+
222
+ /** Detect the first Windows DirectShow audio input device name via ffmpeg.
223
+ * DirectShow has no "default" alias — must enumerate and pick the first audio device.
224
+ */
225
+ function detectWindowsAudioDevice(): string | null {
226
+ try {
227
+ const result = spawnSync("ffmpeg", ["-f", "dshow", "-list_devices", "true", "-i", "dummy"], {
228
+ timeout: 3000,
229
+ encoding: "utf-8",
230
+ stdio: ["pipe", "pipe", "pipe"],
231
+ });
232
+ // ffmpeg outputs device list to stderr
233
+ const output = result.stderr || "";
234
+ // Match lines like: [dshow @ ...] "Microphone (Realtek HD Audio)" (audio)
235
+ const match = output.match(/"([^"]+)"\s*\(audio\)/);
236
+ return match?.[1] || null;
237
+ } catch {
238
+ return null;
239
+ }
240
+ }
241
+
242
+ interface AudioCaptureTool {
243
+ name: string;
244
+ cmd: string;
245
+ args: string[];
246
+ }
247
+
248
+ // Try available audio capture tools in order of preference
249
+ let _cachedAudioTool: AudioCaptureTool | null | undefined;
250
+ function detectAudioCaptureTool(): AudioCaptureTool | null {
251
+ if (_cachedAudioTool !== undefined) return _cachedAudioTool;
252
+
253
+ // 1. SoX rec — purpose-built for recording, best quality
254
+ if (commandExists("rec")) {
255
+ _cachedAudioTool = {
256
+ name: "sox",
257
+ cmd: "rec",
258
+ args: [
259
+ "-q",
260
+ "--buffer",
261
+ "4096",
262
+ "-c",
263
+ String(CHANNELS),
264
+ "-b",
265
+ "16",
266
+ "-e",
267
+ "signed-integer",
268
+ "-t",
269
+ "raw",
270
+ "-",
271
+ "rate",
272
+ String(SAMPLE_RATE),
273
+ ],
274
+ };
275
+ return _cachedAudioTool;
276
+ }
277
+
278
+ // 2. ffmpeg — widely installed, captures from default mic
279
+ if (commandExists("ffmpeg")) {
280
+ const isLinux = process.platform === "linux";
281
+ const isMac = process.platform === "darwin";
282
+ const isWin = process.platform === "win32";
283
+ // Input device varies by platform
284
+ let inputArgs: string[];
285
+ if (isMac) {
286
+ inputArgs = ["-f", "avfoundation", "-i", ":default"];
287
+ } else if (isLinux) {
288
+ inputArgs = ["-f", "pulse", "-i", "default"];
289
+ } else if (isWin) {
290
+ // DirectShow has no "default" alias — enumerate devices and pick the first audio device
291
+ const dshowDevice = detectWindowsAudioDevice();
292
+ inputArgs = dshowDevice
293
+ ? ["-f", "dshow", "-i", `audio=${dshowDevice}`]
294
+ : ["-f", "dshow", "-i", "audio=Microphone"]; // last-resort guess
295
+ } else {
296
+ inputArgs = ["-f", "pulse", "-i", "default"]; // fallback for other platforms
297
+ }
298
+ _cachedAudioTool = {
299
+ name: "ffmpeg",
300
+ cmd: "ffmpeg",
301
+ args: [
302
+ ...inputArgs,
303
+ "-ac",
304
+ String(CHANNELS),
305
+ "-ar",
306
+ String(SAMPLE_RATE),
307
+ "-sample_fmt",
308
+ "s16",
309
+ "-f",
310
+ "s16le",
311
+ "-loglevel",
312
+ "error",
313
+ "pipe:1",
314
+ ],
315
+ };
316
+ return _cachedAudioTool;
317
+ }
318
+
319
+ // 3. arecord — built into Linux ALSA, zero install
320
+ if (process.platform === "linux" && commandExists("arecord")) {
321
+ _cachedAudioTool = {
322
+ name: "arecord",
323
+ cmd: "arecord",
324
+ args: ["-q", "-f", "S16_LE", "-r", String(SAMPLE_RATE), "-c", String(CHANNELS), "-t", "raw"],
325
+ };
326
+ return _cachedAudioTool;
327
+ }
328
+
329
+ _cachedAudioTool = null;
330
+ return null;
331
+ }
332
+
333
+ // ─── Deepgram WebSocket Streaming ────────────────────────────────────────────
334
+
335
+ interface StreamingSession {
336
+ backend: "deepgram";
337
+ ws: WebSocket;
338
+ recProcess: ChildProcess;
339
+ interimText: string;
340
+ finalizedParts: string[];
341
+ keepAliveTimer: ReturnType<typeof setInterval> | null;
342
+ staleSessionTimer: ReturnType<typeof setTimeout> | null;
343
+ finalizeTimer: ReturnType<typeof setTimeout> | null;
344
+ closed: boolean;
345
+ stopRequested: boolean;
346
+ hadAudioData: boolean; // Track if we received any audio data
347
+ hadSpeech: boolean; // Track if Deepgram detected any speech
348
+ receivedMessage: boolean; // Track if we got ANY message from Deepgram
349
+ onTranscript: (interim: string, finals: string[]) => void;
350
+ onDone: (fullText: string, meta: { hadAudio: boolean; hadSpeech: boolean }) => void;
351
+ onError: (err: string) => void;
352
+ }
353
+
354
+ /** Union of session types — Deepgram streaming or local batch */
355
+ type VoiceSession = StreamingSession | LocalSession;
356
+
357
+ function startStreamingSession(
358
+ config: VoiceConfig,
359
+ callbacks: {
360
+ onTranscript: (interim: string, finals: string[]) => void;
361
+ onDone: (fullText: string, meta: { hadAudio: boolean; hadSpeech: boolean }) => void;
362
+ onError: (err: string) => void;
363
+ }
364
+ ): StreamingSession | null {
365
+ const apiKey = resolveDeepgramApiKey(config);
366
+ voiceDebug("startStreamingSession", { hasApiKey: !!apiKey });
367
+ if (!apiKey) {
368
+ voiceDebug("startStreamingSession → no API key, calling onError");
369
+ callbacks.onError("DEEPGRAM_API_KEY not set");
370
+ return null;
371
+ }
372
+
373
+ // ── Audio capture: try rec (SoX) → ffmpeg → arecord (Linux ALSA) ──
374
+ const audioTool = detectAudioCaptureTool();
375
+ if (!audioTool) {
376
+ voiceDebug("startStreamingSession → no audio capture tool found");
377
+ callbacks.onError("No audio capture tool found. Install one of: sox, ffmpeg, or arecord (Linux)");
378
+ return null;
379
+ }
380
+ voiceDebug("Using audio capture tool:", audioTool.name);
381
+
382
+ const recProc = spawn(audioTool.cmd, audioTool.args, { stdio: ["pipe", "pipe", "pipe"] });
383
+
384
+ recProc.stderr?.on("data", (d: Buffer) => {
385
+ const msg = d.toString().trim();
386
+ // Suppress noisy but harmless messages
387
+ if (msg.includes("buffer overrun") || msg.includes("Discarding") || msg.includes("Last message repeated")) return;
388
+ voiceDebug(`${audioTool.name} stderr:`, msg);
389
+ });
390
+
391
+ const wsUrl = buildDeepgramWsUrl(config);
392
+ const ws = new WebSocket(wsUrl, {
393
+ headers: {
394
+ Authorization: `Token ${apiKey}`,
395
+ },
396
+ } as any);
397
+
398
+ // Connection timeout — abort if Deepgram doesn't respond within 10s
399
+ const wsConnectTimeout = setTimeout(() => {
400
+ if (ws.readyState !== WebSocket.OPEN) {
401
+ voiceDebug("WebSocket connection timeout (10s)");
402
+ try {
403
+ ws.close();
404
+ } catch {}
405
+ try {
406
+ recProc.kill("SIGTERM");
407
+ } catch {}
408
+ callbacks.onError("Deepgram connection timed out (10s). Check your network.");
409
+ }
410
+ }, 10_000);
411
+
412
+ const session: StreamingSession = {
413
+ backend: "deepgram",
414
+ ws,
415
+ recProcess: recProc,
416
+ interimText: "",
417
+ finalizedParts: [],
418
+ keepAliveTimer: null,
419
+ staleSessionTimer: null,
420
+ finalizeTimer: null,
421
+ closed: false,
422
+ stopRequested: false,
423
+ hadAudioData: false,
424
+ hadSpeech: false,
425
+ receivedMessage: false,
426
+ onTranscript: callbacks.onTranscript,
427
+ onDone: callbacks.onDone,
428
+ onError: callbacks.onError,
429
+ };
430
+
431
+ // Handle HTTP error responses before WebSocket upgrade (e.g., 400 Bad Request, 401 Unauthorized)
432
+ // Only available with Node.js `ws` package — skip if using browser-style WebSocket
433
+ if (typeof (ws as any).on === "function") {
434
+ (ws as any).on("unexpected-response", (_req: any, res: any) => {
435
+ let body = "";
436
+ res.on("data", (d: Buffer) => {
437
+ body += d.toString();
438
+ });
439
+ res.on("end", () => {
440
+ voiceDebug("WebSocket unexpected-response", { status: res.statusCode, body });
441
+ if (!session.closed) {
442
+ failStreamingSession(session, `Deepgram HTTP ${res.statusCode}: ${body.slice(0, 200)}`);
443
+ }
444
+ });
445
+ });
446
+ }
447
+
448
+ ws.onopen = () => {
449
+ clearTimeout(wsConnectTimeout);
450
+ voiceDebug("WebSocket onopen → streaming audio");
451
+ try {
452
+ ws.send(JSON.stringify({ type: "KeepAlive" }));
453
+ } catch {}
454
+
455
+ session.keepAliveTimer = setInterval(() => {
456
+ if (ws.readyState === WebSocket.OPEN) {
457
+ try {
458
+ ws.send(JSON.stringify({ type: "KeepAlive" }));
459
+ } catch {}
460
+ }
461
+ }, KEEPALIVE_INTERVAL_MS);
462
+
463
+ recProc.stdout?.on("data", (chunk: Buffer) => {
464
+ if (ws.readyState === WebSocket.OPEN) {
465
+ session.hadAudioData = true;
466
+ try {
467
+ ws.send(new Uint8Array(chunk));
468
+ } catch {}
469
+ // Feed audio data to level meter for reactive waveform
470
+ updateAudioLevel(chunk);
471
+ // Start stale-session watchdog on first audio chunk
472
+ if (!session.staleSessionTimer && !session.receivedMessage) {
473
+ session.staleSessionTimer = setTimeout(() => {
474
+ if (!session.closed && !session.receivedMessage) {
475
+ voiceDebug("Stale session: no Deepgram response after 15s of audio");
476
+ failStreamingSession(session, "No response from Deepgram (15s). Check your API key and network.");
477
+ }
478
+ }, 15_000);
479
+ }
480
+ }
481
+ });
482
+ };
483
+
484
+ ws.onmessage = (event: MessageEvent) => {
485
+ try {
486
+ const msg = typeof event.data === "string" ? JSON.parse(event.data) : null;
487
+ if (!msg) return;
488
+
489
+ // Cancel stale-session watchdog on first response
490
+ if (!session.receivedMessage) {
491
+ session.receivedMessage = true;
492
+ if (session.staleSessionTimer) {
493
+ clearTimeout(session.staleSessionTimer);
494
+ session.staleSessionTimer = null;
495
+ }
496
+ }
497
+
498
+ if (msg.type === "Results") {
499
+ const alt = msg.channel?.alternatives?.[0];
500
+ const transcript = alt?.transcript || "";
501
+
502
+ if (transcript.trim()) {
503
+ session.hadSpeech = true;
504
+ }
505
+
506
+ if (msg.is_final) {
507
+ if (transcript.trim()) {
508
+ session.finalizedParts.push(transcript.trim());
509
+ }
510
+ session.interimText = "";
511
+ } else {
512
+ session.interimText = transcript;
513
+ }
514
+
515
+ session.onTranscript(session.interimText, session.finalizedParts);
516
+ } else if (msg.type === "Error" || msg.type === "error") {
517
+ failStreamingSession(session, msg.message || msg.description || "Deepgram error");
518
+ }
519
+ } catch (err) {
520
+ voiceDebug("onmessage parse error", { error: String(err) });
521
+ }
522
+ };
523
+
524
+ ws.onerror = (ev) => {
525
+ clearTimeout(wsConnectTimeout);
526
+ const errMsg = (ev as any)?.message || (ev as any)?.error?.message || "unknown";
527
+ voiceDebug("WebSocket onerror", { readyState: ws.readyState, error: errMsg });
528
+ if (!session.closed) {
529
+ failStreamingSession(session, `WebSocket error: ${errMsg}`);
530
+ }
531
+ };
532
+
533
+ ws.onclose = (ev) => {
534
+ clearTimeout(wsConnectTimeout);
535
+ const code = (ev as any)?.code;
536
+ const reason = (ev as any)?.reason;
537
+ voiceDebug("WebSocket onclose", { code, reason, closed: session.closed });
538
+ if (!session.closed) {
539
+ // Unexpected close — distinguish normal completion from network drops
540
+ if (session.stopRequested || code === 1000 || code === 1001 || session.finalizedParts.length > 0) {
541
+ if (session.interimText.trim()) {
542
+ session.finalizedParts.push(session.interimText.trim());
543
+ session.interimText = "";
544
+ }
545
+ // Normal close or we have usable transcript data — finalize
546
+ finalizeSession(session);
547
+ } else {
548
+ // Abnormal close with no transcript — treat as error
549
+ failStreamingSession(session, `Connection lost (code ${code ?? "unknown"}${reason ? `: ${reason}` : ""})`);
550
+ }
551
+ }
552
+ };
553
+
554
+ recProc.on("error", (err) => {
555
+ voiceDebug("SoX process error:", err.message);
556
+ if (!session.closed) {
557
+ failStreamingSession(session, `SoX error: ${err.message}`);
558
+ }
559
+ });
560
+
561
+ recProc.on("close", (code, signal) => {
562
+ voiceDebug("SoX process closed", { code, signal, closed: session.closed, wsState: ws.readyState });
563
+ // Only send CloseStream if the session isn't already being torn down
564
+ // (stopStreamingSession sends its own CloseStream before killing SoX)
565
+ if (!session.closed && !session.stopRequested && ws.readyState === WebSocket.OPEN) {
566
+ try {
567
+ ws.send(JSON.stringify({ type: "CloseStream" }));
568
+ } catch {}
569
+ }
570
+ });
571
+
572
+ return session;
573
+ }
574
+
575
+ function stopStreamingSession(session: StreamingSession): void {
576
+ if (session.closed) return;
577
+ session.stopRequested = true;
578
+
579
+ try {
580
+ session.recProcess.kill("SIGTERM");
581
+ } catch {}
582
+
583
+ if (session.ws.readyState === WebSocket.OPEN) {
584
+ try {
585
+ session.ws.send(JSON.stringify({ type: "CloseStream" }));
586
+ } catch {}
587
+ }
588
+
589
+ if (!session.finalizeTimer) {
590
+ session.finalizeTimer = setTimeout(() => {
591
+ session.finalizeTimer = null;
592
+ if (session.closed) return;
593
+ if (session.interimText.trim()) {
594
+ session.finalizedParts.push(session.interimText.trim());
595
+ session.interimText = "";
596
+ }
597
+ finalizeSession(session);
598
+ }, STREAM_FINALIZE_TIMEOUT_MS);
599
+ }
600
+ }
601
+
602
+ function finalizeSession(session: StreamingSession): void {
603
+ if (session.closed) return;
604
+ session.closed = true;
605
+ voiceDebug("finalizeSession", {
606
+ hadAudio: session.hadAudioData,
607
+ hadSpeech: session.hadSpeech,
608
+ parts: session.finalizedParts.length,
609
+ });
610
+
611
+ if (session.staleSessionTimer) {
612
+ clearTimeout(session.staleSessionTimer);
613
+ session.staleSessionTimer = null;
614
+ }
615
+ if (session.finalizeTimer) {
616
+ clearTimeout(session.finalizeTimer);
617
+ session.finalizeTimer = null;
618
+ }
619
+ if (session.keepAliveTimer) {
620
+ clearInterval(session.keepAliveTimer);
621
+ session.keepAliveTimer = null;
622
+ }
623
+
624
+ try {
625
+ session.ws.close();
626
+ } catch {}
627
+ try {
628
+ session.recProcess.kill("SIGKILL");
629
+ } catch {}
630
+
631
+ const fullText = session.finalizedParts.join(" ").trim();
632
+ session.onDone(fullText, {
633
+ hadAudio: session.hadAudioData,
634
+ hadSpeech: session.hadSpeech,
635
+ });
636
+ }
637
+
638
+ function failStreamingSession(session: StreamingSession, err: string): void {
639
+ if (session.closed) return;
640
+ session.closed = true;
641
+ session.stopRequested = true;
642
+
643
+ if (session.staleSessionTimer) {
644
+ clearTimeout(session.staleSessionTimer);
645
+ session.staleSessionTimer = null;
646
+ }
647
+ if (session.finalizeTimer) {
648
+ clearTimeout(session.finalizeTimer);
649
+ session.finalizeTimer = null;
650
+ }
651
+ if (session.keepAliveTimer) {
652
+ clearInterval(session.keepAliveTimer);
653
+ session.keepAliveTimer = null;
654
+ }
655
+
656
+ try {
657
+ session.ws.close();
658
+ } catch {}
659
+ try {
660
+ session.recProcess.kill("SIGKILL");
661
+ } catch {}
662
+ session.onError(err);
663
+ }
664
+
665
+ // ─── Abort helper — nuke everything synchronously ────────────────────────────
666
+
667
+ function abortSession(session: VoiceSession | null): void {
668
+ if (!session || session.closed) return;
669
+ // Replace callbacks with no-ops BEFORE the backend-specific abort path so any
670
+ // in-flight async work (sherpa transcription, late ws messages, recProcess close)
671
+ // that resolves after abort cannot reach the recording state machine — including
672
+ // across session replacement, where the surviving callbacks would otherwise
673
+ // write into the new session's editor / fire notifications on the new ctx.
674
+ session.onTranscript = () => {};
675
+ session.onDone = () => {};
676
+ session.onError = () => {};
677
+ if (session.backend === "local") {
678
+ abortLocalSession(session);
679
+ return;
680
+ }
681
+ session.closed = true;
682
+ if (session.staleSessionTimer) {
683
+ clearTimeout(session.staleSessionTimer);
684
+ session.staleSessionTimer = null;
685
+ }
686
+ if (session.keepAliveTimer) {
687
+ clearInterval(session.keepAliveTimer);
688
+ session.keepAliveTimer = null;
689
+ }
690
+ try {
691
+ session.ws.close();
692
+ } catch {}
693
+ try {
694
+ session.recProcess.kill("SIGKILL");
695
+ } catch {}
696
+ }
697
+
698
+ // ─── Extension ───────────────────────────────────────────────────────────────
699
+
700
+ export default function (pi: ExtensionAPI) {
701
+ let config = DEFAULT_CONFIG;
702
+ let configSource: VoiceSettingsScope | "default" = "default";
703
+ let currentCwd = process.cwd();
704
+ let voiceState: VoiceState = "idle";
705
+ let ctx: ExtensionContext | null = null;
706
+ let recordingStart = 0;
707
+ let statusTimer: ReturnType<typeof setInterval> | null = null;
708
+ let terminalInputUnsub: (() => void) | null = null;
709
+
710
+ // ─── v7.1 Settings UI: widget registry + render ticker (per session) ──
711
+ // Created lazily on first use (session_start) and torn down in
712
+ // voiceCleanup. Owns all DisposableWidget instances for the session.
713
+ let widgetRegistry: WidgetRegistry | null = null;
714
+ let renderTicker: RenderTicker | null = null;
715
+ function getOrInitVoiceUi(): { registry: WidgetRegistry; ticker: RenderTicker } {
716
+ if (!widgetRegistry) widgetRegistry = makeWidgetRegistry();
717
+ if (!renderTicker) renderTicker = makeRenderTicker();
718
+ return { registry: widgetRegistry, ticker: renderTicker };
719
+ }
720
+ /** Currently-mounted install widgets keyed by modelId — for [esc] routing. */
721
+ const activeInstallWidgets = new Map<string, TtsInstallProgressWidget>();
722
+ /** Currently-mounted playback indicator (one at a time) for [esc] routing. */
723
+ let activePlaybackIndicator: TtsPlaybackIndicator | null = null;
724
+
725
+ /**
726
+ * v7.1.3 — read the active hold-threshold (ms) from config with bounds.
727
+ * Outside [200, 3000] falls back to the default (700ms). Bounds chosen
728
+ * so a typo can't make the experience completely broken: 200ms is
729
+ * effectively single-tap, 3000ms is "RSI-friendly slow press."
730
+ */
731
+ function getHoldThresholdMs(): number {
732
+ const v = (config as any).holdThresholdMs;
733
+ if (typeof v === "number" && Number.isFinite(v) && v >= 200 && v <= 3000) return v;
734
+ return HOLD_THRESHOLD_DEFAULT_MS;
735
+ }
736
+
737
+ // ─── Toggle Shortcut (resolved once at startup, used everywhere) ──────
738
+ const resolvedToggleShortcut = loadGlobalToggleShortcut();
739
+ const toggleShortcutLabel = resolvedToggleShortcut
740
+ .split("+")
741
+ .map((p) => (p.length <= 1 ? p.toUpperCase() : p[0]!.toUpperCase() + p.slice(1)))
742
+ .join("+");
743
+
744
+ // Streaming session state
745
+ let activeSession: VoiceSession | null = null;
746
+ let preRecordingSession: StreamingSession | null = null; // Started during warmup, promoted on confirm (Deepgram only)
747
+
748
+ let lastStopTime = 0; // For Escape-to-clear-editor within 30s of recording
749
+ let lastEscapeTime = 0; // For double-escape to clear editor
750
+ let recordingStartedAt = 0; // When recording actually started (for grace period)
751
+ let editorTextBeforeVoice = ""; // Snapshot of editor text before recording started
752
+
753
+ // Hold-to-talk state
754
+ let kittyReleaseDetected = false;
755
+ let spaceDownTime: number | null = null;
756
+ let holdActivationTimer: ReturnType<typeof setTimeout> | null = null;
757
+ let spaceConsumed = false; // True once threshold passed and recording started
758
+ let releaseGapTimer: GapTimer | null = null; // gap-based release detection (see release-controller.ts)
759
+ let warmupWidgetTimer: ReturnType<typeof setInterval> | null = null;
760
+ let spacePressCount = 0; // Count of rapid space presses (for non-Kitty hold detection)
761
+ let lastSpacePressTime = 0; // Timestamp of last space press event
762
+ let holdConfirmed = false; // True once we've confirmed user is holding (not tapping)
763
+ let errorCooldownUntil = 0; // After an error, block re-activation until this timestamp
764
+ let lastNonSpaceKeyTime = 0; // Timestamp of last non-space keypress (typing cooldown)
765
+ let tailRecordingTimer: ReturnType<typeof setTimeout> | null = null; // Delayed stop after release
766
+
767
+ // ─── Recording History ───────────────────────────────────────────────────
768
+
769
+ interface RecordingHistoryEntry {
770
+ text: string;
771
+ timestamp: number;
772
+ duration: number;
773
+ mode: "hold" | "toggle" | "dictate";
774
+ }
775
+
776
+ const recordingHistory: RecordingHistoryEntry[] = [];
777
+ const MAX_HISTORY = 50;
778
+
779
+ function addToHistory(text: string, duration: number, mode: "hold" | "toggle" | "dictate" = "hold") {
780
+ recordingHistory.unshift({ text, timestamp: Date.now(), duration, mode });
781
+ if (recordingHistory.length > MAX_HISTORY) recordingHistory.pop();
782
+ }
783
+
784
+ // ─── Continuous Dictation Mode ───────────────────────────────────────────
785
+
786
+ let dictationMode = false;
787
+
788
+ // ─── Sound Feedback ──────────────────────────────────────────────────────
789
+
790
+ // Pre-resolve sound paths once at load time (not per-play)
791
+ const _soundPaths: Record<string, string | null> = {};
792
+ for (const [type, file] of Object.entries({
793
+ start: "/System/Library/Sounds/Tink.aiff",
794
+ stop: "/System/Library/Sounds/Pop.aiff",
795
+ error: "/System/Library/Sounds/Basso.aiff",
796
+ })) {
797
+ _soundPaths[type] = fs.existsSync(file) ? file : null;
798
+ }
799
+
800
+ function playSound(type: "start" | "stop" | "error") {
801
+ const file = _soundPaths[type];
802
+ if (!file) return;
803
+ try {
804
+ const proc = spawn("afplay", [file], { stdio: "ignore", detached: true });
805
+ proc.unref();
806
+ proc.on("error", () => {}); // Prevent unhandled error crash
807
+ } catch {}
808
+ }
809
+
810
+ // ─── Voice UI ────────────────────────────────────────────────────────────
811
+
812
+ function updateVoiceStatus() {
813
+ if (!ctx?.hasUI) return;
814
+ switch (voiceState) {
815
+ case "idle": {
816
+ if (!config.enabled) {
817
+ ctx.ui.setStatus("voice", undefined);
818
+ break;
819
+ }
820
+ const modeTag = !config.onboarding.completed ? "SETUP" : config.backend === "local" ? "LOCAL" : "STREAM";
821
+ ctx.ui.setStatus("voice", `MIC ${modeTag}`);
822
+ break;
823
+ }
824
+ case "warmup":
825
+ ctx.ui.setStatus("voice", "MIC HOLD...");
826
+ break;
827
+ case "recording": {
828
+ const secs = Math.round((Date.now() - recordingStart) / 1000);
829
+ // Live audio level meter in status bar
830
+ const meterLen = 4;
831
+ const meterFilled = Math.round(audioLevelSmoothed * meterLen);
832
+ const meter = "█".repeat(meterFilled) + "░".repeat(meterLen - meterFilled);
833
+ ctx.ui.setStatus("voice", `REC ${secs}s ${meter}`);
834
+ break;
835
+ }
836
+ case "finalizing":
837
+ if (config.backend === "local") {
838
+ ctx.ui.setStatus("voice", "STT...");
839
+ } else {
840
+ // Don't show "STT..." — live transcript handles it
841
+ ctx.ui.setStatus("voice", "");
842
+ }
843
+ break;
844
+ }
845
+ }
846
+
847
+ function setVoiceState(newState: VoiceState) {
848
+ const prev = voiceState;
849
+ voiceState = newState;
850
+ if (prev !== newState) {
851
+ voiceDebug(`STATE: ${prev} → ${newState}`);
852
+ }
853
+ const shared = ((globalThis as any).__piListen ??= {});
854
+ shared.recording = newState === "recording";
855
+ updateVoiceStatus();
856
+ }
857
+
858
+ // ─── Cleanup helpers ─────────────────────────────────────────────────────
859
+
860
+ function clearHoldTimer() {
861
+ if (holdActivationTimer) {
862
+ clearTimeout(holdActivationTimer);
863
+ holdActivationTimer = null;
864
+ }
865
+ }
866
+
867
+ function clearReleaseTimer() {
868
+ releaseGapTimer?.cancel();
869
+ }
870
+
871
+ function clearWarmupWidget() {
872
+ if (warmupWidgetTimer) {
873
+ clearInterval(warmupWidgetTimer);
874
+ warmupWidgetTimer = null;
875
+ }
876
+ }
877
+
878
+ function clearRecordingAnimTimer() {
879
+ if (_recWidgetAnimTimer) {
880
+ clearInterval(_recWidgetAnimTimer);
881
+ _recWidgetAnimTimer = null;
882
+ }
883
+ }
884
+
885
+ function hideWidget() {
886
+ if (ctx?.hasUI) ctx.ui.setWidget("voice-recording", undefined);
887
+ }
888
+
889
+ /** Reset all hold-to-talk state to idle. Call after any recording stop/error/cancel. */
890
+ function resetHoldState(opts?: { cooldown?: number }) {
891
+ spaceConsumed = false;
892
+ spaceDownTime = null;
893
+ spacePressCount = 0;
894
+ holdConfirmed = false;
895
+ clearHoldTimer();
896
+ clearReleaseTimer();
897
+ abortPreRecording();
898
+ if (opts?.cooldown) errorCooldownUntil = Date.now() + opts.cooldown;
899
+ }
900
+
901
+ function voiceCleanup() {
902
+ // v7.1: cancel in-flight installs FIRST so their AbortControllers
903
+ // fire before we drop UI state. Without this, a session_shutdown
904
+ // during a download would leave the network/disk work running
905
+ // after the widget slot is cleared (Codex v6 finding #3).
906
+ for (const w of Array.from(activeInstallWidgets.values())) {
907
+ try {
908
+ w.cancel();
909
+ } catch (err) {
910
+ voiceDebug("install widget cancel threw during voiceCleanup", String(err));
911
+ }
912
+ }
913
+ activeInstallWidgets.clear();
914
+ // Stop any active playback the same way.
915
+ try {
916
+ activePlaybackIndicator?.stop();
917
+ } catch (err) {
918
+ voiceDebug("playback stop threw during voiceCleanup", String(err));
919
+ }
920
+ activePlaybackIndicator = null;
921
+ // Drain registry — each widget self-clears its slot via dispose().
922
+ try {
923
+ widgetRegistry?.disposeAll();
924
+ } catch (err) {
925
+ voiceDebug("widgetRegistry.disposeAll threw", String(err));
926
+ }
927
+ try {
928
+ renderTicker?.dispose();
929
+ } catch (err) {
930
+ voiceDebug("renderTicker.dispose threw", String(err));
931
+ }
932
+ widgetRegistry = null;
933
+ renderTicker = null;
934
+
935
+ if (statusTimer) {
936
+ clearInterval(statusTimer);
937
+ statusTimer = null;
938
+ }
939
+ cancelDelayedStop();
940
+ clearWarmupWidget();
941
+ clearRecordingAnimTimer();
942
+ // Reset audio levels
943
+ audioLevel = 0;
944
+ audioLevelSmoothed = 0;
945
+ if (activeSession) {
946
+ abortSession(activeSession);
947
+ activeSession = null;
948
+ }
949
+
950
+ resetHoldState(); // includes clearHoldTimer + clearReleaseTimer
951
+ _startingRecording = false;
952
+ lastSpacePressTime = 0;
953
+ lastNonSpaceKeyTime = 0;
954
+ errorCooldownUntil = 0;
955
+ editorTextBeforeVoice = "";
956
+ dictationMode = false;
957
+ recordingStart = 0;
958
+ recordingStartedAt = 0;
959
+ lastStopTime = 0;
960
+ if (terminalInputUnsub) {
961
+ terminalInputUnsub();
962
+ terminalInputUnsub = null;
963
+ }
964
+ hideWidget();
965
+ setVoiceState("idle");
966
+ }
967
+
968
+ async function finalizeAndSaveSetup(
969
+ uiCtx: ExtensionContext | ExtensionCommandContext,
970
+ nextConfig: VoiceConfig,
971
+ selectedScope: VoiceSettingsScope,
972
+ summaryLines: string[],
973
+ source: "first-run" | "setup-command"
974
+ ) {
975
+ const isLocal = nextConfig.backend === "local";
976
+ const hasKey = !!resolveDeepgramApiKey(nextConfig);
977
+ // Local backend is always valid (sherpa handles everything). Deepgram needs API key.
978
+ const validated = isLocal || hasKey;
979
+ config = finalizeOnboardingConfig(nextConfig, { validated, source });
980
+ configSource = selectedScope;
981
+ const savedPath = saveConfig(config, selectedScope, currentCwd);
982
+ const statusHeader = validated
983
+ ? "Voice setup complete."
984
+ : "Voice setup saved, but DEEPGRAM_API_KEY is still required.";
985
+ uiCtx.ui.notify(
986
+ [statusHeader, ...summaryLines, "", `Saved to ${savedPath}`].join("\n"),
987
+ validated ? "info" : "warning"
988
+ );
989
+ }
990
+
991
+ // ─── Warmup Widget ──────────────────────────────────────────────────────
992
+ // ─── Minimal Voice Indicators ──────────────────────────────────────
993
+
994
+ function getRecordDot(): string {
995
+ // v7.2 — soft pulse between full and dim. Two-state instead of
996
+ // three-state (cleaner, more like a breathing LED indicator
997
+ // than the v7.0 multi-glyph approach).
998
+ const phase = (Math.sin(Date.now() / 700) + 1) / 2;
999
+ return phase > 0.5 ? "●" : "○";
1000
+ }
1001
+
1002
+ function buildMiniWave(level: number): string {
1003
+ // Legacy block-bar wave — retained for compatibility with any
1004
+ // caller that still uses it. New code should use buildAuroraWave.
1005
+ const bars = "▁▂▃▄▅▆▇█";
1006
+ const len = 12;
1007
+ let out = "";
1008
+ const t = Date.now() / 1000;
1009
+ const energy = Math.pow(level, 0.7);
1010
+ for (let i = 0; i < len; i++) {
1011
+ const pos = i / len;
1012
+ const wave1 = Math.sin(t * 4.5 + i * 0.9) * 0.35;
1013
+ const wave2 = Math.sin(t * 7.2 + i * 1.4 + 2.0) * 0.15;
1014
+ const center = 1.0 - Math.abs(pos - 0.5) * 1.2;
1015
+ const base = 0.15 + energy * 0.85;
1016
+ const value = Math.max(0, Math.min(1, (wave1 + wave2 + 0.5) * base * center));
1017
+ const idx = Math.min(bars.length - 1, Math.round(value * (bars.length - 1)));
1018
+ out += bars[idx];
1019
+ }
1020
+ return out;
1021
+ }
1022
+
1023
+ /**
1024
+ * v7.2 world-class — Liquid Braille audio waveform with truecolor
1025
+ * Aurora gradient. Per Gemini design recommendation:
1026
+ * - 8 effective vertical levels per CELL (4 dots × 2 columns) via
1027
+ * braille — vs 8 levels per cell with block-bars.
1028
+ * - 2 audio samples per cell width — 2× density of block-bar wave.
1029
+ * - Per-cell color picks an aurora stop based on local peak;
1030
+ * loud peaks "burn" into hot peach/red, soft tails stay cool
1031
+ * lavender. RGB interpolated at runtime.
1032
+ * Output is `cells` cells wide rendered as a single string with
1033
+ * inline ANSI 24-bit escapes; ends with a reset. Caller-side
1034
+ * width math should use `cells` not the byte-length of the result.
1035
+ */
1036
+ function buildAuroraWave(level: number, cells = 16): string {
1037
+ // Generate 2*cells audio "samples" via multi-frequency sine
1038
+ // + the live RMS energy. Same organic motion as the legacy
1039
+ // wave but at higher density.
1040
+ const samples = 2 * cells;
1041
+ const t = Date.now() / 1000;
1042
+ const energy = Math.pow(level, 0.7);
1043
+ const arr: number[] = [];
1044
+ for (let i = 0; i < samples; i++) {
1045
+ const pos = i / samples;
1046
+ const wave1 = Math.sin(t * 4.5 + i * 0.45) * 0.35;
1047
+ const wave2 = Math.sin(t * 7.2 + i * 0.7 + 2.0) * 0.15;
1048
+ const center = 1.0 - Math.abs(pos - 0.5) * 1.0;
1049
+ const base = 0.1 + energy * 0.9;
1050
+ const value = Math.max(0, Math.min(1, (wave1 + wave2 + 0.5) * base * center));
1051
+ arr.push(value);
1052
+ }
1053
+ // Lazy-import to keep voice.ts hot path fast on cold start.
1054
+ const { liquidBraille, auroraColor } = require("./voice/ui-aura") as typeof import("./voice/ui-aura");
1055
+ return liquidBraille(arr, auroraColor);
1056
+ }
1057
+
1058
+ // ─── Warmup Widget ──────────────────────────────────────────────────
1059
+ function showWarmupWidget() {
1060
+ if (!ctx?.hasUI) return;
1061
+
1062
+ const startTime = Date.now();
1063
+
1064
+ const renderWarmup = () => {
1065
+ if (!ctx?.hasUI) return;
1066
+ const elapsed = Date.now() - startTime;
1067
+ const progress = Math.min(elapsed / getHoldThresholdMs(), 1);
1068
+
1069
+ ctx.ui.setWidget(
1070
+ "voice-recording",
1071
+ (_tui, theme) => {
1072
+ return {
1073
+ invalidate() {},
1074
+ render(width: number): string[] {
1075
+ // v7.2 world-class — Same Floating Island chrome
1076
+ // as the active recording widget, with the
1077
+ // progress bar inside. Establishes visual
1078
+ // continuity between warmup → recording (same
1079
+ // island, content shifts, no jump).
1080
+ const { island, auroraColor, titleBreathe } =
1081
+ require("./voice/ui-aura") as typeof import("./voice/ui-aura");
1082
+ const dim = (s: string) => theme.fg("dim", s);
1083
+ const muted = (s: string) => theme.fg("muted", s);
1084
+ const accent = (s: string) => theme.fg("accent", s);
1085
+ const islandW = Math.max(36, Math.min(46, width - 2));
1086
+
1087
+ // Aurora gradient progress: ▰ filled / ▱ empty
1088
+ // with truecolor across the filled portion.
1089
+ const innerW = islandW - 2;
1090
+ const fixedW = 3 /* " ○ " */ + 1; /* trail */
1091
+ const meterCells = Math.max(12, innerW - fixedW);
1092
+ const filled = Math.round(progress * meterCells);
1093
+ let bar = "";
1094
+ for (let i = 0; i < meterCells; i++) {
1095
+ if (i < filled) {
1096
+ // Color stop based on position along filled portion.
1097
+ const t = filled === 0 ? 0 : i / Math.max(1, meterCells - 1);
1098
+ bar += auroraColor(t) + "▰";
1099
+ } else {
1100
+ bar += dim("▱");
1101
+ }
1102
+ }
1103
+ bar += "\x1b[0m";
1104
+
1105
+ const dot = progress < 1 ? muted("○") : accent("●");
1106
+ const content = ` ${dot} ${bar} `;
1107
+ // Breathing title — same aurora-cycle as recording widget
1108
+ // so the warmup → recording transition feels seamless.
1109
+ const titleStyled = titleBreathe(Date.now()) + "\x1b[1mVoice Mode\x1b[0m";
1110
+ const footer = progress < 1 ? dim("hold to record") : accent("ready");
1111
+ return island({ width: islandW, title: titleStyled, content, footer, dim });
1112
+ },
1113
+ };
1114
+ },
1115
+ { placement: "belowEditor" }
1116
+ );
1117
+ };
1118
+
1119
+ renderWarmup();
1120
+ warmupWidgetTimer = setInterval(renderWarmup, 90);
1121
+ }
1122
+
1123
+ // ─── Recording Widget ───────────────────────────────────────────────
1124
+ let _recWidgetAnimTimer: ReturnType<typeof setInterval> | null = null;
1125
+
1126
+ function showRecordingWidget() {
1127
+ if (!ctx?.hasUI) return;
1128
+
1129
+ // Stop warmup animation if still running — seamless takeover,
1130
+ // no gap between warmup and recording widgets (same widget ID).
1131
+ clearWarmupWidget();
1132
+
1133
+ _recWidgetAnimTimer = setInterval(() => {
1134
+ showRecordingWidgetFrame();
1135
+ }, 150);
1136
+
1137
+ showRecordingWidgetFrame();
1138
+ }
1139
+
1140
+ function showRecordingWidgetFrame() {
1141
+ if (!ctx?.hasUI) return;
1142
+
1143
+ // Minimal recording indicator below editor
1144
+ ctx.ui.setWidget(
1145
+ "voice-recording",
1146
+ (_tui, theme) => {
1147
+ return {
1148
+ invalidate() {},
1149
+ render(width: number): string[] {
1150
+ // v7.2 world-class — Floating Island + Liquid Braille +
1151
+ // Aurora gradient + breathing title + activity chip
1152
+ // + 300 ms fade-in transition from warmup.
1153
+ const { island, titleBreathe, activityTag } =
1154
+ require("./voice/ui-aura") as typeof import("./voice/ui-aura");
1155
+ const now = Date.now();
1156
+ const elapsed = (now - recordingStart) / 1000;
1157
+ const mins = Math.floor(elapsed / 60);
1158
+ const secs = elapsed % 60;
1159
+ const timeStr = mins > 0 ? `${mins}:${String(Math.floor(secs)).padStart(2, "0")}` : `${secs.toFixed(1)}s`;
1160
+ const dim = (s: string) => theme.fg("dim", s);
1161
+ const muted = (s: string) => theme.fg("muted", s);
1162
+ const accent = (s: string) => theme.fg("accent", s);
1163
+
1164
+ // Activity chip — one-glance "is sound coming in?"
1165
+ const chip = activityTag(audioLevelSmoothed, dim);
1166
+ const chipPlain = chip.replace(/\x1b\[[\d;]*[A-Za-z]/g, "");
1167
+
1168
+ // Compact 36-48 cols. Reserved cells:
1169
+ // " ● "(3) + wave + " · TIME "(timer+4) + " CHIP "(chip+3) + " "(1)
1170
+ const islandW = Math.max(36, Math.min(48, width - 2));
1171
+ const innerW = islandW - 2;
1172
+ const fixedW = 3 + (4 + timeStr.length) + (3 + chipPlain.length) + 1;
1173
+ const waveCells = Math.max(8, innerW - fixedW);
1174
+
1175
+ // Fade-in over first 300 ms — wave amplitude
1176
+ // scales 0→1 so the recording widget grows out
1177
+ // of warmup rather than snapping in.
1178
+ const sinceStart = Math.max(0, now - recordingStart);
1179
+ const fade = Math.min(1, sinceStart / 300);
1180
+
1181
+ const dot = theme.fg("error", getRecordDot());
1182
+ const wave = buildAuroraWave(audioLevelSmoothed * fade, waveCells);
1183
+
1184
+ // Breathing title — slow aurora-color cycle.
1185
+ const titleStyled = titleBreathe(now) + "\x1b[1mVoice Input\x1b[0m";
1186
+
1187
+ const content = ` ${dot} ${wave} ${dim("·")} ${muted(timeStr)} ${chip} `;
1188
+ const footerStyled = `${dim("release")} ${accent("↑")}`;
1189
+ return island({ width: islandW, title: titleStyled, content, footer: footerStyled, dim });
1190
+ },
1191
+ };
1192
+ },
1193
+ { placement: "belowEditor" }
1194
+ );
1195
+ }
1196
+
1197
+ // ─── Live Transcript ────────────────────────────────────────────────────
1198
+ // Instead of showing transcript in a widget, put it directly in the editor
1199
+ // input area so users see it where they type.
1200
+
1201
+ function updateLiveTranscriptWidget(interim: string, finals: string[]) {
1202
+ if (!ctx?.hasUI) return;
1203
+
1204
+ // DON'T stop the waveform animation — keep it running!
1205
+ // We still want the ● REC waveform + timer to show.
1206
+ // Just update the editor text with the live transcript.
1207
+
1208
+ const finalized = finals.join(" ");
1209
+ const displayText = finalized + (interim ? (finalized ? " " : "") + interim : "");
1210
+
1211
+ // Show live text directly in the editor input (prepend any existing text)
1212
+ if (displayText.trim()) {
1213
+ const prefix = editorTextBeforeVoice ? editorTextBeforeVoice + " " : "";
1214
+ ctx.ui.setEditorText(prefix + displayText);
1215
+ }
1216
+ }
1217
+
1218
+ // ─── Voice: Start / Stop ─────────────────────────────────────────────────
1219
+
1220
+ let _startingRecording = false; // Re-entrancy guard for startVoiceRecording
1221
+
1222
+ async function startVoiceRecording(): Promise<boolean> {
1223
+ voiceDebug("startVoiceRecording called", { voiceState, hasUI: !!ctx?.hasUI, starting: _startingRecording });
1224
+ if (!ctx?.hasUI) return false;
1225
+ if (_startingRecording) return false; // Prevent overlapping starts during corruption guard sleep
1226
+ _startingRecording = true;
1227
+
1228
+ abortActiveSpeak();
1229
+
1230
+ try {
1231
+ // ── SESSION CORRUPTION GUARD ──
1232
+ // If we're still finalizing from a previous recording, abort it first.
1233
+ // This prevents the "slow connection overlaps new recording" bug.
1234
+ if (voiceState === "finalizing" || voiceState === "recording") {
1235
+ abortSession(activeSession);
1236
+ activeSession = null;
1237
+ clearRecordingAnimTimer();
1238
+ clearWarmupWidget();
1239
+ hideWidget();
1240
+ setVoiceState("idle");
1241
+ // Brief pause to let resources release
1242
+ await new Promise((r) => setTimeout(r, CORRUPTION_GUARD_MS));
1243
+ }
1244
+
1245
+ // ── STALE TRANSCRIPT CLEANUP ──
1246
+ // Don't hideWidget() here — the warmup widget is still showing and
1247
+ // showRecordingWidget() will seamlessly replace it using the same
1248
+ // widget ID. Hiding it first causes a visible gap (jitter).
1249
+
1250
+ recordingStart = Date.now();
1251
+
1252
+ // Snapshot editor text before voice overwrites it with live transcript
1253
+ editorTextBeforeVoice = ctx?.hasUI ? ctx.ui.getEditorText() || "" : "";
1254
+
1255
+ return startStreamingRecording();
1256
+ } finally {
1257
+ _startingRecording = false;
1258
+ }
1259
+ }
1260
+
1261
+ // ── Pre-recording: start capturing audio during warmup so we don't miss words ──
1262
+ function startPreRecording() {
1263
+ abortActiveSpeak();
1264
+ if (preRecordingSession) return; // Already started
1265
+ if (config.backend === "local") return; // No pre-recording for local batch mode
1266
+ if (!resolveDeepgramApiKey(config)) return; // No key — skip silently
1267
+ if (!detectAudioCaptureTool()) return; // No audio tool — skip silently
1268
+
1269
+ voiceDebug("startPreRecording → capturing audio during warmup");
1270
+
1271
+ const session = startStreamingSession(config, {
1272
+ onTranscript: (interim, finals) => {
1273
+ // During warmup, silently accumulate transcript
1274
+ // (don't update UI — user hasn't committed to voice yet)
1275
+ voiceDebug("preRecording transcript", { interim: interim.slice(0, 50), finals: finals.length });
1276
+ },
1277
+ onDone: (fullText, meta) => {
1278
+ // Pre-recording ended (user released during warmup) — discard
1279
+ voiceDebug("preRecording onDone (discarded)", { fullText: fullText.slice(0, 50) });
1280
+ if (preRecordingSession === session) preRecordingSession = null;
1281
+ },
1282
+ onError: (err: string) => {
1283
+ voiceDebug("preRecording onError (ignored)", { err });
1284
+ if (preRecordingSession === session) preRecordingSession = null;
1285
+ },
1286
+ });
1287
+
1288
+ if (session) {
1289
+ preRecordingSession = session;
1290
+ }
1291
+ }
1292
+
1293
+ function abortPreRecording() {
1294
+ if (preRecordingSession) {
1295
+ voiceDebug("abortPreRecording → discarding warmup audio");
1296
+ abortSession(preRecordingSession);
1297
+ preRecordingSession = null;
1298
+ }
1299
+ }
1300
+
1301
+ async function startStreamingRecording(): Promise<boolean> {
1302
+ voiceDebug("startStreamingRecording called", {
1303
+ hasKey: !!resolveDeepgramApiKey(config),
1304
+ hasPreRecording: !!preRecordingSession,
1305
+ });
1306
+ setVoiceState("recording");
1307
+
1308
+ // ── Callbacks for the active recording session ──
1309
+ const recordingCallbacks = {
1310
+ onTranscript: (interim: string, finals: string[]) => {
1311
+ // Live transcript update — this is the key UX feature
1312
+ updateLiveTranscriptWidget(interim, finals);
1313
+ updateVoiceStatus();
1314
+ },
1315
+ onDone: (fullText: string, meta: { hadAudio: boolean; hadSpeech: boolean }) => {
1316
+ voiceDebug("onDone callback", { fullText: fullText.slice(0, 100), meta, voiceState, spaceConsumed });
1317
+ activeSession = null;
1318
+ clearRecordingAnimTimer();
1319
+ if (statusTimer) {
1320
+ clearInterval(statusTimer);
1321
+ statusTimer = null;
1322
+ }
1323
+ lastStopTime = Date.now();
1324
+
1325
+ if (!fullText.trim()) {
1326
+ // ── DISTINGUISH SILENCE VS NO SPEECH ──
1327
+ hideWidget();
1328
+ playSound("error");
1329
+ // Full state reset on empty result
1330
+ resetHoldState({ cooldown: 3000 });
1331
+ if (!meta.hadAudio) {
1332
+ ctx?.ui.notify("Microphone captured no audio. Check mic permissions.", "error");
1333
+ } else if (!meta.hadSpeech) {
1334
+ ctx?.ui.notify("Microphone captured silence — no speech detected.", "warning");
1335
+ } else {
1336
+ ctx?.ui.notify("No speech detected.", "warning");
1337
+ }
1338
+ setVoiceState("idle");
1339
+ return;
1340
+ }
1341
+
1342
+ hideWidget();
1343
+
1344
+ if (ctx?.hasUI) {
1345
+ const prefix = editorTextBeforeVoice ? editorTextBeforeVoice + " " : "";
1346
+ const isLocal = config.backend === "local";
1347
+ const finalText = prefix + fullText;
1348
+
1349
+ if (isLocal) {
1350
+ // Local backend (batch mode): no interim transcripts were sent to the editor,
1351
+ // so we must always insert the final text. This is the ONLY place it arrives.
1352
+ ctx.ui.setEditorText(finalText);
1353
+ } else {
1354
+ // Streaming backend: interim transcripts already updated the editor live.
1355
+ // Only set final text if the editor still has content (user didn't hit Enter).
1356
+ const currentEditorText = ctx.ui.getEditorText?.() ?? "";
1357
+ if (currentEditorText.trim()) {
1358
+ ctx.ui.setEditorText(finalText);
1359
+ }
1360
+ }
1361
+
1362
+ // v7.1.1 — auto-submit on STT (config.autoSubmitOnSpeak).
1363
+ // When enabled, the transcribed text is sent to the
1364
+ // agent immediately instead of sitting in the editor
1365
+ // waiting for [enter]. Defaults OFF; user toggles via
1366
+ // /voice-autosubmit or settings panel.
1367
+ if (config.autoSubmitOnSpeak === true && finalText.trim().length > 0) {
1368
+ // v7.2.3 — if the agent is currently mid-turn
1369
+ // (especially mid-retry), DON'T auto-submit.
1370
+ // followUp queueing during a retry pile-up
1371
+ // makes the agent look like it's "looping" —
1372
+ // each queued message gets processed only
1373
+ // after the broken turn finishes. Better UX:
1374
+ // keep transcribed text in the editor, notify
1375
+ // the user, and let them press [enter]
1376
+ // manually when ready.
1377
+ if (agentBusy) {
1378
+ voiceDebug("autoSubmitOnSpeak: agent busy — leaving text in editor");
1379
+ try {
1380
+ ctx.ui.notify("Agent is busy — voice text held in editor. Press [↵] to send when ready.", "info");
1381
+ } catch {
1382
+ /* notify may fail silently */
1383
+ }
1384
+ // Skip dispatch but DON'T early-return —
1385
+ // the rest of the onDone handler still
1386
+ // needs to run state cleanup
1387
+ // (resetHoldState / setVoiceState("idle")).
1388
+ } else {
1389
+ // `sendUserMessage` lives on the `pi` ExtensionAPI
1390
+ // surface, NOT on `ctx`. Always triggers a turn.
1391
+ // `deliverAs: "followUp"` queues mid-stream
1392
+ // messages instead of throwing "Agent is already
1393
+ // processing".
1394
+ //
1395
+ // v7.2.2 (godspeed architect finding) — only
1396
+ // clear the editor AFTER send confirms. If
1397
+ // send rejects synchronously OR returns a
1398
+ // rejected Promise, the user's dictated text
1399
+ // stays visible so they can re-send.
1400
+ const send = (pi as any).sendUserMessage as ((text: string, opts?: any) => unknown) | undefined;
1401
+ if (typeof send === "function") {
1402
+ voiceDebug("autoSubmitOnSpeak: dispatching", { len: finalText.length });
1403
+ // godspeed architect finding — only clear if the
1404
+ // editor STILL contains exactly the dispatched
1405
+ // transcript. If the user typed more chars or
1406
+ // edited it while send was pending, leave their
1407
+ // edits alone.
1408
+ const dispatchedText = finalText;
1409
+ const clearAfterSuccess = () => {
1410
+ try {
1411
+ const cur = ctx?.ui.getEditorText?.() ?? "";
1412
+ if (cur === dispatchedText) {
1413
+ ctx?.ui.setEditorText("");
1414
+ }
1415
+ } catch {
1416
+ /* ui may be gone */
1417
+ }
1418
+ editorTextBeforeVoice = "";
1419
+ };
1420
+ try {
1421
+ const r = send(finalText, { deliverAs: "followUp" });
1422
+ if (r && typeof (r as Promise<unknown>).then === "function") {
1423
+ (r as Promise<unknown>).then(clearAfterSuccess).catch((err) => {
1424
+ voiceDebug("autoSubmitOnSpeak: sendUserMessage rejected", String(err));
1425
+ // Editor stays populated — user can re-press [enter].
1426
+ });
1427
+ } else {
1428
+ // Sync return (or undefined) — assume success.
1429
+ clearAfterSuccess();
1430
+ }
1431
+ } catch (err) {
1432
+ voiceDebug("autoSubmitOnSpeak: sendUserMessage threw sync", String(err));
1433
+ // Editor stays populated.
1434
+ }
1435
+ } else {
1436
+ voiceDebug("autoSubmitOnSpeak: pi.sendUserMessage not available on this Pi version");
1437
+ ctx.ui.notify(
1438
+ "Auto-submit ON but unavailable on this Pi version (need pi.sendUserMessage). " +
1439
+ "Press [enter] to send, or update Pi.",
1440
+ "warning"
1441
+ );
1442
+ }
1443
+ } // end else (agent not busy)
1444
+ }
1445
+
1446
+ const elapsed = ((Date.now() - recordingStart) / 1000).toFixed(1);
1447
+ addToHistory(fullText, parseFloat(elapsed));
1448
+ }
1449
+ playSound("stop");
1450
+ // Full state reset on successful completion
1451
+ resetHoldState();
1452
+ setVoiceState("idle");
1453
+ },
1454
+ onError: (err: string) => {
1455
+ activeSession = null;
1456
+ clearRecordingAnimTimer();
1457
+ if (statusTimer) {
1458
+ clearInterval(statusTimer);
1459
+ statusTimer = null;
1460
+ }
1461
+ hideWidget();
1462
+
1463
+ // ── STOP THE LOOP ──
1464
+ // On error, fully reset ALL hold state AND set a cooldown
1465
+ // so incoming key-repeat events can't re-trigger activation.
1466
+ resetHoldState({ cooldown: 5000 });
1467
+ clearWarmupWidget();
1468
+
1469
+ ctx?.ui.notify(`Voice error: ${err}`, "error");
1470
+ playSound("error");
1471
+ setVoiceState("idle");
1472
+ },
1473
+ };
1474
+
1475
+ // ── Promote pre-recording, start local, or start streaming ──
1476
+ let session: VoiceSession | null;
1477
+
1478
+ if (config.backend === "local") {
1479
+ // Local backend: buffer audio, transcribe on stop
1480
+ const audioTool = detectAudioCaptureTool();
1481
+ if (!audioTool) {
1482
+ recordingCallbacks.onError("No audio capture tool found. Install one of: sox, ffmpeg, or arecord (Linux)");
1483
+ resetHoldState();
1484
+ setVoiceState("idle");
1485
+ return false;
1486
+ }
1487
+ const recProc = spawn(audioTool.cmd, audioTool.args, { stdio: ["pipe", "pipe", "pipe"] });
1488
+ recProc.stderr?.on("data", (d: Buffer) => {
1489
+ const msg = d.toString().trim();
1490
+ if (msg.includes("buffer overrun") || msg.includes("Discarding") || msg.includes("Last message repeated"))
1491
+ return;
1492
+ voiceDebug(`${audioTool.name} stderr:`, msg);
1493
+ });
1494
+ session = startLocalSession(recProc, recordingCallbacks);
1495
+
1496
+ // Feed audio level meter for waveform animation
1497
+ recProc.stdout?.on("data", (chunk: Buffer) => {
1498
+ updateAudioLevel(chunk);
1499
+ });
1500
+ } else if (preRecordingSession) {
1501
+ // Promote: swap callbacks so pre-recorded audio feeds into real UI
1502
+ voiceDebug("Promoting pre-recording session to active");
1503
+ session = preRecordingSession;
1504
+ preRecordingSession = null;
1505
+ session.onTranscript = recordingCallbacks.onTranscript;
1506
+ session.onDone = recordingCallbacks.onDone;
1507
+ session.onError = recordingCallbacks.onError;
1508
+ // Flush any transcript already accumulated during warmup
1509
+ if (session.finalizedParts.length > 0 || session.interimText) {
1510
+ updateLiveTranscriptWidget(session.interimText, session.finalizedParts);
1511
+ }
1512
+ } else {
1513
+ session = startStreamingSession(config, recordingCallbacks);
1514
+ }
1515
+
1516
+ if (!session) {
1517
+ // startStreamingSession returned null — reset ALL state
1518
+ resetHoldState();
1519
+ setVoiceState("idle");
1520
+ return false;
1521
+ }
1522
+
1523
+ activeSession = session;
1524
+
1525
+ // Status timer for elapsed time
1526
+ statusTimer = setInterval(() => {
1527
+ if (voiceState === "recording") {
1528
+ updateVoiceStatus();
1529
+ const elapsed = (Date.now() - recordingStart) / 1000;
1530
+ if (elapsed >= MAX_RECORDING_SECS) {
1531
+ stopVoiceRecording();
1532
+ }
1533
+ }
1534
+ }, 1000);
1535
+
1536
+ showRecordingWidget();
1537
+ playSound("start");
1538
+
1539
+ // #13: arm the gap-based release timer only once recording is ready
1540
+ // (voiceState flipped to recording, success exit). Only hold sessions
1541
+ // (spaceDownTime != null — the user is holding SPACE) and non-Kitty
1542
+ // terminals arm: holding re-arms on each repeat, and release (repeat
1543
+ // stream stops) is perceived within RELEASE_DETECT_RECORDING_MS, so a
1544
+ // key-up inside the startup window also stops ~250ms later. Toggle/
1545
+ // dictation have no repeat stream keeping the timer alive — arming
1546
+ // would auto-stop recording ~250ms after start — so they must be
1547
+ // excluded (spaceDownTime == null). Kitty relies on the real key-release
1548
+ // event, so it stays clear.
1549
+ if (decideRecordingStartTimer({ kittyReleaseDetected, isHold: spaceDownTime != null }) === "arm") {
1550
+ resetReleaseDetect();
1551
+ }
1552
+ return true;
1553
+ }
1554
+
1555
+ // ── Tail recording: keep capturing for 1.5s after space release ──
1556
+ function scheduleDelayedStop() {
1557
+ cancelDelayedStop(); // Clear any existing timer
1558
+ voiceDebug("scheduleDelayedStop → will stop in", TAIL_RECORDING_MS, "ms");
1559
+ tailRecordingTimer = setTimeout(() => {
1560
+ tailRecordingTimer = null;
1561
+ voiceDebug("tailRecordingTimer fired → stopping recording");
1562
+ stopVoiceRecording();
1563
+ }, TAIL_RECORDING_MS);
1564
+ }
1565
+
1566
+ function cancelDelayedStop() {
1567
+ if (tailRecordingTimer) {
1568
+ clearTimeout(tailRecordingTimer);
1569
+ tailRecordingTimer = null;
1570
+ voiceDebug("cancelDelayedStop → tail recording timer cleared");
1571
+ }
1572
+ }
1573
+
1574
+ async function stopVoiceRecording() {
1575
+ cancelDelayedStop(); // Safety: clear any pending delayed stop
1576
+ voiceDebug("stopVoiceRecording called", { voiceState, hasActiveSession: !!activeSession });
1577
+ if (voiceState !== "recording" || !ctx) return;
1578
+ if (statusTimer) {
1579
+ clearInterval(statusTimer);
1580
+ statusTimer = null;
1581
+ }
1582
+
1583
+ if (activeSession) {
1584
+ setVoiceState("finalizing");
1585
+ clearRecordingAnimTimer();
1586
+ hideWidget();
1587
+ if (activeSession.backend === "local") {
1588
+ // Local: show which model is transcribing + estimated time
1589
+ const modelName =
1590
+ LOCAL_MODELS.find((m) => m.id === (config.localModel || "whisper-small"))?.name ||
1591
+ config.localModel ||
1592
+ "local model";
1593
+ ctx?.ui.notify(`Transcribing with ${modelName}…`, "info");
1594
+ await stopLocalSession(activeSession, config);
1595
+ } else {
1596
+ stopStreamingSession(activeSession);
1597
+ }
1598
+ } else {
1599
+ // No active session — shouldn't happen, but recover gracefully
1600
+ voiceDebug("stopVoiceRecording: no active session, resetting to idle");
1601
+ hideWidget();
1602
+ setVoiceState("idle");
1603
+ }
1604
+ }
1605
+
1606
+ // ─── Hold-to-Talk State Machine ─────────────────────────────────────────
1607
+ //
1608
+ // SPACE key handling with STRICT hold-duration detection.
1609
+ //
1610
+ // TWO TERMINAL MODES:
1611
+ //
1612
+ // A) KITTY PROTOCOL (Ghostty on Linux, Kitty, WezTerm, etc.):
1613
+ // True key-down/repeat/release events. On first SPACE press,
1614
+ // immediately enter warmup (show countdown). If released before
1615
+ // HOLD_THRESHOLD_MS → cancel warmup, type a space. If held past
1616
+ // threshold → start recording. True release event stops recording.
1617
+ // No timer-based release detection needed.
1618
+ //
1619
+ // B) NON-KITTY (macOS Terminal, Ghostty on macOS, etc.):
1620
+ // No key-release event. Holding sends rapid press events (~30-90ms apart).
1621
+ // A single tap sends exactly ONE press.
1622
+ // Algorithm:
1623
+ // 1. First SPACE press → record time, start release-detect timer.
1624
+ // 2. No more presses within RELEASE_DETECT_MS (500ms) → TAP → type space.
1625
+ // 3. Rapid presses arrive → user is HOLDING. After REPEAT_CONFIRM_COUNT
1626
+ // rapid presses → enter warmup, show countdown.
1627
+ // 4. After HOLD_THRESHOLD_MS (1200ms) from first press → start recording.
1628
+ // 5. Recording continues while key-repeat events arrive.
1629
+ // Gap > RELEASE_DETECT_MS after RECORDING_GRACE_MS → stop.
1630
+ //
1631
+ // The RECORDING_GRACE_MS prevents the state transition at recording start
1632
+ // from being mistaken for a key release (brief gap in events).
1633
+
1634
+ function onSpaceReleaseDetected() {
1635
+ // GapTimer has already self-cleared (single-shot); nothing to reset here.
1636
+ voiceDebug("onSpaceReleaseDetected", {
1637
+ voiceState,
1638
+ holdConfirmed,
1639
+ spaceConsumed,
1640
+ spaceDownTime,
1641
+ spacePressCount,
1642
+ timeSinceRecStart: spaceConsumed ? Date.now() - recordingStartedAt : null,
1643
+ });
1644
+
1645
+ // If we never confirmed this was a hold (< REPEAT_CONFIRM_COUNT rapid presses),
1646
+ // then it was a TAP → space already passed through naturally (not consumed)
1647
+ if (!holdConfirmed && voiceState === "idle") {
1648
+ resetHoldState();
1649
+ clearWarmupWidget();
1650
+ hideWidget();
1651
+ // No need to type a space — the first press was NOT consumed,
1652
+ // so it already reached the focused UI component naturally.
1653
+ return;
1654
+ }
1655
+
1656
+ // Released during warmup — cancel (user held but not long enough)
1657
+ if (voiceState === "warmup") {
1658
+ resetHoldState();
1659
+ abortPreRecording();
1660
+ clearWarmupWidget();
1661
+ hideWidget();
1662
+ setVoiceState("idle");
1663
+ spaceDownTime = null;
1664
+ spaceConsumed = false;
1665
+ spacePressCount = 0;
1666
+ holdConfirmed = false;
1667
+ // Don't type a space — user clearly intended to trigger voice but let go too early
1668
+ ctx?.ui.notify("Hold SPACE longer to activate voice.", "info");
1669
+ return;
1670
+ }
1671
+
1672
+ // Released during recording — but ONLY if grace period has passed.
1673
+ // The grace period prevents the recording-start transition from being
1674
+ // mistaken for a key release.
1675
+ if (spaceConsumed && voiceState === "recording") {
1676
+ const timeSinceRecordingStart = Date.now() - recordingStartedAt;
1677
+ voiceDebug("release detected during recording", { timeSinceRecordingStart, RECORDING_GRACE_MS });
1678
+ if (timeSinceRecordingStart < RECORDING_GRACE_MS) {
1679
+ // Too soon after recording started — this is likely a false release
1680
+ // caused by the state transition. Re-arm the release detector.
1681
+ voiceDebug(" → too soon, re-arming (grace period)");
1682
+ resetReleaseDetect();
1683
+ return;
1684
+ }
1685
+ voiceDebug(" → scheduling delayed stop (tail recording)");
1686
+ resetHoldState();
1687
+ scheduleDelayedStop();
1688
+ }
1689
+ }
1690
+
1691
+ function resetReleaseDetect() {
1692
+ clearReleaseTimer();
1693
+ if (voiceState === "warmup" || voiceState === "recording" || spaceDownTime || spaceConsumed || holdConfirmed) {
1694
+ // Use longer timeout during active recording — key repeats can be
1695
+ // irregular when the system is under load (Deepgram streaming, etc.)
1696
+ const timeout = voiceState === "recording" || spaceConsumed ? RELEASE_DETECT_RECORDING_MS : RELEASE_DETECT_MS;
1697
+ voiceDebug("resetReleaseDetect", { timeout, voiceState, spaceConsumed });
1698
+ releaseGapTimer = new GapTimer(realTimerPort, { ms: timeout, onFire: onSpaceReleaseDetected });
1699
+ releaseGapTimer.arm();
1700
+ }
1701
+ }
1702
+
1703
+ function setupHoldToTalk() {
1704
+ if (!ctx?.hasUI) return;
1705
+
1706
+ if (terminalInputUnsub) {
1707
+ terminalInputUnsub();
1708
+ terminalInputUnsub = null;
1709
+ }
1710
+
1711
+ terminalInputUnsub = ctx.ui.onTerminalInput((data: string) => {
1712
+ if (!config.enabled) return undefined;
1713
+
1714
+ // v7.1 §4 — escape priority routing for v7.1 widgets. When
1715
+ // no overlay (panel/help/picker) is in front (those run
1716
+ // inside ctx.ui.custom() and consume their own input), the
1717
+ // fallthrough order is: install widget → playback indicator
1718
+ // → editor. Most recent install wins precedence.
1719
+ if (matchesKey(data, Key.escape)) {
1720
+ if (activeInstallWidgets.size > 0) {
1721
+ // Most recent install — Maps preserve insertion order.
1722
+ const ids = Array.from(activeInstallWidgets.keys());
1723
+ const lastId = ids[ids.length - 1]!;
1724
+ const w = activeInstallWidgets.get(lastId);
1725
+ if (w) {
1726
+ w.cancel();
1727
+ return { consume: true };
1728
+ }
1729
+ }
1730
+ if (activePlaybackIndicator) {
1731
+ activePlaybackIndicator.stop();
1732
+ return { consume: true };
1733
+ }
1734
+ }
1735
+
1736
+ // v7.1 §11 — F1 always opens help, regardless of context.
1737
+ // `?` is intentionally NOT bound here because the user is
1738
+ // in the editor and ? is a literal character. Help is
1739
+ // reachable by F1 or /voice-help instead.
1740
+ if (matchesKey(data, Key.f1) && ctx?.hasUI) {
1741
+ openHelpOverlay(ctx as unknown as ExtensionCommandContext).catch(() => {});
1742
+ return { consume: true };
1743
+ }
1744
+
1745
+ // ── Track non-space keypresses for typing cooldown ──
1746
+ // If user was just typing (non-space key within TYPING_COOLDOWN_MS),
1747
+ // don't let space holds activate voice — they're just typing.
1748
+ if (!matchesKey(data, "space") && !isKeyRelease(data) && !isKeyRepeat(data)) {
1749
+ // Regular keypress that isn't space — user is typing
1750
+ if (data.length > 0 && data.charCodeAt(0) >= 32) {
1751
+ lastNonSpaceKeyTime = Date.now();
1752
+ }
1753
+ }
1754
+
1755
+ // ── SPACE handling ──
1756
+ if (matchesKey(data, "space")) {
1757
+ // ── ERROR COOLDOWN: block all voice activation for 5s after an error ──
1758
+ if (errorCooldownUntil > Date.now()) {
1759
+ // During cooldown, let space through as a normal character
1760
+ return undefined;
1761
+ }
1762
+
1763
+ // ── TYPING COOLDOWN: if user was just typing, let space through ──
1764
+ // Apple-style: if a non-space key was pressed recently, this space
1765
+ // is part of typing (e.g., "hello world"), not a voice activation.
1766
+ // Only applies to NEW activations — don't interrupt active recording.
1767
+ if (
1768
+ voiceState === "idle" &&
1769
+ !spaceConsumed &&
1770
+ lastNonSpaceKeyTime > 0 &&
1771
+ Date.now() - lastNonSpaceKeyTime < TYPING_COOLDOWN_MS
1772
+ ) {
1773
+ return undefined;
1774
+ }
1775
+
1776
+ voiceDebug("SPACE event", {
1777
+ isRelease: isKeyRelease(data),
1778
+ isRepeat: isKeyRepeat(data),
1779
+ voiceState,
1780
+ kittyReleaseDetected,
1781
+ holdConfirmed,
1782
+ spaceConsumed,
1783
+ spacePressCount,
1784
+ spaceDownTime: spaceDownTime ? Date.now() - spaceDownTime : null,
1785
+ dataHex: Buffer.from(data).toString("hex"),
1786
+ });
1787
+
1788
+ // ── Kitty key-release (true release event) ──
1789
+ if (isKeyRelease(data)) {
1790
+ kittyReleaseDetected = true;
1791
+ clearReleaseTimer();
1792
+
1793
+ // Released during warmup → cancel
1794
+ // If released very quickly (< 300ms), it was a tap → type a space
1795
+ // If released after 300ms+, user was trying voice → show hint
1796
+ if (voiceState === "warmup") {
1797
+ const holdDuration = spaceDownTime ? Date.now() - spaceDownTime : 0;
1798
+ resetHoldState();
1799
+ abortPreRecording();
1800
+ clearWarmupWidget();
1801
+ hideWidget();
1802
+ setVoiceState("idle");
1803
+ if (holdDuration < 300) {
1804
+ // Quick tap — just type a space
1805
+ if (ctx?.hasUI) ctx.ui.setEditorText((ctx.ui.getEditorText() || "") + " ");
1806
+ } else {
1807
+ // Held long enough to see warmup but let go → show hint
1808
+ ctx?.ui.notify("Hold SPACE longer to activate voice.", "info");
1809
+ }
1810
+ return { consume: true };
1811
+ }
1812
+
1813
+ // Tap: released before warmup even started (shouldn't happen in
1814
+ // Kitty path since we enter warmup on first press, but handle anyway)
1815
+ if (spaceDownTime && !holdConfirmed && voiceState === "idle") {
1816
+ resetHoldState();
1817
+ if (ctx?.hasUI) ctx.ui.setEditorText((ctx.ui.getEditorText() || "") + " ");
1818
+ return { consume: true };
1819
+ }
1820
+
1821
+ // Released during recording → schedule delayed stop (tail recording)
1822
+ if (spaceConsumed && voiceState === "recording") {
1823
+ resetHoldState();
1824
+ scheduleDelayedStop();
1825
+ return { consume: true };
1826
+ }
1827
+
1828
+ spaceDownTime = null;
1829
+ spaceConsumed = false;
1830
+ spacePressCount = 0;
1831
+ holdConfirmed = false;
1832
+ return undefined;
1833
+ }
1834
+
1835
+ // ── Kitty key-repeat ──
1836
+ if (isKeyRepeat(data)) {
1837
+ // #13: terminals without key-release events (e.g. Ghostty on
1838
+ // macOS, non-kitty modes) signal "released" purely by a GAP in
1839
+ // repeat events. Every repeat must re-arm the release-detect
1840
+ // timer — recording start clears it, and if nothing re-arms it
1841
+ // the release is never detected → hold-to-talk locks up forever.
1842
+ if (shouldArmReleaseDetectOnRepeat({ voiceState, kittyReleaseDetected })) {
1843
+ resetReleaseDetect();
1844
+ }
1845
+ // Already in recording/finalizing — consume (release timer kept alive above)
1846
+ if (voiceState === "recording" || voiceState === "finalizing" || spaceConsumed) {
1847
+ return { consume: true };
1848
+ }
1849
+ // Already in warmup — consume (hold timer is running)
1850
+ if (voiceState === "warmup") {
1851
+ return { consume: true };
1852
+ }
1853
+
1854
+ // During initial hold detection: if we took PATH B on first
1855
+ // press (because kittyReleaseDetected was false), we need to
1856
+ // count these repeats to confirm the hold. Update state so
1857
+ // onSpaceReleaseDetected won't fire a false tap.
1858
+ if (spaceDownTime && !holdConfirmed) {
1859
+ // NOTE: Do NOT set kittyReleaseDetected here!
1860
+ // Ghostty on macOS sends repeat events but NO release events.
1861
+ // Only a true isKeyRelease() should flip the Kitty flag.
1862
+
1863
+ const now = Date.now();
1864
+ spacePressCount++;
1865
+ lastSpacePressTime = now;
1866
+
1867
+ // Enough repeats to confirm hold — enter warmup
1868
+ if (spacePressCount >= REPEAT_CONFIRM_COUNT) {
1869
+ holdConfirmed = true;
1870
+ setVoiceState("warmup");
1871
+ showWarmupWidget();
1872
+ startPreRecording();
1873
+
1874
+ const alreadyElapsed = now - (spaceDownTime || now);
1875
+ const remaining = Math.max(0, getHoldThresholdMs() - alreadyElapsed);
1876
+
1877
+ holdActivationTimer = setTimeout(() => {
1878
+ holdActivationTimer = null;
1879
+ if (voiceState === "warmup") {
1880
+ // Don't clearWarmupWidget() here — showRecordingWidget()
1881
+ // seamlessly replaces it using the same widget ID.
1882
+ spaceConsumed = true;
1883
+ recordingStartedAt = Date.now();
1884
+ // Clear release timer during async recording startup to
1885
+ // prevent a false stop. The 250ms gap timer is re-armed
1886
+ // once recording is actually ready — see startStreamingRecording (#13).
1887
+ clearReleaseTimer();
1888
+ voiceDebug("holdActivationTimer fired → starting recording (Kitty repeat path)");
1889
+ startVoiceRecording()
1890
+ .then((ok) => {
1891
+ if (!ok) {
1892
+ resetHoldState();
1893
+ setVoiceState("idle");
1894
+ }
1895
+ })
1896
+ .catch((err) => {
1897
+ voiceDebug("startVoiceRecording THREW", { error: String(err) });
1898
+ resetHoldState({ cooldown: 5000 });
1899
+ setVoiceState("idle");
1900
+ });
1901
+ } else {
1902
+ spaceDownTime = null;
1903
+ spaceConsumed = false;
1904
+ spacePressCount = 0;
1905
+ holdConfirmed = false;
1906
+ }
1907
+ }, remaining);
1908
+ }
1909
+
1910
+ // Re-arm gap-based release detection — this is how
1911
+ // Ghostty-on-macOS (repeats but no release) detects
1912
+ // the key being released. (Preserved from the original
1913
+ // ordering — the recording-path re-arm above is separate.)
1914
+ resetReleaseDetect();
1915
+ return { consume: true };
1916
+ }
1917
+
1918
+ return { consume: true };
1919
+ }
1920
+
1921
+ // === Key PRESS (not repeat, not release) ===
1922
+ //
1923
+ // TWO TERMINAL MODES:
1924
+ //
1925
+ // A) Kitty protocol (kittyReleaseDetected = true):
1926
+ // Press fires ONCE on key-down. Repeats come as isKeyRepeat().
1927
+ // Release comes as isKeyRelease(). NO timer-based release detection
1928
+ // needed — the true release event handles everything.
1929
+ // On first press: enter warmup immediately and start hold timer.
1930
+ // (No need to wait for repeats to confirm hold.)
1931
+ //
1932
+ // B) Non-Kitty (macOS Terminal, etc.):
1933
+ // Holding a key sends rapid "press" events (~30-90ms apart).
1934
+ // A single tap sends exactly ONE press. There is NO release event.
1935
+ // We detect "tap vs hold" by counting rapid presses, and detect
1936
+ // "release" when no press arrives within RELEASE_DETECT_MS.
1937
+
1938
+ // If finalizing → ignore
1939
+ if (voiceState === "finalizing") {
1940
+ return { consume: true };
1941
+ }
1942
+
1943
+ // If already recording → cancel any pending delayed stop and keep going
1944
+ if (voiceState === "recording") {
1945
+ cancelDelayedStop(); // User re-pressed — they want to keep recording
1946
+ spaceConsumed = true; // Re-arm hold state for the continued recording
1947
+ spaceDownTime = Date.now();
1948
+ holdConfirmed = true;
1949
+ if (!kittyReleaseDetected) {
1950
+ voiceDebug("SPACE during recording → cancel delayed stop, re-arm release detect");
1951
+ resetReleaseDetect();
1952
+ } else {
1953
+ voiceDebug("SPACE during recording → cancel delayed stop (Kitty)");
1954
+ }
1955
+ return { consume: true };
1956
+ }
1957
+
1958
+ // If already in warmup → consume
1959
+ if (voiceState === "warmup") {
1960
+ if (!kittyReleaseDetected) {
1961
+ voiceDebug("SPACE during warmup → re-arm release detect");
1962
+ resetReleaseDetect();
1963
+ }
1964
+ return { consume: true };
1965
+ }
1966
+
1967
+ // If we've already consumed space for this hold → consume
1968
+ // This handles the gap between holdActivationTimer firing and
1969
+ // voiceState transitioning to "recording" (async gap)
1970
+ if (spaceConsumed) {
1971
+ if (!kittyReleaseDetected) {
1972
+ voiceDebug("SPACE while spaceConsumed (async gap) → re-arm release detect");
1973
+ resetReleaseDetect();
1974
+ }
1975
+ return { consume: true };
1976
+ }
1977
+
1978
+ // ──────────────────────────────────────────────────────────
1979
+ // PATH A: Kitty protocol — true key events available
1980
+ // ──────────────────────────────────────────────────────────
1981
+ if (kittyReleaseDetected) {
1982
+ // First press → immediately enter warmup (release event
1983
+ // will cancel if it was a tap)
1984
+ if (voiceState === "idle") {
1985
+ spaceDownTime = Date.now();
1986
+ spaceConsumed = false;
1987
+ spacePressCount = 1;
1988
+ lastSpacePressTime = Date.now();
1989
+ holdConfirmed = true; // Kitty: trust the press, release cancels
1990
+
1991
+ setVoiceState("warmup");
1992
+ showWarmupWidget();
1993
+ startPreRecording();
1994
+
1995
+ holdActivationTimer = setTimeout(() => {
1996
+ holdActivationTimer = null;
1997
+ if (voiceState === "warmup") {
1998
+ // Don't clearWarmupWidget() here — showRecordingWidget()
1999
+ // seamlessly replaces it using the same widget ID.
2000
+ spaceConsumed = true;
2001
+ recordingStartedAt = Date.now();
2002
+ voiceDebug("holdActivationTimer fired → starting recording (Kitty path)");
2003
+ startVoiceRecording()
2004
+ .then((ok) => {
2005
+ if (!ok) {
2006
+ resetHoldState();
2007
+ setVoiceState("idle");
2008
+ }
2009
+ })
2010
+ .catch((err) => {
2011
+ voiceDebug("startVoiceRecording THREW", { error: String(err) });
2012
+ resetHoldState({ cooldown: 5000 });
2013
+ setVoiceState("idle");
2014
+ });
2015
+ } else {
2016
+ spaceDownTime = null;
2017
+ spaceConsumed = false;
2018
+ spacePressCount = 0;
2019
+ holdConfirmed = false;
2020
+ }
2021
+ }, getHoldThresholdMs());
2022
+
2023
+ return { consume: true };
2024
+ }
2025
+ return { consume: true };
2026
+ }
2027
+
2028
+ // ──────────────────────────────────────────────────────────
2029
+ // PATH B: Non-Kitty — gap-based hold/release detection
2030
+ // ──────────────────────────────────────────────────────────
2031
+ // Holding a key sends rapid press events.
2032
+ // We count presses and measure gaps to detect holds vs taps.
2033
+ if (spaceDownTime) {
2034
+ const now = Date.now();
2035
+ const gap = now - lastSpacePressTime;
2036
+
2037
+ if (gap < REPEAT_CONFIRM_MS) {
2038
+ // Rapid press = user is holding
2039
+ spacePressCount++;
2040
+ lastSpacePressTime = now;
2041
+
2042
+ if (spacePressCount >= REPEAT_CONFIRM_COUNT && !holdConfirmed) {
2043
+ holdConfirmed = true;
2044
+ setVoiceState("warmup");
2045
+ showWarmupWidget();
2046
+ startPreRecording();
2047
+
2048
+ const alreadyElapsed = now - spaceDownTime;
2049
+ const remaining = Math.max(0, getHoldThresholdMs() - alreadyElapsed);
2050
+
2051
+ holdActivationTimer = setTimeout(() => {
2052
+ holdActivationTimer = null;
2053
+ if (voiceState === "warmup") {
2054
+ // Don't clearWarmupWidget() here — showRecordingWidget()
2055
+ // seamlessly replaces it using the same widget ID.
2056
+ spaceConsumed = true;
2057
+ recordingStartedAt = Date.now();
2058
+ // CRITICAL: Clear release timer and DO NOT re-arm.
2059
+ // The next key-repeat press event will re-arm it.
2060
+ // Without this, the async startVoiceRecording creates
2061
+ // a gap where the release timer fires falsely.
2062
+ clearReleaseTimer();
2063
+ voiceDebug("holdActivationTimer fired → starting recording (non-Kitty)");
2064
+ startVoiceRecording()
2065
+ .then((ok) => {
2066
+ if (!ok) {
2067
+ resetHoldState();
2068
+ setVoiceState("idle");
2069
+ }
2070
+ // Do NOT re-arm release detect here!
2071
+ // The next SPACE key-repeat event will do it.
2072
+ // Re-arming here causes false stops because
2073
+ // the timer fires during the async gap.
2074
+ })
2075
+ .catch((err) => {
2076
+ voiceDebug("startVoiceRecording THREW", { error: String(err) });
2077
+ resetHoldState({ cooldown: 5000 });
2078
+ setVoiceState("idle");
2079
+ });
2080
+ } else {
2081
+ spaceDownTime = null;
2082
+ spaceConsumed = false;
2083
+ spacePressCount = 0;
2084
+ holdConfirmed = false;
2085
+ }
2086
+ }, remaining);
2087
+ }
2088
+
2089
+ resetReleaseDetect();
2090
+ return { consume: true };
2091
+ } else {
2092
+ // Gap too large → previous hold abandoned, new tap
2093
+ const wasInWarmup = (voiceState as VoiceState) === "warmup";
2094
+ resetHoldState();
2095
+ abortPreRecording();
2096
+ clearWarmupWidget();
2097
+ hideWidget();
2098
+ if (wasInWarmup) setVoiceState("idle");
2099
+ // Only type a space if we weren't already in warmup
2100
+ // (if we were in warmup, user was trying to activate voice, not type)
2101
+ // Note: first space already passed through naturally (not consumed)
2102
+ // so we don't need to manually type it here
2103
+ // Fall through to treat this as a new first press
2104
+ }
2105
+ }
2106
+
2107
+ // IDLE — first SPACE press (non-Kitty path)
2108
+ // Do NOT consume — let it pass through to whatever UI is focused
2109
+ // (editor, search box, picker, etc.). Only start consuming after
2110
+ // we confirm it's a hold via REPEAT_CONFIRM_COUNT rapid presses.
2111
+ if (voiceState === "idle") {
2112
+ spaceDownTime = Date.now();
2113
+ spaceConsumed = false;
2114
+ spacePressCount = 1;
2115
+ lastSpacePressTime = Date.now();
2116
+ holdConfirmed = false;
2117
+
2118
+ resetReleaseDetect();
2119
+
2120
+ // Don't consume — let the space reach the focused UI component
2121
+ return undefined;
2122
+ }
2123
+
2124
+ if (spaceConsumed) return { consume: true };
2125
+ return undefined;
2126
+ }
2127
+
2128
+ // ── Any other key pressed → cancel potential hold ──
2129
+ if (spaceDownTime && !holdConfirmed && voiceState === "idle") {
2130
+ resetHoldState();
2131
+ // No need to insert a space manually — the first space press was
2132
+ // already allowed to pass through to the focused UI component.
2133
+ return undefined;
2134
+ }
2135
+
2136
+ if (voiceState === "warmup" && holdConfirmed && !spaceConsumed) {
2137
+ clearWarmupWidget();
2138
+ hideWidget();
2139
+ resetHoldState();
2140
+ setVoiceState("idle");
2141
+ return undefined;
2142
+ }
2143
+
2144
+ // ── Escape key — cancel voice / double-escape clears editor ──
2145
+ // Skip release/repeat events — only act on actual presses
2146
+ if (matchesKey(data, "escape") && !isKeyRelease(data) && !isKeyRepeat(data)) {
2147
+ // During recording: cancel recording and clear transcript
2148
+ if (voiceState === "recording" || voiceState === "warmup" || voiceState === "finalizing") {
2149
+ voiceDebug("Escape pressed → canceling voice");
2150
+ abortPreRecording();
2151
+ if (activeSession) {
2152
+ abortSession(activeSession);
2153
+ activeSession = null;
2154
+ }
2155
+ clearRecordingAnimTimer();
2156
+ clearWarmupWidget();
2157
+ hideWidget();
2158
+ if (statusTimer) {
2159
+ clearInterval(statusTimer);
2160
+ statusTimer = null;
2161
+ }
2162
+ // Restore editor text to what it was before recording
2163
+ if (ctx?.hasUI) ctx.ui.setEditorText(editorTextBeforeVoice);
2164
+ resetHoldState();
2165
+ playSound("error");
2166
+ setVoiceState("idle");
2167
+ lastEscapeTime = Date.now();
2168
+ return { consume: true };
2169
+ }
2170
+
2171
+ // In idle: double-escape (two presses within 500ms) clears editor
2172
+ if (voiceState === "idle") {
2173
+ const now = Date.now();
2174
+ if (lastEscapeTime > 0 && now - lastEscapeTime < 500) {
2175
+ if (ctx?.hasUI) {
2176
+ const currentText = ctx.ui.getEditorText() || "";
2177
+ if (currentText.trim()) {
2178
+ ctx.ui.setEditorText("");
2179
+ lastEscapeTime = 0;
2180
+ return { consume: true };
2181
+ }
2182
+ }
2183
+ }
2184
+ lastEscapeTime = now;
2185
+ }
2186
+ }
2187
+
2188
+ return undefined;
2189
+ });
2190
+ }
2191
+
2192
+ // ─── Shortcuts ───────────────────────────────────────────────────────────
2193
+
2194
+ // resolvedToggleShortcut is a runtime config string, but pi.registerShortcut
2195
+ // wants the literal-union KeyId type. The string was validated by
2196
+ // isValidShortcut() in loadGlobalToggleShortcut() (modifier+key shape with
2197
+ // a known modifier set), so asserting `as KeyId` is honest — `as any` was
2198
+ // strictly worse because it hid the intent and would have masked a real
2199
+ // signature change. If pi-tui later promotes KeyId to `string`, this
2200
+ // assertion just becomes a no-op.
2201
+ pi.registerShortcut(resolvedToggleShortcut as KeyId, {
2202
+ description: "Toggle voice recording (start/stop)",
2203
+ handler: async (handlerCtx) => {
2204
+ ctx = handlerCtx;
2205
+ if (!config.enabled) {
2206
+ handlerCtx.ui.notify("Voice disabled. Use /voice on", "warning");
2207
+ return;
2208
+ }
2209
+ if (dictationMode) {
2210
+ // The configured toggle shortcut stops dictation mode
2211
+ dictationMode = false;
2212
+ if (voiceState === "recording") {
2213
+ await stopVoiceRecording();
2214
+ }
2215
+ handlerCtx.ui.notify("Dictation mode stopped.", "info");
2216
+ return;
2217
+ }
2218
+ if (voiceState === "idle") {
2219
+ spaceConsumed = true;
2220
+ const ok = await startVoiceRecording();
2221
+ if (!ok) {
2222
+ spaceConsumed = false;
2223
+ }
2224
+ } else if (voiceState === "recording") {
2225
+ resetHoldState();
2226
+ await stopVoiceRecording();
2227
+ } else if (voiceState === "warmup") {
2228
+ // Cancel warmup
2229
+ abortPreRecording();
2230
+ clearWarmupWidget();
2231
+ hideWidget();
2232
+ resetHoldState();
2233
+ setVoiceState("idle");
2234
+ }
2235
+ // voiceState === "finalizing" → ignore (wait for transcript)
2236
+ },
2237
+ });
2238
+
2239
+ // ─── Lifecycle ───────────────────────────────────────────────────────────
2240
+
2241
+ pi.on("session_start", async (event, startCtx) => {
2242
+ // `event.reason` was added in pi-mono 0.65.0 ("startup" | "reload" | "new" |
2243
+ // "resume" | "fork"). Older Pi versions don't include it. Narrow defensively
2244
+ // so the same code typechecks against both 0.57-era and 0.65+ types.
2245
+ const reason = (event as { reason?: string } | undefined)?.reason ?? "startup";
2246
+ const isStartup = reason === "startup";
2247
+
2248
+ // Non-startup transitions: pi-mono >= 0.68.0 fires session_shutdown first
2249
+ // (and awaits it), so voiceCleanup() has already run. This is just belt-
2250
+ // and-suspenders for older Pi versions that may skip session_shutdown on
2251
+ // session replacement. The try/catch matches session_shutdown — a throw
2252
+ // inside voiceCleanup (e.g. a child process kill EPERM under load) must
2253
+ // not abort handler execution and leave ctx unassigned.
2254
+ if (!isStartup) {
2255
+ try {
2256
+ voiceCleanup();
2257
+ } catch (err) {
2258
+ voiceDebug("voiceCleanup threw during session_start", { error: String(err) });
2259
+ }
2260
+ }
2261
+
2262
+ ctx = startCtx;
2263
+ currentCwd = startCtx.cwd;
2264
+ const loaded = loadConfigWithSource(startCtx.cwd);
2265
+ config = loaded.config;
2266
+ configSource = loaded.source;
2267
+
2268
+ // v7.1.3 — version banner emitted to debug log on every session
2269
+ // start. Lets users / support verify which extension build is
2270
+ // actually loaded after a `pi install .` (path-installed
2271
+ // extensions cache modules across `pi install` reinstalls — only
2272
+ // a fresh `pi` process picks up source changes).
2273
+ voiceDebug("pi-listen v7.1.3 loaded", { reason });
2274
+
2275
+ // Migration / setup runs on EVERY session_start, regardless of reason.
2276
+ // Only the first-run notification is gated on isStartup.
2277
+ if (config.onboarding.completed) {
2278
+ // Always refresh the status bar — when voice is disabled,
2279
+ // updateVoiceStatus() clears the entry so users don't see stale
2280
+ // "MIC STREAM" text from a prior session. Hold-to-talk wiring
2281
+ // only runs when enabled.
2282
+ updateVoiceStatus();
2283
+ if (config.enabled) {
2284
+ setupHoldToTalk();
2285
+ }
2286
+ return;
2287
+ }
2288
+
2289
+ // Onboarding not complete. Bail before the migration / hint UI work if the
2290
+ // session has no UI surface — non-interactive sessions can't display
2291
+ // notifications anyway, and migration wiring (setupHoldToTalk) requires UI.
2292
+ if (!startCtx.hasUI) return;
2293
+
2294
+ // Try migration if a backend is already configured.
2295
+ const hasKey = !!resolveDeepgramApiKey(config);
2296
+ const hasLocalModel = config.backend === "local" && !!config.localModel;
2297
+ const audioTool = detectAudioCaptureTool();
2298
+ if (hasKey || hasLocalModel) {
2299
+ // Backend configured (Deepgram key or local model) — auto-activate.
2300
+ // Migration runs every transition; the welcome notification is gated
2301
+ // on isStartup so /new and /resume don't re-spam the hint.
2302
+ config.onboarding.completed = true;
2303
+ config.onboarding.completedAt = new Date().toISOString();
2304
+ config.onboarding.source = "migration";
2305
+ const configToSave = getSessionStartPersistedConfig({
2306
+ config,
2307
+ envDeepgramApiKey: process.env.DEEPGRAM_API_KEY,
2308
+ });
2309
+ saveConfig(configToSave, config.scope === "project" ? "project" : "global", currentCwd);
2310
+ updateVoiceStatus();
2311
+ setupHoldToTalk();
2312
+ if (!isStartup) return;
2313
+ const backendLabel = hasLocalModel
2314
+ ? `Local model: ${LOCAL_MODELS.find((m) => m.id === config.localModel)?.name || config.localModel} (offline, batch mode)`
2315
+ : "Deepgram Nova-3 (cloud, live streaming)";
2316
+ const lines = [
2317
+ "pi-listen ready!",
2318
+ "",
2319
+ " Hold SPACE to record → release to transcribe",
2320
+ ` ${toggleShortcutLabel} to toggle recording`,
2321
+ ` Backend: ${backendLabel}`,
2322
+ ` Audio: ${audioTool ? `${audioTool.name}` : "NONE — install sox or ffmpeg"}`,
2323
+ "",
2324
+ " /voice-settings to change backend, model, or language",
2325
+ ];
2326
+ startCtx.ui.notify(lines.join("\n"), audioTool ? "info" : "warning");
2327
+ return;
2328
+ }
2329
+
2330
+ // No backend configured — show install hint only on actual startup.
2331
+ if (!isStartup) return;
2332
+ const lines = [
2333
+ "pi-listen installed — voice input for Pi",
2334
+ "",
2335
+ " Two backends available:",
2336
+ " • Deepgram — cloud, live streaming, $200 free credit (6–12 months of use)",
2337
+ " • Local models — fully offline, no API key, auto-downloads on first use",
2338
+ "",
2339
+ ` Audio capture: ${audioTool ? `${audioTool.name} ✓` : "not found — install sox or ffmpeg"}`,
2340
+ "",
2341
+ " Run /voice-settings to choose your backend and get started.",
2342
+ ];
2343
+ startCtx.ui.notify(lines.join("\n"), "info");
2344
+ });
2345
+
2346
+ pi.on("session_shutdown", async (event) => {
2347
+ // Synchronous teardown FIRST. Pi-mono >= 0.65.0 awaits the handler promise
2348
+ // before firing the replacement session_start, so the late-ctx-null race
2349
+ // the audit flagged is not present on current Pi. Keeping the order
2350
+ // (cleanup → null → await import) is still cheap insurance for older Pi
2351
+ // versions whose replacement path may not await. The try/catch is so a
2352
+ // throw inside voiceCleanup (e.g. a child process kill EPERM under load)
2353
+ // can't leak ctx or skip the recognizer cache clear.
2354
+ try {
2355
+ voiceCleanup();
2356
+ } catch (err) {
2357
+ voiceDebug("voiceCleanup threw during shutdown", { error: String(err) });
2358
+ }
2359
+ ctx = null;
2360
+
2361
+ // Clear the sherpa recognizer cache ONLY on terminal quit. On older Pi
2362
+ // versions (< 0.65.0) shutdown handlers are not awaited before the
2363
+ // replacement session_start, so an `await import()` here can race with
2364
+ // the new session re-initializing the recognizer — and our late
2365
+ // clearRecognizerCache() would wipe the recognizer the new session just
2366
+ // created. Keeping the cache across non-quit transitions is also faster:
2367
+ // /reload, /new, /fork, /resume typically reuse the same model+language,
2368
+ // so the recognizer is still hot. Per-session language/model changes are
2369
+ // already invalidated in voice/settings-panel.ts when the user picks a
2370
+ // different model. Reason is undefined on pre-0.65 Pi (where shutdown
2371
+ // only ever fired on quit), so we treat undefined the same as "quit".
2372
+ const reason = (event as { reason?: string } | undefined)?.reason;
2373
+ if (reason === "quit" || reason === undefined) {
2374
+ try {
2375
+ const { clearRecognizerCache } = await import("./voice/sherpa-engine");
2376
+ clearRecognizerCache();
2377
+ } catch {}
2378
+ }
2379
+ });
2380
+
2381
+ // Note: pi-mono < 0.65.0 fired a discrete "session_switch" event for
2382
+ // /new, /resume, /fork. That event was removed in 0.65.0 in favor of the
2383
+ // session_shutdown → session_start (with reason) flow handled above.
2384
+ // We don't register a shim here because package.json:peerDependencies
2385
+ // requires "@earendil-works/pi-coding-agent": "*", so a host without
2386
+ // the new flow can't install this extension in the first place.
2387
+
2388
+ // ─── Auto-speak (TTS after assistant turn ends) ─────────────────────
2389
+ //
2390
+ // When ttsAutoSpeak is true AND ttsEnabled is true, subscribe to
2391
+ // `turn_end` and pipe the assistant's response through speak() with
2392
+ // the same text-filter and length-cap logic the manual command uses.
2393
+ //
2394
+ // The handler is always registered; it short-circuits when the flags
2395
+ // are off. This way toggling `ttsAutoSpeak` via /voice-speak-toggle
2396
+ // doesn't require a session restart.
2397
+ //
2398
+ // Rate limit: track the last auto-speak timestamp and skip if a new
2399
+ // turn ends within ~3 seconds. Prevents the agent's rapid-fire
2400
+ // short responses from queueing up unread audio.
2401
+ let lastAutoSpeakAt = 0;
2402
+ const AUTO_SPEAK_RATE_LIMIT_MS = 3000;
2403
+
2404
+ // v7.2.3 — track agent busy state so autoSubmit can skip dispatch
2405
+ // when the agent is mid-turn (and especially mid-retry). Otherwise
2406
+ // holding space during a "Retrying (n/3) in Xs..." cycle queues
2407
+ // followUp messages that pile onto the failing turn — feels like
2408
+ // the voice extension is "looping" because the agent never recovers.
2409
+ let agentBusy = false;
2410
+ pi.on("agent_start", async () => {
2411
+ agentBusy = true;
2412
+ });
2413
+ pi.on("agent_end", async () => {
2414
+ agentBusy = false;
2415
+ });
2416
+
2417
+ // v7.2.1 — streaming auto-speak. Instead of waiting for `turn_end`
2418
+ // (which only fires AFTER the full response is generated), subscribe
2419
+ // to `message_update` and `message_end`:
2420
+ // - message_update fires as the LLM streams tokens. We extract
2421
+ // the accumulated text, find new sentence boundaries since
2422
+ // the last speak, and queue those sentences for synthesis.
2423
+ // - message_end flushes any remaining buffer (one final speak
2424
+ // for trailing text without a sentence terminator).
2425
+ // Result: TTS starts speaking the FIRST sentence within ~1 second
2426
+ // of the agent producing it, instead of after the entire response
2427
+ // completes. Sentence chunking + pipelined synth (already in
2428
+ // speak.ts) keeps audio flowing continuously.
2429
+ //
2430
+ // Per-message tracking: each assistant message has its own
2431
+ // `spokenLen` cursor so concurrent messages (compaction, sub-turns)
2432
+ // don't cross-talk. Map keyed by message id.
2433
+
2434
+ interface MessageStreamState {
2435
+ spokenLen: number; // chars of accumulated text already queued for speech
2436
+ pending: Promise<void>; // chain of in-flight speak() calls — serialize per message
2437
+ }
2438
+ const messageStreams = new Map<string, MessageStreamState>();
2439
+
2440
+ const SENTENCE_TERMINATORS = /[.!?](?=\s|$)|[\n]{1,}/g;
2441
+
2442
+ function extractAccumulatedText(message: any): string {
2443
+ if (!message || !Array.isArray(message.content)) return "";
2444
+ return (message.content as any[])
2445
+ .filter((c) => c?.type === "text" && typeof c.text === "string")
2446
+ .map((c) => c.text as string)
2447
+ .join("");
2448
+ }
2449
+
2450
+ /** Detect last sentence boundary at or before `maxIdx` in `text`.
2451
+ * Returns the index AFTER the terminator (so [0..idx] is a complete
2452
+ * block). Returns -1 if none found. */
2453
+ function lastSentenceEnd(text: string, fromIdx: number): number {
2454
+ let lastEnd = -1;
2455
+ const re = /[.!?](?=\s|$)|\n+/g;
2456
+ let m: RegExpExecArray | null;
2457
+ while ((m = re.exec(text)) !== null) {
2458
+ const end = m.index + m[0].length;
2459
+ if (end > fromIdx) lastEnd = end;
2460
+ }
2461
+ return lastEnd;
2462
+ }
2463
+
2464
+ // Eager-speak threshold — if the buffered new text grows past this
2465
+ // many chars without a sentence terminator (e.g. very long
2466
+ // sentence mid-stream), speak what we have at the next clause
2467
+ // boundary (`,` `;` ` — `) or flush as-is. Keeps latency low for
2468
+ // long-winded responses.
2469
+ const EAGER_SPEAK_CHARS = 80;
2470
+ const EAGER_CLAUSE_RE = /[,;:—](?=\s)/g;
2471
+
2472
+ function lastClauseEnd(text: string): number {
2473
+ let last = -1;
2474
+ EAGER_CLAUSE_RE.lastIndex = 0;
2475
+ let m: RegExpExecArray | null;
2476
+ while ((m = EAGER_CLAUSE_RE.exec(text)) !== null) {
2477
+ last = m.index + m[0].length;
2478
+ }
2479
+ return last;
2480
+ }
2481
+
2482
+ async function maybeSpeakNew(messageId: string, fullText: string, isFinal: boolean): Promise<void> {
2483
+ if (!config.ttsEnabled || !config.ttsAutoSpeak) return;
2484
+ // Don't speak while STT is hot — feedback loop hazard.
2485
+ if (voiceState === "warmup" || voiceState === "recording" || voiceState === "finalizing") return;
2486
+ if (!fullText) return;
2487
+
2488
+ let state = messageStreams.get(messageId);
2489
+ if (!state) {
2490
+ state = { spokenLen: 0, pending: Promise.resolve() };
2491
+ messageStreams.set(messageId, state);
2492
+ }
2493
+
2494
+ const newText = fullText.slice(state.spokenLen);
2495
+ if (!newText) return;
2496
+
2497
+ let speakUpTo: number;
2498
+ if (isFinal) {
2499
+ // Flush everything remaining.
2500
+ speakUpTo = newText.length;
2501
+ } else {
2502
+ // Prefer sentence boundary; fall back to clause boundary
2503
+ // if we've buffered > EAGER_SPEAK_CHARS without one.
2504
+ const boundary = lastSentenceEnd(newText, 0);
2505
+ if (boundary > 0) {
2506
+ speakUpTo = boundary;
2507
+ } else if (newText.length >= EAGER_SPEAK_CHARS) {
2508
+ const clause = lastClauseEnd(newText);
2509
+ if (clause <= 0) return; // not even a clause yet — wait
2510
+ speakUpTo = clause;
2511
+ } else {
2512
+ return; // wait for more text
2513
+ }
2514
+ }
2515
+
2516
+ const chunk = newText.slice(0, speakUpTo).trim();
2517
+ state.spokenLen += speakUpTo;
2518
+ if (!chunk) return;
2519
+ voiceDebug("autoSpeak.streaming", { id: messageId, chars: chunk.length, isFinal });
2520
+
2521
+ // Strip code blocks / links / emojis / abbreviations to spoken form.
2522
+ const { prepareForSpeech } = await import("./voice/tts-text-filter");
2523
+ const prepared = prepareForSpeech(chunk, {
2524
+ maxChars: 2000,
2525
+ stripCodeBlocks: true,
2526
+ collapseLinks: true,
2527
+ });
2528
+ if (prepared.skipped || !prepared.text.trim()) return;
2529
+
2530
+ // Serialize per-message: chain onto pending so chunks play in
2531
+ // order, never overlapping for the same message.
2532
+ const text = prepared.text;
2533
+ state.pending = state.pending.then(async () => {
2534
+ try {
2535
+ if (ctx) await runSpeak(ctx, text);
2536
+ } catch {
2537
+ // Auto-speak failures are non-blocking.
2538
+ }
2539
+ });
2540
+ }
2541
+
2542
+ // Diagnostic: stream log shows when message_update / message_end
2543
+ // actually fire (some Pi versions don't emit message_update
2544
+ // per-token). Lets us verify the streaming path is alive.
2545
+ const streamDiag = (s: string) => {
2546
+ try {
2547
+ const fs2 = require("node:fs") as typeof import("node:fs");
2548
+ fs2.appendFileSync("/tmp/pi-listen-stream.log", `[${new Date().toISOString()}] ${s}\n`);
2549
+ } catch {
2550
+ /* best-effort */
2551
+ }
2552
+ };
2553
+ let mu_count = 0;
2554
+ pi.on("message_update", async (event) => {
2555
+ const msg = (event as any)?.message;
2556
+ if (!msg || msg.role !== "assistant") return;
2557
+ const id = (msg.id as string) || "current";
2558
+ const fullText = extractAccumulatedText(msg);
2559
+ mu_count++;
2560
+ if (mu_count <= 5 || mu_count % 10 === 0) {
2561
+ streamDiag(`message_update #${mu_count} id=${id} chars=${fullText.length}`);
2562
+ }
2563
+ await maybeSpeakNew(id, fullText, false);
2564
+ });
2565
+
2566
+ pi.on("message_end", async (event) => {
2567
+ const msg = (event as any)?.message;
2568
+ if (!msg || msg.role !== "assistant") return;
2569
+ const id = (msg.id as string) || "current";
2570
+ const fullText = extractAccumulatedText(msg);
2571
+ streamDiag(`message_end id=${id} chars=${fullText.length} updates_seen=${mu_count}`);
2572
+ mu_count = 0;
2573
+ await maybeSpeakNew(id, fullText, true);
2574
+ // Drop the stream state once flushed — prevents unbounded growth.
2575
+ const state = messageStreams.get(id);
2576
+ if (state) {
2577
+ state.pending.finally(() => {
2578
+ if (messageStreams.get(id) === state) messageStreams.delete(id);
2579
+ });
2580
+ }
2581
+ });
2582
+
2583
+ // Legacy turn_end fallback — only relevant on Pi versions that
2584
+ // don't fire message_update yet. Tracked via lastAutoSpeakAt so
2585
+ // it doesn't double-speak when message_update already covered the
2586
+ // content.
2587
+ pi.on("turn_end", async (event, _evtCtx) => {
2588
+ if (!config.ttsEnabled || !config.ttsAutoSpeak) return;
2589
+ const message = (event as any)?.message;
2590
+ if (!message || message.role !== "assistant") return;
2591
+ const id = (message.id as string) || "current";
2592
+ // If message_update already drained this message's stream
2593
+ // state, the streaming path handled it — skip.
2594
+ const state = messageStreams.get(id);
2595
+ if (state) return;
2596
+ if (voiceState === "warmup" || voiceState === "recording" || voiceState === "finalizing") return;
2597
+ const now = Date.now();
2598
+ if (now - lastAutoSpeakAt < AUTO_SPEAK_RATE_LIMIT_MS) return;
2599
+ const text = extractAccumulatedText(message).trim();
2600
+ if (!text) return;
2601
+ const { prepareForSpeech } = await import("./voice/tts-text-filter");
2602
+ const prepared = prepareForSpeech(text, { maxChars: 2000, stripCodeBlocks: true, collapseLinks: true });
2603
+ if (prepared.skipped) return;
2604
+ lastAutoSpeakAt = now;
2605
+ try {
2606
+ if (ctx) await runSpeak(ctx, prepared.text);
2607
+ } catch {
2608
+ /* non-blocking */
2609
+ }
2610
+ });
2611
+
2612
+ // ─── /voice command ──────────────────────────────────────────────────────
2613
+
2614
+ pi.registerCommand("voice", {
2615
+ description: "Voice: /voice [on|off|stop|dictate|history|test|info|setup]",
2616
+ handler: async (args, cmdCtx) => {
2617
+ ctx = cmdCtx;
2618
+ const sub = (args || "").trim().toLowerCase();
2619
+
2620
+ if (sub === "on") {
2621
+ config.enabled = true;
2622
+ updateVoiceStatus();
2623
+ setupHoldToTalk();
2624
+ const backendInfo =
2625
+ config.backend === "local"
2626
+ ? `Voice enabled (local model: ${config.localModel || "whisper-small"}).`
2627
+ : "Voice enabled (Deepgram streaming).";
2628
+ cmdCtx.ui.notify(
2629
+ [
2630
+ backendInfo,
2631
+ "",
2632
+ " Hold SPACE → release to transcribe",
2633
+ ` ${toggleShortcutLabel} → toggle recording on/off`,
2634
+ " Quick SPACE tap → types a space (no voice)",
2635
+ " Escape × 2 → clear editor",
2636
+ "",
2637
+ " /voice-settings → open settings panel",
2638
+ " /voice dictate → continuous mode (no hold)",
2639
+ " /voice test → verify setup",
2640
+ "",
2641
+ " Say 'undo', 'clear', 'new line', 'period' during dictation",
2642
+ ].join("\n"),
2643
+ "info"
2644
+ );
2645
+ return;
2646
+ }
2647
+
2648
+ if (sub === "off") {
2649
+ config.enabled = false;
2650
+ voiceCleanup();
2651
+ ctx.ui.setStatus("voice", undefined);
2652
+ cmdCtx.ui.notify("Voice disabled.", "info");
2653
+ return;
2654
+ }
2655
+
2656
+ if (sub === "stop") {
2657
+ if (dictationMode) {
2658
+ dictationMode = false;
2659
+ if (voiceState === "recording") {
2660
+ await stopVoiceRecording();
2661
+ }
2662
+ cmdCtx.ui.notify("Dictation mode stopped.", "info");
2663
+ } else if (voiceState === "recording") {
2664
+ await stopVoiceRecording();
2665
+ cmdCtx.ui.notify("Recording stopped and transcribed.", "info");
2666
+ } else if (voiceState === "warmup") {
2667
+ abortPreRecording();
2668
+ clearWarmupWidget();
2669
+ hideWidget();
2670
+ resetHoldState();
2671
+ setVoiceState("idle");
2672
+ cmdCtx.ui.notify("Warmup cancelled.", "info");
2673
+ } else {
2674
+ cmdCtx.ui.notify("No recording in progress.", "info");
2675
+ }
2676
+ return;
2677
+ }
2678
+
2679
+ // /voice dictate — continuous dictation mode
2680
+
2681
+ if (sub === "dictate") {
2682
+ if (!config.enabled) {
2683
+ cmdCtx.ui.notify("Voice disabled. Use /voice on", "warning");
2684
+ return;
2685
+ }
2686
+ if (dictationMode) {
2687
+ cmdCtx.ui.notify("Already in dictation mode. /voice stop to end.", "info");
2688
+ return;
2689
+ }
2690
+ dictationMode = true;
2691
+ editorTextBeforeVoice = ctx?.hasUI ? ctx.ui.getEditorText() || "" : "";
2692
+ const ok = await startVoiceRecording();
2693
+ if (ok) {
2694
+ cmdCtx.ui.notify(
2695
+ [
2696
+ "🎤 Continuous dictation mode active.",
2697
+ "",
2698
+ " Speak freely — no need to hold SPACE.",
2699
+ " /voice stop → finalize and stop",
2700
+ ` ${toggleShortcutLabel} → also stops dictation`,
2701
+ ].join("\n"),
2702
+ "info"
2703
+ );
2704
+ } else {
2705
+ dictationMode = false;
2706
+ cmdCtx.ui.notify("Failed to start dictation.", "error");
2707
+ }
2708
+ return;
2709
+ }
2710
+
2711
+ // /voice history — show recent transcriptions
2712
+ if (sub === "history") {
2713
+ if (recordingHistory.length === 0) {
2714
+ cmdCtx.ui.notify("No recording history yet.", "info");
2715
+ return;
2716
+ }
2717
+ const lines = ["📜 Recent transcriptions:", ""];
2718
+ const show = recordingHistory.slice(0, 20);
2719
+ for (const entry of show) {
2720
+ const time = new Date(entry.timestamp).toLocaleTimeString();
2721
+ const dur = entry.duration.toFixed(1);
2722
+ const preview = entry.text.slice(0, 60) + (entry.text.length > 60 ? "…" : "");
2723
+ lines.push(` ${time} (${dur}s): ${preview}`);
2724
+ }
2725
+ if (recordingHistory.length > 20) {
2726
+ lines.push(` … and ${recordingHistory.length - 20} more`);
2727
+ }
2728
+ cmdCtx.ui.notify(lines.join("\n"), "info");
2729
+ return;
2730
+ }
2731
+
2732
+ if (sub === "test") {
2733
+ cmdCtx.ui.notify("Testing voice setup…", "info");
2734
+ const isLocal = config.backend === "local";
2735
+ const dgKey = resolveDeepgramApiKey(config);
2736
+ const tool = detectAudioCaptureTool();
2737
+
2738
+ const lines = [
2739
+ "Voice diagnostics:",
2740
+ "",
2741
+ ` Backend: ${isLocal ? "local" : "deepgram"}`,
2742
+ "",
2743
+ " Audio capture:",
2744
+ ` tool: ${tool ? `${tool.name} (${tool.cmd})` : "NONE FOUND"}`,
2745
+ ];
2746
+ if (!tool) {
2747
+ lines.push(" available: sox ✗ ffmpeg ✗ arecord ✗");
2748
+ lines.push(" install one: brew install sox (or ffmpeg)");
2749
+ }
2750
+
2751
+ if (isLocal) {
2752
+ lines.push(` local model: ${config.localModel || "whisper-small"}`);
2753
+ lines.push(` local endpoint: ${config.localEndpoint || DEFAULT_LOCAL_ENDPOINT}`);
2754
+ } else {
2755
+ lines.push(` DEEPGRAM_API_KEY: ${dgKey ? "set (" + dgKey.slice(0, 8) + "…)" : "NOT SET"}`);
2756
+ }
2757
+ lines.push("");
2758
+ lines.push(" Config:");
2759
+ lines.push(` language: ${config.language}`);
2760
+ lines.push(` onboarding: ${config.onboarding.completed ? "complete" : "incomplete"}`);
2761
+ lines.push(` hold threshold: ${getHoldThresholdMs()}ms`);
2762
+ lines.push(` toggle shortcut: ${resolvedToggleShortcut}`);
2763
+ lines.push(` kitty protocol: ${kittyReleaseDetected ? "detected" : "not detected"}`);
2764
+ lines.push(` state: ${voiceState}`);
2765
+
2766
+ // Mic capture test using detected tool
2767
+ if (tool) {
2768
+ const testFile = path.join(os.tmpdir(), "pi-voice-test.wav");
2769
+ let testProc;
2770
+ if (tool.name === "sox") {
2771
+ testProc = spawn("rec", ["-q", "-r", "16000", "-c", "1", "-b", "16", "-d", "1", testFile], {
2772
+ stdio: "pipe",
2773
+ });
2774
+ } else if (tool.name === "ffmpeg") {
2775
+ const isMac = process.platform === "darwin";
2776
+ const isLinux = process.platform === "linux";
2777
+ let testInputArgs: string[];
2778
+ if (isMac) testInputArgs = ["-f", "avfoundation", "-i", ":default"];
2779
+ else if (isLinux) testInputArgs = ["-f", "pulse", "-i", "default"];
2780
+ else {
2781
+ const dshowDev = detectWindowsAudioDevice();
2782
+ testInputArgs = dshowDev
2783
+ ? ["-f", "dshow", "-i", `audio=${dshowDev}`]
2784
+ : ["-f", "dshow", "-i", "audio=Microphone"];
2785
+ }
2786
+ const inputArgs = testInputArgs;
2787
+ testProc = spawn(
2788
+ "ffmpeg",
2789
+ [...inputArgs, "-t", "1", "-ar", "16000", "-ac", "1", "-y", "-loglevel", "error", testFile],
2790
+ { stdio: "pipe" }
2791
+ );
2792
+ } else {
2793
+ testProc = spawn("arecord", ["-q", "-f", "S16_LE", "-r", "16000", "-c", "1", "-d", "1", testFile], {
2794
+ stdio: "pipe",
2795
+ });
2796
+ }
2797
+ testProc.on("error", () => {});
2798
+ await new Promise<void>((resolve) => {
2799
+ let resolved = false;
2800
+ const done = () => {
2801
+ if (!resolved) {
2802
+ resolved = true;
2803
+ resolve();
2804
+ }
2805
+ };
2806
+ testProc.on("close", done);
2807
+ setTimeout(() => {
2808
+ try {
2809
+ testProc.kill();
2810
+ } catch {}
2811
+ done();
2812
+ }, 3000);
2813
+ });
2814
+ if (fs.existsSync(testFile)) {
2815
+ const size = fs.statSync(testFile).size;
2816
+ lines.push(` mic capture: OK (${size} bytes via ${tool.name})`);
2817
+ try {
2818
+ fs.unlinkSync(testFile);
2819
+ } catch {}
2820
+ } else {
2821
+ lines.push(` mic capture: FAILED — ${tool.name} ran but no audio captured`);
2822
+ }
2823
+ } else {
2824
+ lines.push(" mic capture: skipped (no audio tool)");
2825
+ }
2826
+
2827
+ if (isLocal && config.localEndpoint) {
2828
+ // External local server connectivity check
2829
+ const serverCheck = await checkLocalServer(config.localEndpoint);
2830
+ if (serverCheck.ok) {
2831
+ lines.push(" local server: OK (reachable)");
2832
+ } else {
2833
+ lines.push(` local server: NOT REACHABLE — ${serverCheck.error || "connection refused"}`);
2834
+ }
2835
+ } else if (isLocal) {
2836
+ // In-process sherpa-onnx mode — check module availability
2837
+ try {
2838
+ const { initSherpa, isSherpaAvailable } = await import("./voice/sherpa-engine");
2839
+ if (!isSherpaAvailable()) await initSherpa();
2840
+ const { isSherpaAvailable: checkAgain, getSherpaError } = await import("./voice/sherpa-engine");
2841
+ if (checkAgain()) {
2842
+ lines.push(" sherpa-onnx: OK (in-process mode)");
2843
+ } else {
2844
+ lines.push(` sherpa-onnx: NOT AVAILABLE — ${getSherpaError() || "unknown"}`);
2845
+ }
2846
+ } catch (e: any) {
2847
+ lines.push(` sherpa-onnx: NOT AVAILABLE — ${e?.message || e}`);
2848
+ }
2849
+ } else if (dgKey) {
2850
+ // Deepgram API key validation
2851
+ try {
2852
+ const res = await fetch("https://api.deepgram.com/v1/projects", {
2853
+ method: "GET",
2854
+ headers: { Authorization: `Token ${dgKey}` },
2855
+ signal: AbortSignal.timeout(5000),
2856
+ });
2857
+ if (res.ok) {
2858
+ lines.push(" Deepgram API: OK (key validated)");
2859
+ } else if (res.status === 401 || res.status === 403) {
2860
+ lines.push(" Deepgram API: INVALID KEY — check your API key");
2861
+ } else {
2862
+ lines.push(` Deepgram API: ERROR (HTTP ${res.status})`);
2863
+ }
2864
+ } catch (err) {
2865
+ const msg = err instanceof Error ? err.message : String(err);
2866
+ lines.push(` Deepgram API: UNREACHABLE — ${msg}`);
2867
+ }
2868
+ }
2869
+
2870
+ // Summary
2871
+ lines.push("");
2872
+ let ready: boolean;
2873
+ if (isLocal && config.localEndpoint) {
2874
+ const serverOk = (await checkLocalServer(config.localEndpoint)).ok;
2875
+ ready = !!tool && serverOk;
2876
+ if (!tool) {
2877
+ lines.push(" Setup needed — install any one of:");
2878
+ lines.push(" brew install sox # macOS (recommended)");
2879
+ lines.push(" apt install sox # Linux");
2880
+ } else if (!serverOk) {
2881
+ lines.push(" Setup needed — start a local transcription server:");
2882
+ lines.push(" whisper.cpp: ./build/bin/whisper-server -m models/ggml-small.bin --port 8080");
2883
+ lines.push(" Or any OpenAI-compatible transcription server");
2884
+ } else {
2885
+ lines.push(" All checks passed — voice is ready!");
2886
+ lines.push(` Hold SPACE to record, or use ${toggleShortcutLabel} to toggle.`);
2887
+ }
2888
+ } else if (isLocal) {
2889
+ // In-process sherpa-onnx mode — no server needed
2890
+ ready = !!tool;
2891
+ if (!tool) {
2892
+ lines.push(" Setup needed — install any one of:");
2893
+ lines.push(" brew install sox # macOS (recommended)");
2894
+ lines.push(" apt install sox # Linux");
2895
+ } else {
2896
+ lines.push(" All checks passed — voice is ready (in-process sherpa-onnx)!");
2897
+ lines.push(` Hold SPACE to record, or use ${toggleShortcutLabel} to toggle.`);
2898
+ }
2899
+ } else {
2900
+ ready = !!dgKey && !!tool;
2901
+ if (!dgKey) {
2902
+ lines.push(" Setup needed:");
2903
+ lines.push(" 1. Get a free key → https://dpgr.am/pi-voice ($200 free credit)");
2904
+ lines.push(' 2. export DEEPGRAM_API_KEY="your-key" (add to ~/.zshrc)');
2905
+ lines.push(" 3. Or run /voice-settings to configure");
2906
+ } else if (!tool) {
2907
+ lines.push(" Setup needed — install any one of:");
2908
+ lines.push(" brew install sox # macOS (recommended)");
2909
+ lines.push(" brew install ffmpeg # macOS (alternative)");
2910
+ lines.push(" apt install sox # Linux");
2911
+ lines.push(" apt install ffmpeg # Linux (alternative)");
2912
+ lines.push(" choco install sox # Windows");
2913
+ } else {
2914
+ lines.push(" All checks passed — voice is ready!");
2915
+ lines.push(` Hold SPACE to record, or use ${toggleShortcutLabel} to toggle.`);
2916
+ }
2917
+ }
2918
+
2919
+ cmdCtx.ui.notify(lines.join("\n"), ready ? "info" : "warning");
2920
+ return;
2921
+ }
2922
+
2923
+ // /voice language, /voice setup, /voice info → open settings panel
2924
+ if (sub === "language" || sub === "lang" || sub.startsWith("language ") || sub.startsWith("lang ")) {
2925
+ await openSettingsPanel(cmdCtx);
2926
+ return;
2927
+ }
2928
+
2929
+ if (sub === "info" || sub === "setup" || sub === "reconfigure" || sub === "settings" || sub === "config") {
2930
+ await openSettingsPanel(cmdCtx);
2931
+ return;
2932
+ }
2933
+
2934
+ // Default: toggle
2935
+ config.enabled = !config.enabled;
2936
+ if (!config.enabled) {
2937
+ voiceCleanup();
2938
+ } else {
2939
+ setupHoldToTalk();
2940
+ }
2941
+ updateVoiceStatus();
2942
+ cmdCtx.ui.notify(`Voice ${config.enabled ? "enabled" : "disabled"}.`, "info");
2943
+ },
2944
+ });
2945
+
2946
+ // ─── /voice-setup → redirects to settings panel ─────────────────────────
2947
+
2948
+ pi.registerCommand("voice-setup", {
2949
+ description: "Open pi-listen settings panel",
2950
+ handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx),
2951
+ });
2952
+
2953
+ // ─── /voice-language → redirects to settings panel ───────────────────────
2954
+
2955
+ pi.registerCommand("voice-language", {
2956
+ description: "Open pi-listen settings to change language",
2957
+ handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx),
2958
+ });
2959
+
2960
+ // ─── /voice-help → v7.1 §11 keyboard / command reference ────────────────
2961
+
2962
+ pi.registerCommand("voice-help", {
2963
+ description: "Show pi-listen keyboard + command reference",
2964
+ handler: async (_args, cmdCtx) => openHelpOverlay(cmdCtx),
2965
+ });
2966
+
2967
+ async function openHelpOverlay(cmdCtx: ExtensionCommandContext): Promise<void> {
2968
+ if (!cmdCtx.hasUI) {
2969
+ cmdCtx.ui.notify("pi-listen: hold space=record · /voice-speak <text> · /voice-settings · /voice-help", "info");
2970
+ return;
2971
+ }
2972
+ const { HelpOverlay } = await import("./voice/ui-help-overlay");
2973
+ await cmdCtx.ui.custom<void>((_tui, theme, _kb, done) => new HelpOverlay({ theme }, done), {
2974
+ overlay: true,
2975
+ overlayOptions: { width: "70%", minWidth: 60, maxHeight: "80%", anchor: "center" },
2976
+ });
2977
+ }
2978
+
2979
+ // ─── Settings panel (shared handler) ────────────────────────────────────
2980
+
2981
+ async function openSettingsPanel(cmdCtx: ExtensionCommandContext, initialTab?: number) {
2982
+ ctx = cmdCtx;
2983
+
2984
+ const { detectDevice, getModelFitness, formatDeviceSummary } = await import("./voice/device");
2985
+ const { getDownloadedModels, deleteModel, ensureModelDownloaded } = await import("./voice/model-download");
2986
+ const { isSherpaAvailable, clearRecognizerCache } = await import("./voice/sherpa-engine");
2987
+ const { VoiceSettingsPanel } = await import("./voice/settings-panel");
2988
+ type PanelAction = import("./voice/settings-panel").PanelAction;
2989
+ const { LANGUAGES } = await import("./voice/onboarding");
2990
+ const { resolveDeepgramApiKey } = await import("./voice/deepgram");
2991
+
2992
+ const device = detectDevice();
2993
+ // Construct the panel inside the custom() callback so the host theme
2994
+ // is in scope. Without this the panel falls back to raw ANSI which
2995
+ // clashes with non-default themes (Catppuccin Mocha etc.).
2996
+ const panelDeps = {
2997
+ config,
2998
+ device,
2999
+ cwd: currentCwd,
3000
+ getModelFitness,
3001
+ getDownloadedModels,
3002
+ deleteModel,
3003
+ isSherpaAvailable,
3004
+ formatDeviceSummary,
3005
+ saveConfig: (cfg: VoiceConfig, scope: VoiceSettingsScope, cwd: string) => saveConfig(cfg, scope, cwd),
3006
+ clearRecognizerCache: () => {
3007
+ try {
3008
+ clearRecognizerCache();
3009
+ } catch {}
3010
+ },
3011
+ resolveApiKey: () => resolveDeepgramApiKey(config) ?? undefined,
3012
+ deepgramLanguages: LANGUAGES.map((l) => ({ name: l.name, code: l.code, popular: l.popular })),
3013
+ };
3014
+
3015
+ let panel!: InstanceType<typeof VoiceSettingsPanel>;
3016
+ const result = await cmdCtx.ui.custom<PanelAction>(
3017
+ (_tui, theme, _kb, done) => {
3018
+ panel = new VoiceSettingsPanel({ ...panelDeps, theme }, initialTab);
3019
+ panel.onClose = (action) => done(action);
3020
+ return panel;
3021
+ },
3022
+ {
3023
+ overlay: true,
3024
+ overlayOptions: {
3025
+ width: "70%",
3026
+ minWidth: 44,
3027
+ maxHeight: "80%",
3028
+ anchor: "center",
3029
+ },
3030
+ }
3031
+ );
3032
+
3033
+ // Post-close: handle the speak-test action by re-using the
3034
+ // /voice-speak-test command path. We do this AFTER the panel has
3035
+ // closed so the test sample plays without the picker overlay
3036
+ // interfering with the audio cue (Pi's terminal renderer paints
3037
+ // the panel as an overlay; closing it first gives a clean
3038
+ // playback experience).
3039
+ if (result?.type === "speak-test") {
3040
+ await runSpeak(cmdCtx, "The quick brown fox jumps over the lazy dog.", { forceEnabled: true });
3041
+ return;
3042
+ }
3043
+
3044
+ // Post-close: handle the TTS install action triggered by selecting
3045
+ // a not-yet-installed model in the Speak tab Model picker.
3046
+ // v7.1: surfaces progress through the new sticky install widget
3047
+ // (`tts-install-progress.ts`) keyed by modelId so concurrent
3048
+ // installs of different models coexist without slot collision.
3049
+ if (result?.type === "tts-install" && result.modelId) {
3050
+ const { ensureTtsModelInstalled, getTtsModel } = await import("./voice/tts-local-models");
3051
+ const model = getTtsModel(result.modelId);
3052
+ // runInstallWithWidget rethrows on failure (Codex v6 #2) so
3053
+ // callers like runSpeak can short-circuit. This call site is
3054
+ // terminal — no further work — so swallow the rethrow after
3055
+ // notifications were already emitted by the helper itself.
3056
+ try {
3057
+ await runInstallWithWidget(cmdCtx, model.id, model.name, model.sizeBytes ?? 0, ensureTtsModelInstalled);
3058
+ } catch {
3059
+ /* notify already emitted in runInstallWithWidget */
3060
+ }
3061
+ return;
3062
+ }
3063
+
3064
+ // Post-close: handle download action with full pre-checks + progress
3065
+ if (result?.type === "download" && result.modelId) {
3066
+ const model = LOCAL_MODELS.find((m) => m.id === result.modelId);
3067
+ if (model) {
3068
+ const { checkDownloadPrereqs, createProgressTracker, verifyDownload, formatBytes } = await import(
3069
+ "./voice/model-download"
3070
+ );
3071
+ const { initSherpa, isSherpaAvailable, getSherpaError } = await import("./voice/sherpa-engine");
3072
+
3073
+ // ── Step 1: Check sherpa-onnx dependency ──
3074
+ if (!isSherpaAvailable()) {
3075
+ cmdCtx.ui.notify("Initializing sherpa-onnx runtime…", "info");
3076
+ const ok = await initSherpa();
3077
+ if (!ok) {
3078
+ cmdCtx.ui.notify(
3079
+ [
3080
+ "sherpa-onnx is required for local models but failed to initialize.",
3081
+ `Error: ${getSherpaError() || "unknown"}`,
3082
+ "",
3083
+ "To fix:",
3084
+ " 1. Ensure sherpa-onnx-node is installed: bun add sherpa-onnx-node",
3085
+ " 2. Check platform compatibility (macOS/Linux x64/arm64)",
3086
+ " 3. Or switch to Deepgram (cloud) backend in /voice-settings",
3087
+ ].join("\n"),
3088
+ "error"
3089
+ );
3090
+ return;
3091
+ }
3092
+ }
3093
+
3094
+ // ── Step 2: Pre-download checks (disk, network, permissions) ──
3095
+ cmdCtx.ui.notify(`Checking prerequisites for ${model.name} (${model.size})…`, "info");
3096
+ const preCheck = await checkDownloadPrereqs(model.sherpaModel.downloadUrls, model.sizeBytes);
3097
+ if (!preCheck.ok) {
3098
+ cmdCtx.ui.notify(
3099
+ [
3100
+ `Cannot download ${model.name}:`,
3101
+ "",
3102
+ ...preCheck.issues.map((i) => ` • ${i}`),
3103
+ "",
3104
+ "Resolve the above and try again via /voice-models.",
3105
+ ].join("\n"),
3106
+ "error"
3107
+ );
3108
+ return;
3109
+ }
3110
+
3111
+ // ── Step 3: Download with real-time progress ──
3112
+ const tracker = createProgressTracker(model.name);
3113
+ cmdCtx.ui.notify(`Starting download: ${model.name} (${model.size})…`, "info");
3114
+
3115
+ try {
3116
+ await ensureModelDownloaded(model.id, model.sherpaModel.downloadUrls, model.sizeBytes, (raw) => {
3117
+ const rich = tracker(raw);
3118
+ if (rich) cmdCtx.ui.notify(rich.line, "info");
3119
+ });
3120
+ } catch (err: any) {
3121
+ const msg = err?.message || String(err);
3122
+ const lines = [`Download failed: ${model.name}`];
3123
+ if (msg.includes("timed out") || msg.includes("Timeout")) {
3124
+ lines.push("The download timed out. Check your internet speed and try again.");
3125
+ } else if (msg.includes("ENOSPC") || msg.includes("no space")) {
3126
+ lines.push("Disk is full. Free up space and try again.");
3127
+ } else if (msg.includes("HTTP 4") || msg.includes("HTTP 5")) {
3128
+ lines.push(`Server error: ${msg}`);
3129
+ lines.push("The model server may be temporarily down. Try again in a few minutes.");
3130
+ } else {
3131
+ lines.push(`Error: ${msg}`);
3132
+ }
3133
+ lines.push("", "Partial downloads are auto-resumed on next attempt.");
3134
+ cmdCtx.ui.notify(lines.join("\n"), "error");
3135
+ return;
3136
+ }
3137
+
3138
+ // ── Step 4: Post-download verification ──
3139
+ const verification = verifyDownload(model.id, model.sherpaModel.downloadUrls, model.sizeBytes);
3140
+ if (!verification.ok) {
3141
+ cmdCtx.ui.notify(
3142
+ [
3143
+ `${model.name} downloaded but verification failed:`,
3144
+ "",
3145
+ ...verification.issues.map((i) => ` • ${i}`),
3146
+ "",
3147
+ "Try: /voice-models → Downloaded tab → delete and re-download.",
3148
+ ].join("\n"),
3149
+ "warning"
3150
+ );
3151
+ return;
3152
+ }
3153
+
3154
+ cmdCtx.ui.notify(`${model.name} downloaded and verified (${model.size}). Ready to use.`, "info");
3155
+ }
3156
+ }
3157
+
3158
+ // Sync voice state after panel changes
3159
+ if (config.enabled) {
3160
+ setupHoldToTalk();
3161
+ } else {
3162
+ voiceCleanup();
3163
+ }
3164
+ updateVoiceStatus();
3165
+ }
3166
+
3167
+ // ─── TTS commands (v6.0.0+) ─────────────────────────────────────────
3168
+ //
3169
+ // The active speech AbortController lives at extension scope so
3170
+ // `/voice-speak-stop` can cancel whatever is currently playing.
3171
+ // Re-entrant `/voice-speak` calls abort the prior one before starting
3172
+ // (no overlapping audio); `null` means nothing is in-flight.
3173
+ let activeSpeak: AbortController | null = null;
3174
+
3175
+ function abortActiveSpeak(): boolean {
3176
+ if (!activeSpeak) return false;
3177
+ try {
3178
+ activeSpeak.abort();
3179
+ } catch {}
3180
+ activeSpeak = null;
3181
+ return true;
3182
+ }
3183
+
3184
+ /**
3185
+ * v7.1: run an install with the new sticky `TtsInstallProgressWidget`.
3186
+ * Replaces the v7.0.x notify-spam loop. Mounts a per-model-id slot
3187
+ * (`installWidgetKey(modelId)`) so two concurrent installs for
3188
+ * different models coexist without clobbering. The widget owns its
3189
+ * own AbortController (currently abort-only via cancel()); the
3190
+ * existing in-flight Map in `ensureTtsModelInstalled` serializes
3191
+ * same-id calls.
3192
+ */
3193
+ async function runInstallWithWidget(
3194
+ cmdCtx: ExtensionCommandContext | ExtensionContext,
3195
+ modelId: string,
3196
+ modelName: string,
3197
+ totalBytesEstimate: number,
3198
+ ensureTtsModelInstalled: (
3199
+ id: string,
3200
+ opts: { signal?: AbortSignal; onProgress?: (info: any) => void }
3201
+ ) => Promise<unknown>,
3202
+ // godspeed architect finding: accept caller signal so
3203
+ // /voice-speak-stop or TTS-disable propagates into the install.
3204
+ // Caller's signal cascades: aborting it triggers our own
3205
+ // AbortController and tears down the widget cleanly.
3206
+ callerSignal?: AbortSignal
3207
+ ): Promise<void> {
3208
+ if (!cmdCtx.hasUI) {
3209
+ // Headless / scripted mode — fall back to a single notify so
3210
+ // users running pi without a TUI still get progress feedback.
3211
+ cmdCtx.ui.notify(`Installing ${modelName}…`, "info");
3212
+ try {
3213
+ await ensureTtsModelInstalled(modelId, {});
3214
+ cmdCtx.ui.notify(`${modelName} ready.`, "info");
3215
+ } catch (err: any) {
3216
+ // Codex v6.5: rethrow so callers like runSpeak short-
3217
+ // circuit instead of proceeding with a model that
3218
+ // isn't installed. Match the TUI branch's
3219
+ // `__alreadyNotified` contract so the outer catch
3220
+ // doesn't emit a duplicate notify.
3221
+ if (err?.name === "AbortError") {
3222
+ cmdCtx.ui.notify(`Install cancelled: ${modelName}`, "warning");
3223
+ } else {
3224
+ cmdCtx.ui.notify(`Install failed: ${err?.message ?? err}`, "error");
3225
+ }
3226
+ if (err && typeof err === "object") {
3227
+ try {
3228
+ (err as any).__alreadyNotified = true;
3229
+ } catch {
3230
+ /* frozen errors */
3231
+ }
3232
+ }
3233
+ throw err;
3234
+ }
3235
+ return;
3236
+ }
3237
+
3238
+ const { registry, ticker } = getOrInitVoiceUi();
3239
+ const controller = new AbortController();
3240
+ // Cascade caller signal → controller. If runSpeak's activeSpeak
3241
+ // signal aborts (user runs /voice-speak-stop or disables TTS),
3242
+ // the install cancels too. Listener removed in finally.
3243
+ let callerAbortListener: (() => void) | null = null;
3244
+ if (callerSignal) {
3245
+ if (callerSignal.aborted) {
3246
+ try {
3247
+ controller.abort();
3248
+ } catch {}
3249
+ } else {
3250
+ callerAbortListener = () => {
3251
+ try {
3252
+ controller.abort();
3253
+ } catch {}
3254
+ };
3255
+ callerSignal.addEventListener("abort", callerAbortListener);
3256
+ }
3257
+ }
3258
+ const widget = new TtsInstallProgressWidget({
3259
+ ui: cmdCtx.ui,
3260
+ modelId,
3261
+ modelName,
3262
+ totalBytesEstimate,
3263
+ registry,
3264
+ ticker,
3265
+ controller,
3266
+ });
3267
+ activeInstallWidgets.set(modelId, widget);
3268
+ try {
3269
+ await ensureTtsModelInstalled(modelId, {
3270
+ signal: controller.signal,
3271
+ onProgress: (info) => widget.onProgress(info),
3272
+ });
3273
+ // Widget self-disposes on phase=done; if ensure resolved
3274
+ // without firing done (very unlikely), make sure cleanup
3275
+ // runs anyway.
3276
+ widget.dispose();
3277
+ cmdCtx.ui.notify(`${modelName} ready.`, "info");
3278
+ } catch (err: any) {
3279
+ widget.dispose();
3280
+ // Codex v6 finding #2: rethrow so the caller (e.g. runSpeak)
3281
+ // can short-circuit instead of proceeding into speak() with
3282
+ // a model that isn't installed. We notify once here with the
3283
+ // install context; the rethrown error will be caught by the
3284
+ // caller's outer try/catch but tagged with `__alreadyNotified`
3285
+ // so the caller can skip its generic notify.
3286
+ if (err?.name === "AbortError") {
3287
+ cmdCtx.ui.notify(`Install cancelled: ${modelName}`, "warning");
3288
+ } else {
3289
+ cmdCtx.ui.notify(`Install failed: ${err?.message ?? err}`, "error");
3290
+ }
3291
+ if (err && typeof err === "object") {
3292
+ try {
3293
+ (err as any).__alreadyNotified = true;
3294
+ } catch {
3295
+ /* frozen errors */
3296
+ }
3297
+ }
3298
+ throw err;
3299
+ } finally {
3300
+ // Codex v6 finding #4: owner-checked delete so an older
3301
+ // finally cannot evict a newer same-id widget. (The
3302
+ // in-flight Map in ensureTtsModelInstalled already serializes
3303
+ // same-id installs, but the side-table here doesn't piggy-
3304
+ // back on that guarantee — defensive owner check.)
3305
+ if (activeInstallWidgets.get(modelId) === widget) {
3306
+ activeInstallWidgets.delete(modelId);
3307
+ }
3308
+ // Always remove the caller-signal listener.
3309
+ if (callerAbortListener && callerSignal) {
3310
+ try {
3311
+ callerSignal.removeEventListener("abort", callerAbortListener);
3312
+ } catch {}
3313
+ }
3314
+ }
3315
+ }
3316
+
3317
+ async function runSpeak(
3318
+ cmdCtx: ExtensionCommandContext | ExtensionContext,
3319
+ text: string,
3320
+ opts: { forceEnabled?: boolean } = {}
3321
+ ): Promise<void> {
3322
+ // `forceEnabled` lets /voice-speak-test bypass the gate without
3323
+ // mutating shared config. The previous mutate-snapshot-restore
3324
+ // pattern raced against /voice-speak-toggle and could clobber the
3325
+ // user's explicit toggle.
3326
+ if (!config.ttsEnabled && !opts.forceEnabled) {
3327
+ cmdCtx.ui.notify("TTS is disabled. Enable in /voice-settings.", "warning");
3328
+ return;
3329
+ }
3330
+ if (voiceState === "recording" || voiceState === "finalizing") {
3331
+ // Speaking while the mic is hot would feedback into STT.
3332
+ cmdCtx.ui.notify("Cannot speak while recording. Stop recording first.", "warning");
3333
+ return;
3334
+ }
3335
+
3336
+ // Cancel any in-flight speech so the new request takes the floor.
3337
+ abortActiveSpeak();
3338
+ const controller = new AbortController();
3339
+ activeSpeak = controller;
3340
+
3341
+ try {
3342
+ const { speak } = await import("./voice/speak");
3343
+ const { getInstalledTtsModelDir, ensureTtsModelInstalled, getTtsModel } = await import(
3344
+ "./voice/tts-local-models"
3345
+ );
3346
+
3347
+ // On the local backend, fetch the model on-demand if missing.
3348
+ // Deepgram backend skips this branch entirely. v7.1: surface
3349
+ // progress through the sticky install widget instead of the
3350
+ // v7.0.x notify-spam loop.
3351
+ if ((config.ttsBackend ?? "local") === "local") {
3352
+ const modelId = config.ttsLocalModel || "kitten-nano-en-v0_2";
3353
+ try {
3354
+ getInstalledTtsModelDir(modelId);
3355
+ } catch {
3356
+ const model = getTtsModel(modelId);
3357
+ await runInstallWithWidget(
3358
+ cmdCtx,
3359
+ modelId,
3360
+ model.name,
3361
+ model.sizeBytes ?? 0,
3362
+ ensureTtsModelInstalled,
3363
+ controller.signal
3364
+ );
3365
+ }
3366
+ }
3367
+
3368
+ // v7.1: mount the honest playback indicator (§6 of plan).
3369
+ // Spinner + state word with no fake amplitude meter. Until
3370
+ // `speak()` exposes a phase callback (v7.2), the indicator
3371
+ // stays on "playing" for the whole synth+play cycle —
3372
+ // honest because audio IS in flight throughout. Disposed
3373
+ // in finally regardless of success/abort/error. Tracked
3374
+ // on `activePlaybackIndicator` so the [esc] router can
3375
+ // stop playback when no install widget owns escape.
3376
+ let indicator: TtsPlaybackIndicator | null = null;
3377
+ if (cmdCtx.hasUI) {
3378
+ const { registry, ticker } = getOrInitVoiceUi();
3379
+ indicator = new TtsPlaybackIndicator({
3380
+ ui: cmdCtx.ui,
3381
+ registry,
3382
+ ticker,
3383
+ onStop: () => abortActiveSpeak(),
3384
+ });
3385
+ indicator.setState("playing");
3386
+ activePlaybackIndicator = indicator;
3387
+ }
3388
+ try {
3389
+ await speak({
3390
+ text,
3391
+ config,
3392
+ signal: controller.signal,
3393
+ resolveModelDir: (id) => getInstalledTtsModelDir(id),
3394
+ });
3395
+ } finally {
3396
+ indicator?.setState("idle"); // self-disposes
3397
+ // Owner-checked clear, mirroring runInstallWithWidget's
3398
+ // Codex v6 #4 fix.
3399
+ if (activePlaybackIndicator === indicator) activePlaybackIndicator = null;
3400
+ }
3401
+ } catch (err: any) {
3402
+ if (err?.name === "AbortError") return;
3403
+ // If the install widget already notified the user with a
3404
+ // scoped message ("Install failed: <model>"), don't emit a
3405
+ // duplicate generic "Speak failed:" notify. The install
3406
+ // widget's notify is more informative for that path.
3407
+ if (!err?.__alreadyNotified) {
3408
+ cmdCtx.ui.notify(`Speak failed: ${err?.message ?? err}`, "error");
3409
+ }
3410
+ } finally {
3411
+ if (activeSpeak === controller) activeSpeak = null;
3412
+ }
3413
+ }
3414
+
3415
+ pi.registerCommand("voice-speak", {
3416
+ description: "Speak the given text (text-to-speech)",
3417
+ handler: async (args, cmdCtx) => {
3418
+ ctx = cmdCtx;
3419
+ const text = (args || "").trim();
3420
+ if (!text) {
3421
+ cmdCtx.ui.notify("Usage: /voice-speak <text>", "warning");
3422
+ return;
3423
+ }
3424
+ await runSpeak(cmdCtx, text);
3425
+ },
3426
+ });
3427
+
3428
+ // v7.1.3 — toggle Deepgram WebSocket streaming TTS (cloud backend).
3429
+ // When ON, /voice-speak uses wss://api.deepgram.com/v1/speak so audio
3430
+ // frames stream into the local player as they arrive (sub-200ms
3431
+ // TTFA in good network conditions). When OFF, the REST `/v1/speak`
3432
+ // path returns a complete WAV.
3433
+ pi.registerCommand("voice-stream", {
3434
+ description: "Toggle Deepgram WebSocket streaming TTS (cloud)",
3435
+ handler: async (args, cmdCtx) => {
3436
+ ctx = cmdCtx;
3437
+ const trimmed = (args || "").trim().toLowerCase();
3438
+ let next: boolean;
3439
+ if (trimmed === "on") next = true;
3440
+ else if (trimmed === "off") next = false;
3441
+ else next = !(config.ttsDeepgramStreaming === true);
3442
+ config.ttsDeepgramStreaming = next;
3443
+ saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
3444
+ cmdCtx.ui.notify(`Deepgram WebSocket streaming TTS: ${next ? "ON" : "OFF"}`, "info");
3445
+ },
3446
+ });
3447
+
3448
+ // v7.1.3 — tune the hold-to-talk activation delay.
3449
+ pi.registerCommand("voice-hold-delay", {
3450
+ description: "Set hold-to-talk delay in ms (200-3000, default 700)",
3451
+ handler: async (args, cmdCtx) => {
3452
+ ctx = cmdCtx;
3453
+ const trimmed = (args || "").trim();
3454
+ if (!trimmed) {
3455
+ cmdCtx.ui.notify(`Hold delay: ${getHoldThresholdMs()}ms (default 700)`, "info");
3456
+ return;
3457
+ }
3458
+ const ms = parseInt(trimmed, 10);
3459
+ if (!Number.isFinite(ms) || ms < 200 || ms > 3000) {
3460
+ cmdCtx.ui.notify(`Invalid value: ${trimmed}. Must be 200-3000 ms.`, "warning");
3461
+ return;
3462
+ }
3463
+ (config as any).holdThresholdMs = ms;
3464
+ saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
3465
+ cmdCtx.ui.notify(`Hold delay set to ${ms}ms.`, "info");
3466
+ },
3467
+ });
3468
+
3469
+ // v7.1.1 — toggle auto-submit on STT (sends transcribed text
3470
+ // directly to the agent instead of just placing it in the editor).
3471
+ pi.registerCommand("voice-autosubmit", {
3472
+ description: "Toggle auto-submit on STT — sends spoken text to the agent immediately",
3473
+ handler: async (args, cmdCtx) => {
3474
+ ctx = cmdCtx;
3475
+ const trimmed = (args || "").trim().toLowerCase();
3476
+ let next: boolean;
3477
+ if (trimmed === "on") next = true;
3478
+ else if (trimmed === "off") next = false;
3479
+ else next = !(config.autoSubmitOnSpeak === true);
3480
+ config.autoSubmitOnSpeak = next;
3481
+ saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
3482
+ cmdCtx.ui.notify(`Auto-submit on speak: ${next ? "ON" : "OFF"}`, "info");
3483
+ },
3484
+ });
3485
+
3486
+ pi.registerCommand("voice-speak-stop", {
3487
+ description: "Stop in-flight TTS playback",
3488
+ handler: async (_args, cmdCtx) => {
3489
+ ctx = cmdCtx;
3490
+ if (abortActiveSpeak()) {
3491
+ cmdCtx.ui.notify("Speech stopped.", "info");
3492
+ } else {
3493
+ cmdCtx.ui.notify("No active speech.", "info");
3494
+ }
3495
+ },
3496
+ });
3497
+
3498
+ pi.registerCommand("voice-speak-toggle", {
3499
+ description: "Toggle TTS on/off (master switch)",
3500
+ handler: async (_args, cmdCtx) => {
3501
+ ctx = cmdCtx;
3502
+ const nowEnabling = !config.ttsEnabled;
3503
+ config.ttsEnabled = nowEnabling;
3504
+ saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
3505
+ if (!nowEnabling) abortActiveSpeak();
3506
+ cmdCtx.ui.notify(`TTS ${nowEnabling ? "enabled" : "disabled"}.`, "info");
3507
+ // First-time enable → v7.1 §9 rich onboarding overlay with
3508
+ // three explicit actions (try / pick model / skip).
3509
+ // Subsequent toggles are silent.
3510
+ if (nowEnabling && !(config as any).ttsOnboardingShown && cmdCtx.hasUI) {
3511
+ try {
3512
+ const { detectDevice } = await import("./voice/device");
3513
+ const { TtsOnboardingOverlay } = await import("./voice/tts-onboarding-overlay");
3514
+ const device = detectDevice();
3515
+ // §9: persist `ttsOnboardingShown = true` BEFORE any
3516
+ // async work so a failed install/cancel never re-prompts.
3517
+ (config as any).ttsOnboardingShown = true;
3518
+ saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
3519
+
3520
+ const result = await cmdCtx.ui.custom<import("./voice/tts-onboarding-overlay").OnboardingResult>(
3521
+ (_tui, theme, _kb, done) => {
3522
+ return new TtsOnboardingOverlay({ systemLocale: device.systemLocale, theme }, done);
3523
+ },
3524
+ {
3525
+ overlay: true,
3526
+ overlayOptions: { width: "70%", minWidth: 60, maxHeight: "60%", anchor: "center" },
3527
+ }
3528
+ );
3529
+ if (result?.kind === "test") {
3530
+ await runSpeak(cmdCtx, "The quick brown fox jumps over the lazy dog.", { forceEnabled: true });
3531
+ } else if (result?.kind === "pickModel") {
3532
+ // Open the settings panel directly on the Speak tab.
3533
+ // Tab order is general/models/downloaded/speak/device → idx 3.
3534
+ await openSettingsPanel(cmdCtx, 3);
3535
+ }
3536
+ } catch (err) {
3537
+ voiceDebug("onboarding overlay threw", String(err));
3538
+ }
3539
+ } else if (nowEnabling && !cmdCtx.hasUI) {
3540
+ // Headless mode — fall back to the v7.0 notify-based hint.
3541
+ try {
3542
+ const { detectDevice } = await import("./voice/device");
3543
+ const { maybeShowTtsOnboarding } = await import("./voice/tts-onboarding");
3544
+ maybeShowTtsOnboarding({
3545
+ ctx: cmdCtx,
3546
+ config,
3547
+ device: detectDevice(),
3548
+ cwd: currentCwd,
3549
+ saveConfig: (cfg, scope, cwd) => saveConfig(cfg, scope, cwd),
3550
+ });
3551
+ } catch {
3552
+ /* onboarding hint is best-effort */
3553
+ }
3554
+ }
3555
+ },
3556
+ });
3557
+
3558
+ pi.registerCommand("voice-speak-test", {
3559
+ description: "Synthesize a sample sentence in the current voice",
3560
+ handler: async (_args, cmdCtx) => {
3561
+ ctx = cmdCtx;
3562
+ // Pass forceEnabled so the test runs even when ttsEnabled is
3563
+ // false — without mutating shared config (a concurrent
3564
+ // /voice-speak-toggle would otherwise be clobbered when the
3565
+ // test's finally restored its snapshot).
3566
+ await runSpeak(cmdCtx, "The quick brown fox jumps over the lazy dog.", { forceEnabled: true });
3567
+ },
3568
+ });
3569
+
3570
+ pi.registerCommand("voice-speak-info", {
3571
+ description: "Show TTS configuration: backend, model, voice, install state",
3572
+ handler: async (_args, cmdCtx) => {
3573
+ ctx = cmdCtx;
3574
+ const { getTtsModel, isTtsModelInstalled, TTS_LOCAL_MODELS } = await import("./voice/tts-local-models");
3575
+ const { DEEPGRAM_TTS_VOICES } = await import("./voice/tts-deepgram");
3576
+ const { resolveDeepgramApiKey } = await import("./voice/deepgram");
3577
+
3578
+ const isLocal = (config.ttsBackend ?? "local") === "local";
3579
+ const lines: string[] = ["TTS configuration:", ""];
3580
+ lines.push(` Enabled: ${config.ttsEnabled ? "yes" : "no"}`);
3581
+ lines.push(` Backend: ${isLocal ? "local (sherpa-onnx)" : "deepgram (cloud REST)"}`);
3582
+ lines.push(` Language: ${config.ttsLanguage ?? config.language ?? "en"}`);
3583
+ lines.push(` Speed: ${(config.ttsSpeed ?? 1.0).toFixed(2)}x`);
3584
+ lines.push(` Auto-speak: ${config.ttsAutoSpeak ? "yes" : "no"}`);
3585
+ lines.push("");
3586
+
3587
+ if (isLocal) {
3588
+ const modelId = config.ttsLocalModel ?? "kitten-nano-en-v0_2";
3589
+ let model;
3590
+ try {
3591
+ model = getTtsModel(modelId);
3592
+ } catch {
3593
+ model = undefined;
3594
+ }
3595
+ const installed = isTtsModelInstalled(modelId);
3596
+ lines.push(" Local backend:");
3597
+ lines.push(` Model: ${modelId}${model ? ` (${model.name}, ${model.size})` : " — unknown id"}`);
3598
+ lines.push(` Installed: ${installed ? "yes" : "NO — first speak will download"}`);
3599
+ if (model) {
3600
+ const sid = typeof config.ttsLocalVoiceId === "number" ? config.ttsLocalVoiceId : model.defaultSid;
3601
+ const voice = model.voices.find((v) => v.sid === sid);
3602
+ lines.push(` Voice sid: ${sid}${voice ? ` (${voice.name})` : ""}`);
3603
+ lines.push(` Languages: ${model.languages.join(", ")}`);
3604
+ lines.push(` Sample rate: ${model.sampleRate} Hz`);
3605
+ }
3606
+ lines.push("");
3607
+ lines.push(` Catalog: ${TTS_LOCAL_MODELS.length} models available — /voice-speak-models to browse`);
3608
+ } else {
3609
+ const voiceId = config.ttsDeepgramVoiceId ?? "aura-asteria-en";
3610
+ const voice = DEEPGRAM_TTS_VOICES.find((v) => v.id === voiceId);
3611
+ const apiKey = resolveDeepgramApiKey(config);
3612
+ lines.push(" Deepgram backend:");
3613
+ lines.push(` Voice: ${voiceId}${voice ? ` (${voice.name})` : ""}`);
3614
+ lines.push(` API key: ${apiKey ? `set (${apiKey.slice(0, 8)}…)` : "NOT SET — set DEEPGRAM_API_KEY"}`);
3615
+ lines.push(` Catalog: ${DEEPGRAM_TTS_VOICES.length} Aura voices surfaced`);
3616
+ }
3617
+
3618
+ lines.push("");
3619
+ lines.push(" Commands: /voice-speak <text> · /voice-speak-stop · /voice-speak-toggle");
3620
+ lines.push(" /voice-speak-test · /voice-speak-models · /voice-settings");
3621
+ cmdCtx.ui.notify(lines.join("\n"), "info");
3622
+ },
3623
+ });
3624
+
3625
+ pi.registerCommand("voice-speak-models", {
3626
+ description: "Browse and install TTS models (opens settings panel on Speak tab)",
3627
+ // initialTab=3 → Speak tab (index 0=General, 1=Models, 2=Downloaded, 3=Speak, 4=Device)
3628
+ handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx, 3),
3629
+ });
3630
+
3631
+ // ─── /voice-settings — unified pi-listen settings panel ─────────────
3632
+
3633
+ pi.registerCommand("voice-settings", {
3634
+ description: "Open pi-listen settings — backend, models, language, device",
3635
+ handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx),
3636
+ });
3637
+
3638
+ // ─── /voice-models — opens settings panel on Models tab ─────────────
3639
+
3640
+ pi.registerCommand("voice-models", {
3641
+ description: "Manage local voice models (opens settings panel)",
3642
+ handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx, 1),
3643
+ });
3644
+ }