pi-voicekit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +341 -0
- package/extensions/voice/config.ts +395 -0
- package/extensions/voice/deepgram.ts +33 -0
- package/extensions/voice/device.ts +382 -0
- package/extensions/voice/hold-to-talk.ts +69 -0
- package/extensions/voice/local.ts +1143 -0
- package/extensions/voice/model-download.ts +636 -0
- package/extensions/voice/onboarding.ts +739 -0
- package/extensions/voice/release-controller.ts +55 -0
- package/extensions/voice/settings-panel.ts +1602 -0
- package/extensions/voice/sherpa-engine.ts +464 -0
- package/extensions/voice/sherpa-loader.ts +143 -0
- package/extensions/voice/sherpa-onnx-node.d.ts +4 -0
- package/extensions/voice/speak.ts +430 -0
- package/extensions/voice/tts-deepgram.ts +454 -0
- package/extensions/voice/tts-engine.ts +653 -0
- package/extensions/voice/tts-install-progress.ts +257 -0
- package/extensions/voice/tts-local-models.ts +1255 -0
- package/extensions/voice/tts-onboarding-overlay.ts +186 -0
- package/extensions/voice/tts-onboarding.ts +87 -0
- package/extensions/voice/tts-playback-indicator.ts +127 -0
- package/extensions/voice/tts-playback.ts +675 -0
- package/extensions/voice/tts-text-filter.ts +404 -0
- package/extensions/voice/ui-aura.ts +272 -0
- package/extensions/voice/ui-help-overlay.ts +161 -0
- package/extensions/voice/ui-icons.ts +124 -0
- package/extensions/voice/ui-locale-labels.ts +110 -0
- package/extensions/voice/ui-picker.ts +209 -0
- package/extensions/voice/ui-render-ticker.ts +171 -0
- package/extensions/voice/ui-widget-base.ts +219 -0
- package/extensions/voice/ui-width.ts +112 -0
- package/extensions/voice.ts +3644 -0
- package/package.json +75 -0
|
@@ -0,0 +1,3644 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* pi-voice — Enterprise-grade voice STT for Pi CLI.
|
|
3
|
+
*
|
|
4
|
+
* Architecture (modeled after Claude Code's voice pipeline):
|
|
5
|
+
*
|
|
6
|
+
* STATE MACHINE
|
|
7
|
+
* ─────────────
|
|
8
|
+
* idle → warmup → recording → finalizing → idle
|
|
9
|
+
* ↑ │
|
|
10
|
+
* └─────────┘ (rapid re-press recovery)
|
|
11
|
+
*
|
|
12
|
+
* warmup: User holds SPACE for ≥ HOLD_THRESHOLD_MS (1200ms).
|
|
13
|
+
* A "keep holding…" hint with countdown is shown. If released before
|
|
14
|
+
* the threshold, a normal space character is typed (or "hold longer" hint shown).
|
|
15
|
+
*
|
|
16
|
+
* recording: SoX captures PCM → Deepgram WebSocket streaming.
|
|
17
|
+
* Live interim + final transcripts update the widget.
|
|
18
|
+
* Release SPACE (or press again in toggle mode) → stop.
|
|
19
|
+
*
|
|
20
|
+
* finalizing: CloseStream sent to Deepgram. Waiting for final
|
|
21
|
+
* transcript. Safety timeout auto-completes.
|
|
22
|
+
*
|
|
23
|
+
* HOLD-TO-TALK DETECTION
|
|
24
|
+
* ──────────────────────
|
|
25
|
+
* Two paths depending on terminal capabilities:
|
|
26
|
+
*
|
|
27
|
+
* A) Kitty protocol (Ghostty on Linux, Kitty, WezTerm):
|
|
28
|
+
* True key-down/repeat/release events available.
|
|
29
|
+
* First SPACE press → enter warmup immediately (show countdown).
|
|
30
|
+
* Released < 300ms → tap → type a space.
|
|
31
|
+
* Released 300ms–2s → show "hold longer" hint.
|
|
32
|
+
* Held ≥ 1.2s → activate recording.
|
|
33
|
+
* True release event stops recording.
|
|
34
|
+
*
|
|
35
|
+
* B) Non-Kitty (macOS Terminal, Ghostty on macOS):
|
|
36
|
+
* No key-release event. Holding sends rapid press events (~30-90ms apart).
|
|
37
|
+
* First SPACE press → record time, start release-detect timer (500ms).
|
|
38
|
+
* No more presses within 500ms → TAP → type a space.
|
|
39
|
+
* Rapid presses detected → user is HOLDING.
|
|
40
|
+
* After REPEAT_CONFIRM_COUNT (6) rapid presses → enter warmup.
|
|
41
|
+
* After HOLD_THRESHOLD_MS (1200ms) from first press → activate recording.
|
|
42
|
+
* Gap > RELEASE_DETECT_MS (500ms) after RECORDING_GRACE_MS (800ms) → stop.
|
|
43
|
+
*
|
|
44
|
+
* ENTERPRISE FALLBACKS
|
|
45
|
+
* ────────────────────
|
|
46
|
+
* • Session corruption guard: new recording request during
|
|
47
|
+
* finalizing automatically cancels the stale session first.
|
|
48
|
+
* • Stale transcript cleanup: any prior transcript is cleared
|
|
49
|
+
* before new recording begins.
|
|
50
|
+
* • Silence vs. no-speech: distinguishes "mic captured silence"
|
|
51
|
+
* from "no speech detected" with distinct user messages.
|
|
52
|
+
*
|
|
53
|
+
* Activation:
|
|
54
|
+
* - Hold SPACE (≥1200ms) → release to finalize
|
|
55
|
+
* - Configurable shortcut (default Ctrl+Shift+V) → toggle start/stop (always works)
|
|
56
|
+
|
|
57
|
+
*
|
|
58
|
+
* Config in ~/.pi/agent/settings.json under "voice": { ... }
|
|
59
|
+
*/
|
|
60
|
+
|
|
61
|
+
import type { ExtensionAPI, ExtensionContext, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
62
|
+
import { isKeyRelease, isKeyRepeat, matchesKey, Key, type KeyId } from "@earendil-works/pi-tui";
|
|
63
|
+
|
|
64
|
+
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
|
|
65
|
+
import * as fs from "node:fs";
|
|
66
|
+
import * as os from "node:os";
|
|
67
|
+
import * as path from "node:path";
|
|
68
|
+
import {
|
|
69
|
+
DEFAULT_CONFIG,
|
|
70
|
+
getSessionStartPersistedConfig,
|
|
71
|
+
loadConfigWithSource,
|
|
72
|
+
loadGlobalToggleShortcut,
|
|
73
|
+
saveConfig,
|
|
74
|
+
type VoiceConfig,
|
|
75
|
+
type VoiceSettingsScope,
|
|
76
|
+
} from "./voice/config";
|
|
77
|
+
import {
|
|
78
|
+
finalizeOnboardingConfig,
|
|
79
|
+
runVoiceOnboarding,
|
|
80
|
+
pickLanguage,
|
|
81
|
+
languageDisplayName,
|
|
82
|
+
modelForLanguage,
|
|
83
|
+
} from "./voice/onboarding";
|
|
84
|
+
import { makeWidgetRegistry, type WidgetRegistry } from "./voice/ui-widget-base";
|
|
85
|
+
import { makeRenderTicker, type RenderTicker } from "./voice/ui-render-ticker";
|
|
86
|
+
import { TtsInstallProgressWidget } from "./voice/tts-install-progress";
|
|
87
|
+
import { TtsPlaybackIndicator } from "./voice/tts-playback-indicator";
|
|
88
|
+
import { buildDeepgramWsUrl, resolveDeepgramApiKey, SAMPLE_RATE, CHANNELS } from "./voice/deepgram";
|
|
89
|
+
import {
|
|
90
|
+
startLocalSession,
|
|
91
|
+
stopLocalSession,
|
|
92
|
+
abortLocalSession,
|
|
93
|
+
checkLocalServer,
|
|
94
|
+
LOCAL_MODELS,
|
|
95
|
+
DEFAULT_LOCAL_ENDPOINT,
|
|
96
|
+
getLanguagesForLocalModel,
|
|
97
|
+
isLanguageSupportedByModel,
|
|
98
|
+
localLanguageDisplayName,
|
|
99
|
+
type LocalSession,
|
|
100
|
+
} from "./voice/local";
|
|
101
|
+
import { shouldArmReleaseDetectOnRepeat, decideRecordingStartTimer } from "./voice/hold-to-talk";
|
|
102
|
+
import { GapTimer, type TimerPort } from "./voice/release-controller";
|
|
103
|
+
|
|
104
|
+
/** Adapter for the real event loop — lets GapTimer run under the real setTimeout. */
|
|
105
|
+
const realTimerPort: TimerPort = {
|
|
106
|
+
once(fn, ms) {
|
|
107
|
+
const id = setTimeout(fn, ms);
|
|
108
|
+
return { cancel: () => clearTimeout(id) };
|
|
109
|
+
},
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
// ─── Types ───────────────────────────────────────────────────────────────────
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Voice state machine — strict transitions only:
|
|
116
|
+
* idle → warmup → recording → finalizing → idle
|
|
117
|
+
* warmup → idle (released before threshold)
|
|
118
|
+
* recording → idle (on error)
|
|
119
|
+
* finalizing → idle (on completion or timeout)
|
|
120
|
+
*/
|
|
121
|
+
type VoiceState = "idle" | "warmup" | "recording" | "finalizing";
|
|
122
|
+
|
|
123
|
+
// ─── Constants ───────────────────────────────────────────────────────────────
|
|
124
|
+
|
|
125
|
+
const KEEPALIVE_INTERVAL_MS = 8000;
|
|
126
|
+
const MAX_RECORDING_SECS = 120;
|
|
127
|
+
const STREAM_FINALIZE_TIMEOUT_MS = 2500;
|
|
128
|
+
|
|
129
|
+
// Hold-to-talk timing — Apple-style deliberate hold detection
|
|
130
|
+
// The goal: typing normally should NEVER accidentally trigger voice.
|
|
131
|
+
// Only a clearly intentional long press activates it.
|
|
132
|
+
// v7.1.3 — hold threshold is configurable via `voice.holdThresholdMs`.
|
|
133
|
+
// Default lowered from 1200ms → 700ms for snappier hold-to-talk activation.
|
|
134
|
+
// Apple Caps Lock uses ~1s; modern push-to-talk apps tend to use 500-800ms.
|
|
135
|
+
// 700ms strikes a balance: too short risks accidental activation while typing
|
|
136
|
+
// (a single tap on the spacebar at the end of a word is ~80ms), too long
|
|
137
|
+
// feels laggy. Users can dial higher via /voice-hold-delay or settings.json.
|
|
138
|
+
const HOLD_THRESHOLD_DEFAULT_MS = 700;
|
|
139
|
+
const RELEASE_DETECT_MS = 500; // Gap in key-repeat that means "released" (non-Kitty)
|
|
140
|
+
// macOS default InitialKeyRepeat is ~375ms, so 500ms
|
|
141
|
+
// ensures the first repeat arrives before we decide "tap"
|
|
142
|
+
const REPEAT_CONFIRM_COUNT = 6; // Need this many rapid repeat presses to confirm "holding"
|
|
143
|
+
// At ~30ms repeat rate, 6 presses ≈ 180ms of continuous holding
|
|
144
|
+
// This filters out brief pauses while typing
|
|
145
|
+
const REPEAT_CONFIRM_MS = 700; // Max gap between presses to count as rapid repeat
|
|
146
|
+
// macOS initial key-repeat delay is ~417-583ms depending on settings
|
|
147
|
+
// Must be > macOS InitialKeyRepeat (~375ms)
|
|
148
|
+
const RECORDING_GRACE_MS = 800; // After recording starts, ignore release for this long
|
|
149
|
+
// Covers async gap from holdActivationTimer → startVoiceRecording
|
|
150
|
+
const RELEASE_DETECT_RECORDING_MS = 250; // During active recording, gap before we consider
|
|
151
|
+
// the key released (non-Kitty only). macOS Terminal
|
|
152
|
+
// key repeat fires every ~30-50ms. 250ms gap = released.
|
|
153
|
+
const TYPING_COOLDOWN_MS = 400; // If ANY non-space key was pressed within this window,
|
|
154
|
+
// ignore space holds (user is typing, not activating voice)
|
|
155
|
+
const TAIL_RECORDING_MS = 1500; // Keep recording for 1.5s after space release to catch
|
|
156
|
+
// trailing words. If user re-presses space within this
|
|
157
|
+
// window, cancel the delayed stop and keep recording.
|
|
158
|
+
const CORRUPTION_GUARD_MS = 200; // Min gap between stop and restart
|
|
159
|
+
|
|
160
|
+
// Debug logging — set PI_VOICE_DEBUG=1 to enable
|
|
161
|
+
const VOICE_DEBUG = !!process.env.PI_VOICE_DEBUG;
|
|
162
|
+
const VOICE_LOG_FILE = path.join(os.tmpdir(), "pi-voice-debug.log");
|
|
163
|
+
|
|
164
|
+
// ─── Audio level tracking (module scope so streaming can access) ──────
|
|
165
|
+
let audioLevel = 0;
|
|
166
|
+
let audioLevelSmoothed = 0;
|
|
167
|
+
|
|
168
|
+
function updateAudioLevel(chunk: Buffer) {
|
|
169
|
+
const len = chunk.length;
|
|
170
|
+
if (len < 2) return;
|
|
171
|
+
const samples = len >> 1;
|
|
172
|
+
let sum = 0;
|
|
173
|
+
// Use Int16Array view when alignment permits (2-byte aligned), else fall back
|
|
174
|
+
if ((chunk.byteOffset & 1) === 0) {
|
|
175
|
+
const view = new Int16Array(chunk.buffer, chunk.byteOffset, samples);
|
|
176
|
+
for (let i = 0; i < view.length; i++) {
|
|
177
|
+
sum += view[i] * view[i];
|
|
178
|
+
}
|
|
179
|
+
} else {
|
|
180
|
+
for (let i = 0; i < len - 1; i += 2) {
|
|
181
|
+
const s = chunk.readInt16LE(i);
|
|
182
|
+
sum += s * s;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
const rms = Math.sqrt(sum / samples);
|
|
186
|
+
// Lower ceiling (2500) so normal speech hits 0.5-0.9 instead of 0.1-0.3
|
|
187
|
+
// Power curve (^0.6) boosts quiet sounds for more visible reactivity
|
|
188
|
+
audioLevel = Math.min(1, Math.pow(Math.min(rms / 2500, 1), 0.6));
|
|
189
|
+
// Faster attack (0.35 old), slower decay — snappy peaks, smooth falloff
|
|
190
|
+
audioLevelSmoothed =
|
|
191
|
+
audioLevel > audioLevelSmoothed
|
|
192
|
+
? audioLevelSmoothed * 0.35 + audioLevel * 0.65
|
|
193
|
+
: audioLevelSmoothed * 0.75 + audioLevel * 0.25;
|
|
194
|
+
// Shared state for other extensions (e.g. pi-pompom mouth animation).
|
|
195
|
+
// Using a namespaced globalThis object instead of pi.events because
|
|
196
|
+
// audio levels update at ~60Hz — event emission would be wasteful.
|
|
197
|
+
const shared = ((globalThis as any).__piListen ??= {});
|
|
198
|
+
shared.audioLevel = audioLevelSmoothed;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function voiceDebug(...args: unknown[]) {
|
|
202
|
+
if (!VOICE_DEBUG) return;
|
|
203
|
+
const ts = new Date().toISOString().split("T")[1];
|
|
204
|
+
const line = `[voice ${ts}] ${args.map((a) => (typeof a === "object" ? JSON.stringify(a) : String(a))).join(" ")}\n`;
|
|
205
|
+
try {
|
|
206
|
+
fs.appendFileSync(VOICE_LOG_FILE, line);
|
|
207
|
+
} catch {}
|
|
208
|
+
process.stderr.write(line);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// Cache command existence checks — avoid sync spawnSync on every recording start
|
|
212
|
+
const _cmdExistsCache = new Map<string, boolean>();
|
|
213
|
+
function commandExists(cmd: string): boolean {
|
|
214
|
+
const cached = _cmdExistsCache.get(cmd);
|
|
215
|
+
if (cached !== undefined) return cached;
|
|
216
|
+
const which = process.platform === "win32" ? "where" : "which";
|
|
217
|
+
const result = spawnSync(which, [cmd], { stdio: "pipe", timeout: 3000 }).status === 0;
|
|
218
|
+
_cmdExistsCache.set(cmd, result);
|
|
219
|
+
return result;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** Detect the first Windows DirectShow audio input device name via ffmpeg.
|
|
223
|
+
* DirectShow has no "default" alias — must enumerate and pick the first audio device.
|
|
224
|
+
*/
|
|
225
|
+
function detectWindowsAudioDevice(): string | null {
|
|
226
|
+
try {
|
|
227
|
+
const result = spawnSync("ffmpeg", ["-f", "dshow", "-list_devices", "true", "-i", "dummy"], {
|
|
228
|
+
timeout: 3000,
|
|
229
|
+
encoding: "utf-8",
|
|
230
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
231
|
+
});
|
|
232
|
+
// ffmpeg outputs device list to stderr
|
|
233
|
+
const output = result.stderr || "";
|
|
234
|
+
// Match lines like: [dshow @ ...] "Microphone (Realtek HD Audio)" (audio)
|
|
235
|
+
const match = output.match(/"([^"]+)"\s*\(audio\)/);
|
|
236
|
+
return match?.[1] || null;
|
|
237
|
+
} catch {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
interface AudioCaptureTool {
|
|
243
|
+
name: string;
|
|
244
|
+
cmd: string;
|
|
245
|
+
args: string[];
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Try available audio capture tools in order of preference
|
|
249
|
+
let _cachedAudioTool: AudioCaptureTool | null | undefined;
|
|
250
|
+
function detectAudioCaptureTool(): AudioCaptureTool | null {
|
|
251
|
+
if (_cachedAudioTool !== undefined) return _cachedAudioTool;
|
|
252
|
+
|
|
253
|
+
// 1. SoX rec — purpose-built for recording, best quality
|
|
254
|
+
if (commandExists("rec")) {
|
|
255
|
+
_cachedAudioTool = {
|
|
256
|
+
name: "sox",
|
|
257
|
+
cmd: "rec",
|
|
258
|
+
args: [
|
|
259
|
+
"-q",
|
|
260
|
+
"--buffer",
|
|
261
|
+
"4096",
|
|
262
|
+
"-c",
|
|
263
|
+
String(CHANNELS),
|
|
264
|
+
"-b",
|
|
265
|
+
"16",
|
|
266
|
+
"-e",
|
|
267
|
+
"signed-integer",
|
|
268
|
+
"-t",
|
|
269
|
+
"raw",
|
|
270
|
+
"-",
|
|
271
|
+
"rate",
|
|
272
|
+
String(SAMPLE_RATE),
|
|
273
|
+
],
|
|
274
|
+
};
|
|
275
|
+
return _cachedAudioTool;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// 2. ffmpeg — widely installed, captures from default mic
|
|
279
|
+
if (commandExists("ffmpeg")) {
|
|
280
|
+
const isLinux = process.platform === "linux";
|
|
281
|
+
const isMac = process.platform === "darwin";
|
|
282
|
+
const isWin = process.platform === "win32";
|
|
283
|
+
// Input device varies by platform
|
|
284
|
+
let inputArgs: string[];
|
|
285
|
+
if (isMac) {
|
|
286
|
+
inputArgs = ["-f", "avfoundation", "-i", ":default"];
|
|
287
|
+
} else if (isLinux) {
|
|
288
|
+
inputArgs = ["-f", "pulse", "-i", "default"];
|
|
289
|
+
} else if (isWin) {
|
|
290
|
+
// DirectShow has no "default" alias — enumerate devices and pick the first audio device
|
|
291
|
+
const dshowDevice = detectWindowsAudioDevice();
|
|
292
|
+
inputArgs = dshowDevice
|
|
293
|
+
? ["-f", "dshow", "-i", `audio=${dshowDevice}`]
|
|
294
|
+
: ["-f", "dshow", "-i", "audio=Microphone"]; // last-resort guess
|
|
295
|
+
} else {
|
|
296
|
+
inputArgs = ["-f", "pulse", "-i", "default"]; // fallback for other platforms
|
|
297
|
+
}
|
|
298
|
+
_cachedAudioTool = {
|
|
299
|
+
name: "ffmpeg",
|
|
300
|
+
cmd: "ffmpeg",
|
|
301
|
+
args: [
|
|
302
|
+
...inputArgs,
|
|
303
|
+
"-ac",
|
|
304
|
+
String(CHANNELS),
|
|
305
|
+
"-ar",
|
|
306
|
+
String(SAMPLE_RATE),
|
|
307
|
+
"-sample_fmt",
|
|
308
|
+
"s16",
|
|
309
|
+
"-f",
|
|
310
|
+
"s16le",
|
|
311
|
+
"-loglevel",
|
|
312
|
+
"error",
|
|
313
|
+
"pipe:1",
|
|
314
|
+
],
|
|
315
|
+
};
|
|
316
|
+
return _cachedAudioTool;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// 3. arecord — built into Linux ALSA, zero install
|
|
320
|
+
if (process.platform === "linux" && commandExists("arecord")) {
|
|
321
|
+
_cachedAudioTool = {
|
|
322
|
+
name: "arecord",
|
|
323
|
+
cmd: "arecord",
|
|
324
|
+
args: ["-q", "-f", "S16_LE", "-r", String(SAMPLE_RATE), "-c", String(CHANNELS), "-t", "raw"],
|
|
325
|
+
};
|
|
326
|
+
return _cachedAudioTool;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
_cachedAudioTool = null;
|
|
330
|
+
return null;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// ─── Deepgram WebSocket Streaming ────────────────────────────────────────────
|
|
334
|
+
|
|
335
|
+
interface StreamingSession {
|
|
336
|
+
backend: "deepgram";
|
|
337
|
+
ws: WebSocket;
|
|
338
|
+
recProcess: ChildProcess;
|
|
339
|
+
interimText: string;
|
|
340
|
+
finalizedParts: string[];
|
|
341
|
+
keepAliveTimer: ReturnType<typeof setInterval> | null;
|
|
342
|
+
staleSessionTimer: ReturnType<typeof setTimeout> | null;
|
|
343
|
+
finalizeTimer: ReturnType<typeof setTimeout> | null;
|
|
344
|
+
closed: boolean;
|
|
345
|
+
stopRequested: boolean;
|
|
346
|
+
hadAudioData: boolean; // Track if we received any audio data
|
|
347
|
+
hadSpeech: boolean; // Track if Deepgram detected any speech
|
|
348
|
+
receivedMessage: boolean; // Track if we got ANY message from Deepgram
|
|
349
|
+
onTranscript: (interim: string, finals: string[]) => void;
|
|
350
|
+
onDone: (fullText: string, meta: { hadAudio: boolean; hadSpeech: boolean }) => void;
|
|
351
|
+
onError: (err: string) => void;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Union of session types — Deepgram streaming or local batch */
|
|
355
|
+
type VoiceSession = StreamingSession | LocalSession;
|
|
356
|
+
|
|
357
|
+
function startStreamingSession(
|
|
358
|
+
config: VoiceConfig,
|
|
359
|
+
callbacks: {
|
|
360
|
+
onTranscript: (interim: string, finals: string[]) => void;
|
|
361
|
+
onDone: (fullText: string, meta: { hadAudio: boolean; hadSpeech: boolean }) => void;
|
|
362
|
+
onError: (err: string) => void;
|
|
363
|
+
}
|
|
364
|
+
): StreamingSession | null {
|
|
365
|
+
const apiKey = resolveDeepgramApiKey(config);
|
|
366
|
+
voiceDebug("startStreamingSession", { hasApiKey: !!apiKey });
|
|
367
|
+
if (!apiKey) {
|
|
368
|
+
voiceDebug("startStreamingSession → no API key, calling onError");
|
|
369
|
+
callbacks.onError("DEEPGRAM_API_KEY not set");
|
|
370
|
+
return null;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// ── Audio capture: try rec (SoX) → ffmpeg → arecord (Linux ALSA) ──
|
|
374
|
+
const audioTool = detectAudioCaptureTool();
|
|
375
|
+
if (!audioTool) {
|
|
376
|
+
voiceDebug("startStreamingSession → no audio capture tool found");
|
|
377
|
+
callbacks.onError("No audio capture tool found. Install one of: sox, ffmpeg, or arecord (Linux)");
|
|
378
|
+
return null;
|
|
379
|
+
}
|
|
380
|
+
voiceDebug("Using audio capture tool:", audioTool.name);
|
|
381
|
+
|
|
382
|
+
const recProc = spawn(audioTool.cmd, audioTool.args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
383
|
+
|
|
384
|
+
recProc.stderr?.on("data", (d: Buffer) => {
|
|
385
|
+
const msg = d.toString().trim();
|
|
386
|
+
// Suppress noisy but harmless messages
|
|
387
|
+
if (msg.includes("buffer overrun") || msg.includes("Discarding") || msg.includes("Last message repeated")) return;
|
|
388
|
+
voiceDebug(`${audioTool.name} stderr:`, msg);
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
const wsUrl = buildDeepgramWsUrl(config);
|
|
392
|
+
const ws = new WebSocket(wsUrl, {
|
|
393
|
+
headers: {
|
|
394
|
+
Authorization: `Token ${apiKey}`,
|
|
395
|
+
},
|
|
396
|
+
} as any);
|
|
397
|
+
|
|
398
|
+
// Connection timeout — abort if Deepgram doesn't respond within 10s
|
|
399
|
+
const wsConnectTimeout = setTimeout(() => {
|
|
400
|
+
if (ws.readyState !== WebSocket.OPEN) {
|
|
401
|
+
voiceDebug("WebSocket connection timeout (10s)");
|
|
402
|
+
try {
|
|
403
|
+
ws.close();
|
|
404
|
+
} catch {}
|
|
405
|
+
try {
|
|
406
|
+
recProc.kill("SIGTERM");
|
|
407
|
+
} catch {}
|
|
408
|
+
callbacks.onError("Deepgram connection timed out (10s). Check your network.");
|
|
409
|
+
}
|
|
410
|
+
}, 10_000);
|
|
411
|
+
|
|
412
|
+
const session: StreamingSession = {
|
|
413
|
+
backend: "deepgram",
|
|
414
|
+
ws,
|
|
415
|
+
recProcess: recProc,
|
|
416
|
+
interimText: "",
|
|
417
|
+
finalizedParts: [],
|
|
418
|
+
keepAliveTimer: null,
|
|
419
|
+
staleSessionTimer: null,
|
|
420
|
+
finalizeTimer: null,
|
|
421
|
+
closed: false,
|
|
422
|
+
stopRequested: false,
|
|
423
|
+
hadAudioData: false,
|
|
424
|
+
hadSpeech: false,
|
|
425
|
+
receivedMessage: false,
|
|
426
|
+
onTranscript: callbacks.onTranscript,
|
|
427
|
+
onDone: callbacks.onDone,
|
|
428
|
+
onError: callbacks.onError,
|
|
429
|
+
};
|
|
430
|
+
|
|
431
|
+
// Handle HTTP error responses before WebSocket upgrade (e.g., 400 Bad Request, 401 Unauthorized)
|
|
432
|
+
// Only available with Node.js `ws` package — skip if using browser-style WebSocket
|
|
433
|
+
if (typeof (ws as any).on === "function") {
|
|
434
|
+
(ws as any).on("unexpected-response", (_req: any, res: any) => {
|
|
435
|
+
let body = "";
|
|
436
|
+
res.on("data", (d: Buffer) => {
|
|
437
|
+
body += d.toString();
|
|
438
|
+
});
|
|
439
|
+
res.on("end", () => {
|
|
440
|
+
voiceDebug("WebSocket unexpected-response", { status: res.statusCode, body });
|
|
441
|
+
if (!session.closed) {
|
|
442
|
+
failStreamingSession(session, `Deepgram HTTP ${res.statusCode}: ${body.slice(0, 200)}`);
|
|
443
|
+
}
|
|
444
|
+
});
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
ws.onopen = () => {
|
|
449
|
+
clearTimeout(wsConnectTimeout);
|
|
450
|
+
voiceDebug("WebSocket onopen → streaming audio");
|
|
451
|
+
try {
|
|
452
|
+
ws.send(JSON.stringify({ type: "KeepAlive" }));
|
|
453
|
+
} catch {}
|
|
454
|
+
|
|
455
|
+
session.keepAliveTimer = setInterval(() => {
|
|
456
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
457
|
+
try {
|
|
458
|
+
ws.send(JSON.stringify({ type: "KeepAlive" }));
|
|
459
|
+
} catch {}
|
|
460
|
+
}
|
|
461
|
+
}, KEEPALIVE_INTERVAL_MS);
|
|
462
|
+
|
|
463
|
+
recProc.stdout?.on("data", (chunk: Buffer) => {
|
|
464
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
465
|
+
session.hadAudioData = true;
|
|
466
|
+
try {
|
|
467
|
+
ws.send(new Uint8Array(chunk));
|
|
468
|
+
} catch {}
|
|
469
|
+
// Feed audio data to level meter for reactive waveform
|
|
470
|
+
updateAudioLevel(chunk);
|
|
471
|
+
// Start stale-session watchdog on first audio chunk
|
|
472
|
+
if (!session.staleSessionTimer && !session.receivedMessage) {
|
|
473
|
+
session.staleSessionTimer = setTimeout(() => {
|
|
474
|
+
if (!session.closed && !session.receivedMessage) {
|
|
475
|
+
voiceDebug("Stale session: no Deepgram response after 15s of audio");
|
|
476
|
+
failStreamingSession(session, "No response from Deepgram (15s). Check your API key and network.");
|
|
477
|
+
}
|
|
478
|
+
}, 15_000);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
});
|
|
482
|
+
};
|
|
483
|
+
|
|
484
|
+
ws.onmessage = (event: MessageEvent) => {
|
|
485
|
+
try {
|
|
486
|
+
const msg = typeof event.data === "string" ? JSON.parse(event.data) : null;
|
|
487
|
+
if (!msg) return;
|
|
488
|
+
|
|
489
|
+
// Cancel stale-session watchdog on first response
|
|
490
|
+
if (!session.receivedMessage) {
|
|
491
|
+
session.receivedMessage = true;
|
|
492
|
+
if (session.staleSessionTimer) {
|
|
493
|
+
clearTimeout(session.staleSessionTimer);
|
|
494
|
+
session.staleSessionTimer = null;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
if (msg.type === "Results") {
|
|
499
|
+
const alt = msg.channel?.alternatives?.[0];
|
|
500
|
+
const transcript = alt?.transcript || "";
|
|
501
|
+
|
|
502
|
+
if (transcript.trim()) {
|
|
503
|
+
session.hadSpeech = true;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
if (msg.is_final) {
|
|
507
|
+
if (transcript.trim()) {
|
|
508
|
+
session.finalizedParts.push(transcript.trim());
|
|
509
|
+
}
|
|
510
|
+
session.interimText = "";
|
|
511
|
+
} else {
|
|
512
|
+
session.interimText = transcript;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
session.onTranscript(session.interimText, session.finalizedParts);
|
|
516
|
+
} else if (msg.type === "Error" || msg.type === "error") {
|
|
517
|
+
failStreamingSession(session, msg.message || msg.description || "Deepgram error");
|
|
518
|
+
}
|
|
519
|
+
} catch (err) {
|
|
520
|
+
voiceDebug("onmessage parse error", { error: String(err) });
|
|
521
|
+
}
|
|
522
|
+
};
|
|
523
|
+
|
|
524
|
+
ws.onerror = (ev) => {
|
|
525
|
+
clearTimeout(wsConnectTimeout);
|
|
526
|
+
const errMsg = (ev as any)?.message || (ev as any)?.error?.message || "unknown";
|
|
527
|
+
voiceDebug("WebSocket onerror", { readyState: ws.readyState, error: errMsg });
|
|
528
|
+
if (!session.closed) {
|
|
529
|
+
failStreamingSession(session, `WebSocket error: ${errMsg}`);
|
|
530
|
+
}
|
|
531
|
+
};
|
|
532
|
+
|
|
533
|
+
ws.onclose = (ev) => {
|
|
534
|
+
clearTimeout(wsConnectTimeout);
|
|
535
|
+
const code = (ev as any)?.code;
|
|
536
|
+
const reason = (ev as any)?.reason;
|
|
537
|
+
voiceDebug("WebSocket onclose", { code, reason, closed: session.closed });
|
|
538
|
+
if (!session.closed) {
|
|
539
|
+
// Unexpected close — distinguish normal completion from network drops
|
|
540
|
+
if (session.stopRequested || code === 1000 || code === 1001 || session.finalizedParts.length > 0) {
|
|
541
|
+
if (session.interimText.trim()) {
|
|
542
|
+
session.finalizedParts.push(session.interimText.trim());
|
|
543
|
+
session.interimText = "";
|
|
544
|
+
}
|
|
545
|
+
// Normal close or we have usable transcript data — finalize
|
|
546
|
+
finalizeSession(session);
|
|
547
|
+
} else {
|
|
548
|
+
// Abnormal close with no transcript — treat as error
|
|
549
|
+
failStreamingSession(session, `Connection lost (code ${code ?? "unknown"}${reason ? `: ${reason}` : ""})`);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
};
|
|
553
|
+
|
|
554
|
+
recProc.on("error", (err) => {
|
|
555
|
+
voiceDebug("SoX process error:", err.message);
|
|
556
|
+
if (!session.closed) {
|
|
557
|
+
failStreamingSession(session, `SoX error: ${err.message}`);
|
|
558
|
+
}
|
|
559
|
+
});
|
|
560
|
+
|
|
561
|
+
recProc.on("close", (code, signal) => {
|
|
562
|
+
voiceDebug("SoX process closed", { code, signal, closed: session.closed, wsState: ws.readyState });
|
|
563
|
+
// Only send CloseStream if the session isn't already being torn down
|
|
564
|
+
// (stopStreamingSession sends its own CloseStream before killing SoX)
|
|
565
|
+
if (!session.closed && !session.stopRequested && ws.readyState === WebSocket.OPEN) {
|
|
566
|
+
try {
|
|
567
|
+
ws.send(JSON.stringify({ type: "CloseStream" }));
|
|
568
|
+
} catch {}
|
|
569
|
+
}
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
return session;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
function stopStreamingSession(session: StreamingSession): void {
|
|
576
|
+
if (session.closed) return;
|
|
577
|
+
session.stopRequested = true;
|
|
578
|
+
|
|
579
|
+
try {
|
|
580
|
+
session.recProcess.kill("SIGTERM");
|
|
581
|
+
} catch {}
|
|
582
|
+
|
|
583
|
+
if (session.ws.readyState === WebSocket.OPEN) {
|
|
584
|
+
try {
|
|
585
|
+
session.ws.send(JSON.stringify({ type: "CloseStream" }));
|
|
586
|
+
} catch {}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
if (!session.finalizeTimer) {
|
|
590
|
+
session.finalizeTimer = setTimeout(() => {
|
|
591
|
+
session.finalizeTimer = null;
|
|
592
|
+
if (session.closed) return;
|
|
593
|
+
if (session.interimText.trim()) {
|
|
594
|
+
session.finalizedParts.push(session.interimText.trim());
|
|
595
|
+
session.interimText = "";
|
|
596
|
+
}
|
|
597
|
+
finalizeSession(session);
|
|
598
|
+
}, STREAM_FINALIZE_TIMEOUT_MS);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
function finalizeSession(session: StreamingSession): void {
|
|
603
|
+
if (session.closed) return;
|
|
604
|
+
session.closed = true;
|
|
605
|
+
voiceDebug("finalizeSession", {
|
|
606
|
+
hadAudio: session.hadAudioData,
|
|
607
|
+
hadSpeech: session.hadSpeech,
|
|
608
|
+
parts: session.finalizedParts.length,
|
|
609
|
+
});
|
|
610
|
+
|
|
611
|
+
if (session.staleSessionTimer) {
|
|
612
|
+
clearTimeout(session.staleSessionTimer);
|
|
613
|
+
session.staleSessionTimer = null;
|
|
614
|
+
}
|
|
615
|
+
if (session.finalizeTimer) {
|
|
616
|
+
clearTimeout(session.finalizeTimer);
|
|
617
|
+
session.finalizeTimer = null;
|
|
618
|
+
}
|
|
619
|
+
if (session.keepAliveTimer) {
|
|
620
|
+
clearInterval(session.keepAliveTimer);
|
|
621
|
+
session.keepAliveTimer = null;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
try {
|
|
625
|
+
session.ws.close();
|
|
626
|
+
} catch {}
|
|
627
|
+
try {
|
|
628
|
+
session.recProcess.kill("SIGKILL");
|
|
629
|
+
} catch {}
|
|
630
|
+
|
|
631
|
+
const fullText = session.finalizedParts.join(" ").trim();
|
|
632
|
+
session.onDone(fullText, {
|
|
633
|
+
hadAudio: session.hadAudioData,
|
|
634
|
+
hadSpeech: session.hadSpeech,
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
function failStreamingSession(session: StreamingSession, err: string): void {
|
|
639
|
+
if (session.closed) return;
|
|
640
|
+
session.closed = true;
|
|
641
|
+
session.stopRequested = true;
|
|
642
|
+
|
|
643
|
+
if (session.staleSessionTimer) {
|
|
644
|
+
clearTimeout(session.staleSessionTimer);
|
|
645
|
+
session.staleSessionTimer = null;
|
|
646
|
+
}
|
|
647
|
+
if (session.finalizeTimer) {
|
|
648
|
+
clearTimeout(session.finalizeTimer);
|
|
649
|
+
session.finalizeTimer = null;
|
|
650
|
+
}
|
|
651
|
+
if (session.keepAliveTimer) {
|
|
652
|
+
clearInterval(session.keepAliveTimer);
|
|
653
|
+
session.keepAliveTimer = null;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
try {
|
|
657
|
+
session.ws.close();
|
|
658
|
+
} catch {}
|
|
659
|
+
try {
|
|
660
|
+
session.recProcess.kill("SIGKILL");
|
|
661
|
+
} catch {}
|
|
662
|
+
session.onError(err);
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
// ─── Abort helper — nuke everything synchronously ────────────────────────────
|
|
666
|
+
|
|
667
|
+
function abortSession(session: VoiceSession | null): void {
|
|
668
|
+
if (!session || session.closed) return;
|
|
669
|
+
// Replace callbacks with no-ops BEFORE the backend-specific abort path so any
|
|
670
|
+
// in-flight async work (sherpa transcription, late ws messages, recProcess close)
|
|
671
|
+
// that resolves after abort cannot reach the recording state machine — including
|
|
672
|
+
// across session replacement, where the surviving callbacks would otherwise
|
|
673
|
+
// write into the new session's editor / fire notifications on the new ctx.
|
|
674
|
+
session.onTranscript = () => {};
|
|
675
|
+
session.onDone = () => {};
|
|
676
|
+
session.onError = () => {};
|
|
677
|
+
if (session.backend === "local") {
|
|
678
|
+
abortLocalSession(session);
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
session.closed = true;
|
|
682
|
+
if (session.staleSessionTimer) {
|
|
683
|
+
clearTimeout(session.staleSessionTimer);
|
|
684
|
+
session.staleSessionTimer = null;
|
|
685
|
+
}
|
|
686
|
+
if (session.keepAliveTimer) {
|
|
687
|
+
clearInterval(session.keepAliveTimer);
|
|
688
|
+
session.keepAliveTimer = null;
|
|
689
|
+
}
|
|
690
|
+
try {
|
|
691
|
+
session.ws.close();
|
|
692
|
+
} catch {}
|
|
693
|
+
try {
|
|
694
|
+
session.recProcess.kill("SIGKILL");
|
|
695
|
+
} catch {}
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
// ─── Extension ───────────────────────────────────────────────────────────────
|
|
699
|
+
|
|
700
|
+
export default function (pi: ExtensionAPI) {
|
|
701
|
+
let config = DEFAULT_CONFIG;
|
|
702
|
+
let configSource: VoiceSettingsScope | "default" = "default";
|
|
703
|
+
let currentCwd = process.cwd();
|
|
704
|
+
let voiceState: VoiceState = "idle";
|
|
705
|
+
let ctx: ExtensionContext | null = null;
|
|
706
|
+
let recordingStart = 0;
|
|
707
|
+
let statusTimer: ReturnType<typeof setInterval> | null = null;
|
|
708
|
+
let terminalInputUnsub: (() => void) | null = null;
|
|
709
|
+
|
|
710
|
+
// ─── v7.1 Settings UI: widget registry + render ticker (per session) ──
|
|
711
|
+
// Created lazily on first use (session_start) and torn down in
|
|
712
|
+
// voiceCleanup. Owns all DisposableWidget instances for the session.
|
|
713
|
+
let widgetRegistry: WidgetRegistry | null = null;
|
|
714
|
+
let renderTicker: RenderTicker | null = null;
|
|
715
|
+
function getOrInitVoiceUi(): { registry: WidgetRegistry; ticker: RenderTicker } {
|
|
716
|
+
if (!widgetRegistry) widgetRegistry = makeWidgetRegistry();
|
|
717
|
+
if (!renderTicker) renderTicker = makeRenderTicker();
|
|
718
|
+
return { registry: widgetRegistry, ticker: renderTicker };
|
|
719
|
+
}
|
|
720
|
+
/** Currently-mounted install widgets keyed by modelId — for [esc] routing. */
|
|
721
|
+
const activeInstallWidgets = new Map<string, TtsInstallProgressWidget>();
|
|
722
|
+
/** Currently-mounted playback indicator (one at a time) for [esc] routing. */
|
|
723
|
+
let activePlaybackIndicator: TtsPlaybackIndicator | null = null;
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* v7.1.3 — read the active hold-threshold (ms) from config with bounds.
|
|
727
|
+
* Outside [200, 3000] falls back to the default (700ms). Bounds chosen
|
|
728
|
+
* so a typo can't make the experience completely broken: 200ms is
|
|
729
|
+
* effectively single-tap, 3000ms is "RSI-friendly slow press."
|
|
730
|
+
*/
|
|
731
|
+
function getHoldThresholdMs(): number {
|
|
732
|
+
const v = (config as any).holdThresholdMs;
|
|
733
|
+
if (typeof v === "number" && Number.isFinite(v) && v >= 200 && v <= 3000) return v;
|
|
734
|
+
return HOLD_THRESHOLD_DEFAULT_MS;
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
// ─── Toggle Shortcut (resolved once at startup, used everywhere) ──────
|
|
738
|
+
const resolvedToggleShortcut = loadGlobalToggleShortcut();
|
|
739
|
+
const toggleShortcutLabel = resolvedToggleShortcut
|
|
740
|
+
.split("+")
|
|
741
|
+
.map((p) => (p.length <= 1 ? p.toUpperCase() : p[0]!.toUpperCase() + p.slice(1)))
|
|
742
|
+
.join("+");
|
|
743
|
+
|
|
744
|
+
// Streaming session state
|
|
745
|
+
let activeSession: VoiceSession | null = null;
|
|
746
|
+
let preRecordingSession: StreamingSession | null = null; // Started during warmup, promoted on confirm (Deepgram only)
|
|
747
|
+
|
|
748
|
+
let lastStopTime = 0; // For Escape-to-clear-editor within 30s of recording
|
|
749
|
+
let lastEscapeTime = 0; // For double-escape to clear editor
|
|
750
|
+
let recordingStartedAt = 0; // When recording actually started (for grace period)
|
|
751
|
+
let editorTextBeforeVoice = ""; // Snapshot of editor text before recording started
|
|
752
|
+
|
|
753
|
+
// Hold-to-talk state
|
|
754
|
+
let kittyReleaseDetected = false;
|
|
755
|
+
let spaceDownTime: number | null = null;
|
|
756
|
+
let holdActivationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
757
|
+
let spaceConsumed = false; // True once threshold passed and recording started
|
|
758
|
+
let releaseGapTimer: GapTimer | null = null; // gap-based release detection (see release-controller.ts)
|
|
759
|
+
let warmupWidgetTimer: ReturnType<typeof setInterval> | null = null;
|
|
760
|
+
let spacePressCount = 0; // Count of rapid space presses (for non-Kitty hold detection)
|
|
761
|
+
let lastSpacePressTime = 0; // Timestamp of last space press event
|
|
762
|
+
let holdConfirmed = false; // True once we've confirmed user is holding (not tapping)
|
|
763
|
+
let errorCooldownUntil = 0; // After an error, block re-activation until this timestamp
|
|
764
|
+
let lastNonSpaceKeyTime = 0; // Timestamp of last non-space keypress (typing cooldown)
|
|
765
|
+
let tailRecordingTimer: ReturnType<typeof setTimeout> | null = null; // Delayed stop after release
|
|
766
|
+
|
|
767
|
+
// ─── Recording History ───────────────────────────────────────────────────
|
|
768
|
+
|
|
769
|
+
interface RecordingHistoryEntry {
|
|
770
|
+
text: string;
|
|
771
|
+
timestamp: number;
|
|
772
|
+
duration: number;
|
|
773
|
+
mode: "hold" | "toggle" | "dictate";
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
const recordingHistory: RecordingHistoryEntry[] = [];
|
|
777
|
+
const MAX_HISTORY = 50;
|
|
778
|
+
|
|
779
|
+
function addToHistory(text: string, duration: number, mode: "hold" | "toggle" | "dictate" = "hold") {
|
|
780
|
+
recordingHistory.unshift({ text, timestamp: Date.now(), duration, mode });
|
|
781
|
+
if (recordingHistory.length > MAX_HISTORY) recordingHistory.pop();
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
// ─── Continuous Dictation Mode ───────────────────────────────────────────
|
|
785
|
+
|
|
786
|
+
let dictationMode = false;
|
|
787
|
+
|
|
788
|
+
// ─── Sound Feedback ──────────────────────────────────────────────────────
|
|
789
|
+
|
|
790
|
+
// Pre-resolve sound paths once at load time (not per-play)
|
|
791
|
+
const _soundPaths: Record<string, string | null> = {};
|
|
792
|
+
for (const [type, file] of Object.entries({
|
|
793
|
+
start: "/System/Library/Sounds/Tink.aiff",
|
|
794
|
+
stop: "/System/Library/Sounds/Pop.aiff",
|
|
795
|
+
error: "/System/Library/Sounds/Basso.aiff",
|
|
796
|
+
})) {
|
|
797
|
+
_soundPaths[type] = fs.existsSync(file) ? file : null;
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
function playSound(type: "start" | "stop" | "error") {
|
|
801
|
+
const file = _soundPaths[type];
|
|
802
|
+
if (!file) return;
|
|
803
|
+
try {
|
|
804
|
+
const proc = spawn("afplay", [file], { stdio: "ignore", detached: true });
|
|
805
|
+
proc.unref();
|
|
806
|
+
proc.on("error", () => {}); // Prevent unhandled error crash
|
|
807
|
+
} catch {}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
// ─── Voice UI ────────────────────────────────────────────────────────────
|
|
811
|
+
|
|
812
|
+
function updateVoiceStatus() {
|
|
813
|
+
if (!ctx?.hasUI) return;
|
|
814
|
+
switch (voiceState) {
|
|
815
|
+
case "idle": {
|
|
816
|
+
if (!config.enabled) {
|
|
817
|
+
ctx.ui.setStatus("voice", undefined);
|
|
818
|
+
break;
|
|
819
|
+
}
|
|
820
|
+
const modeTag = !config.onboarding.completed ? "SETUP" : config.backend === "local" ? "LOCAL" : "STREAM";
|
|
821
|
+
ctx.ui.setStatus("voice", `MIC ${modeTag}`);
|
|
822
|
+
break;
|
|
823
|
+
}
|
|
824
|
+
case "warmup":
|
|
825
|
+
ctx.ui.setStatus("voice", "MIC HOLD...");
|
|
826
|
+
break;
|
|
827
|
+
case "recording": {
|
|
828
|
+
const secs = Math.round((Date.now() - recordingStart) / 1000);
|
|
829
|
+
// Live audio level meter in status bar
|
|
830
|
+
const meterLen = 4;
|
|
831
|
+
const meterFilled = Math.round(audioLevelSmoothed * meterLen);
|
|
832
|
+
const meter = "█".repeat(meterFilled) + "░".repeat(meterLen - meterFilled);
|
|
833
|
+
ctx.ui.setStatus("voice", `REC ${secs}s ${meter}`);
|
|
834
|
+
break;
|
|
835
|
+
}
|
|
836
|
+
case "finalizing":
|
|
837
|
+
if (config.backend === "local") {
|
|
838
|
+
ctx.ui.setStatus("voice", "STT...");
|
|
839
|
+
} else {
|
|
840
|
+
// Don't show "STT..." — live transcript handles it
|
|
841
|
+
ctx.ui.setStatus("voice", "");
|
|
842
|
+
}
|
|
843
|
+
break;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
function setVoiceState(newState: VoiceState) {
|
|
848
|
+
const prev = voiceState;
|
|
849
|
+
voiceState = newState;
|
|
850
|
+
if (prev !== newState) {
|
|
851
|
+
voiceDebug(`STATE: ${prev} → ${newState}`);
|
|
852
|
+
}
|
|
853
|
+
const shared = ((globalThis as any).__piListen ??= {});
|
|
854
|
+
shared.recording = newState === "recording";
|
|
855
|
+
updateVoiceStatus();
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
// ─── Cleanup helpers ─────────────────────────────────────────────────────
|
|
859
|
+
|
|
860
|
+
function clearHoldTimer() {
|
|
861
|
+
if (holdActivationTimer) {
|
|
862
|
+
clearTimeout(holdActivationTimer);
|
|
863
|
+
holdActivationTimer = null;
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
function clearReleaseTimer() {
|
|
868
|
+
releaseGapTimer?.cancel();
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
function clearWarmupWidget() {
|
|
872
|
+
if (warmupWidgetTimer) {
|
|
873
|
+
clearInterval(warmupWidgetTimer);
|
|
874
|
+
warmupWidgetTimer = null;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
function clearRecordingAnimTimer() {
|
|
879
|
+
if (_recWidgetAnimTimer) {
|
|
880
|
+
clearInterval(_recWidgetAnimTimer);
|
|
881
|
+
_recWidgetAnimTimer = null;
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
function hideWidget() {
|
|
886
|
+
if (ctx?.hasUI) ctx.ui.setWidget("voice-recording", undefined);
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/** Reset all hold-to-talk state to idle. Call after any recording stop/error/cancel. */
|
|
890
|
+
function resetHoldState(opts?: { cooldown?: number }) {
|
|
891
|
+
spaceConsumed = false;
|
|
892
|
+
spaceDownTime = null;
|
|
893
|
+
spacePressCount = 0;
|
|
894
|
+
holdConfirmed = false;
|
|
895
|
+
clearHoldTimer();
|
|
896
|
+
clearReleaseTimer();
|
|
897
|
+
abortPreRecording();
|
|
898
|
+
if (opts?.cooldown) errorCooldownUntil = Date.now() + opts.cooldown;
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
function voiceCleanup() {
|
|
902
|
+
// v7.1: cancel in-flight installs FIRST so their AbortControllers
|
|
903
|
+
// fire before we drop UI state. Without this, a session_shutdown
|
|
904
|
+
// during a download would leave the network/disk work running
|
|
905
|
+
// after the widget slot is cleared (Codex v6 finding #3).
|
|
906
|
+
for (const w of Array.from(activeInstallWidgets.values())) {
|
|
907
|
+
try {
|
|
908
|
+
w.cancel();
|
|
909
|
+
} catch (err) {
|
|
910
|
+
voiceDebug("install widget cancel threw during voiceCleanup", String(err));
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
activeInstallWidgets.clear();
|
|
914
|
+
// Stop any active playback the same way.
|
|
915
|
+
try {
|
|
916
|
+
activePlaybackIndicator?.stop();
|
|
917
|
+
} catch (err) {
|
|
918
|
+
voiceDebug("playback stop threw during voiceCleanup", String(err));
|
|
919
|
+
}
|
|
920
|
+
activePlaybackIndicator = null;
|
|
921
|
+
// Drain registry — each widget self-clears its slot via dispose().
|
|
922
|
+
try {
|
|
923
|
+
widgetRegistry?.disposeAll();
|
|
924
|
+
} catch (err) {
|
|
925
|
+
voiceDebug("widgetRegistry.disposeAll threw", String(err));
|
|
926
|
+
}
|
|
927
|
+
try {
|
|
928
|
+
renderTicker?.dispose();
|
|
929
|
+
} catch (err) {
|
|
930
|
+
voiceDebug("renderTicker.dispose threw", String(err));
|
|
931
|
+
}
|
|
932
|
+
widgetRegistry = null;
|
|
933
|
+
renderTicker = null;
|
|
934
|
+
|
|
935
|
+
if (statusTimer) {
|
|
936
|
+
clearInterval(statusTimer);
|
|
937
|
+
statusTimer = null;
|
|
938
|
+
}
|
|
939
|
+
cancelDelayedStop();
|
|
940
|
+
clearWarmupWidget();
|
|
941
|
+
clearRecordingAnimTimer();
|
|
942
|
+
// Reset audio levels
|
|
943
|
+
audioLevel = 0;
|
|
944
|
+
audioLevelSmoothed = 0;
|
|
945
|
+
if (activeSession) {
|
|
946
|
+
abortSession(activeSession);
|
|
947
|
+
activeSession = null;
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
resetHoldState(); // includes clearHoldTimer + clearReleaseTimer
|
|
951
|
+
_startingRecording = false;
|
|
952
|
+
lastSpacePressTime = 0;
|
|
953
|
+
lastNonSpaceKeyTime = 0;
|
|
954
|
+
errorCooldownUntil = 0;
|
|
955
|
+
editorTextBeforeVoice = "";
|
|
956
|
+
dictationMode = false;
|
|
957
|
+
recordingStart = 0;
|
|
958
|
+
recordingStartedAt = 0;
|
|
959
|
+
lastStopTime = 0;
|
|
960
|
+
if (terminalInputUnsub) {
|
|
961
|
+
terminalInputUnsub();
|
|
962
|
+
terminalInputUnsub = null;
|
|
963
|
+
}
|
|
964
|
+
hideWidget();
|
|
965
|
+
setVoiceState("idle");
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
async function finalizeAndSaveSetup(
|
|
969
|
+
uiCtx: ExtensionContext | ExtensionCommandContext,
|
|
970
|
+
nextConfig: VoiceConfig,
|
|
971
|
+
selectedScope: VoiceSettingsScope,
|
|
972
|
+
summaryLines: string[],
|
|
973
|
+
source: "first-run" | "setup-command"
|
|
974
|
+
) {
|
|
975
|
+
const isLocal = nextConfig.backend === "local";
|
|
976
|
+
const hasKey = !!resolveDeepgramApiKey(nextConfig);
|
|
977
|
+
// Local backend is always valid (sherpa handles everything). Deepgram needs API key.
|
|
978
|
+
const validated = isLocal || hasKey;
|
|
979
|
+
config = finalizeOnboardingConfig(nextConfig, { validated, source });
|
|
980
|
+
configSource = selectedScope;
|
|
981
|
+
const savedPath = saveConfig(config, selectedScope, currentCwd);
|
|
982
|
+
const statusHeader = validated
|
|
983
|
+
? "Voice setup complete."
|
|
984
|
+
: "Voice setup saved, but DEEPGRAM_API_KEY is still required.";
|
|
985
|
+
uiCtx.ui.notify(
|
|
986
|
+
[statusHeader, ...summaryLines, "", `Saved to ${savedPath}`].join("\n"),
|
|
987
|
+
validated ? "info" : "warning"
|
|
988
|
+
);
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
// ─── Warmup Widget ──────────────────────────────────────────────────────
|
|
992
|
+
// ─── Minimal Voice Indicators ──────────────────────────────────────
|
|
993
|
+
|
|
994
|
+
function getRecordDot(): string {
|
|
995
|
+
// v7.2 — soft pulse between full and dim. Two-state instead of
|
|
996
|
+
// three-state (cleaner, more like a breathing LED indicator
|
|
997
|
+
// than the v7.0 multi-glyph approach).
|
|
998
|
+
const phase = (Math.sin(Date.now() / 700) + 1) / 2;
|
|
999
|
+
return phase > 0.5 ? "●" : "○";
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
function buildMiniWave(level: number): string {
|
|
1003
|
+
// Legacy block-bar wave — retained for compatibility with any
|
|
1004
|
+
// caller that still uses it. New code should use buildAuroraWave.
|
|
1005
|
+
const bars = "▁▂▃▄▅▆▇█";
|
|
1006
|
+
const len = 12;
|
|
1007
|
+
let out = "";
|
|
1008
|
+
const t = Date.now() / 1000;
|
|
1009
|
+
const energy = Math.pow(level, 0.7);
|
|
1010
|
+
for (let i = 0; i < len; i++) {
|
|
1011
|
+
const pos = i / len;
|
|
1012
|
+
const wave1 = Math.sin(t * 4.5 + i * 0.9) * 0.35;
|
|
1013
|
+
const wave2 = Math.sin(t * 7.2 + i * 1.4 + 2.0) * 0.15;
|
|
1014
|
+
const center = 1.0 - Math.abs(pos - 0.5) * 1.2;
|
|
1015
|
+
const base = 0.15 + energy * 0.85;
|
|
1016
|
+
const value = Math.max(0, Math.min(1, (wave1 + wave2 + 0.5) * base * center));
|
|
1017
|
+
const idx = Math.min(bars.length - 1, Math.round(value * (bars.length - 1)));
|
|
1018
|
+
out += bars[idx];
|
|
1019
|
+
}
|
|
1020
|
+
return out;
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
/**
|
|
1024
|
+
* v7.2 world-class — Liquid Braille audio waveform with truecolor
|
|
1025
|
+
* Aurora gradient. Per Gemini design recommendation:
|
|
1026
|
+
* - 8 effective vertical levels per CELL (4 dots × 2 columns) via
|
|
1027
|
+
* braille — vs 8 levels per cell with block-bars.
|
|
1028
|
+
* - 2 audio samples per cell width — 2× density of block-bar wave.
|
|
1029
|
+
* - Per-cell color picks an aurora stop based on local peak;
|
|
1030
|
+
* loud peaks "burn" into hot peach/red, soft tails stay cool
|
|
1031
|
+
* lavender. RGB interpolated at runtime.
|
|
1032
|
+
* Output is `cells` cells wide rendered as a single string with
|
|
1033
|
+
* inline ANSI 24-bit escapes; ends with a reset. Caller-side
|
|
1034
|
+
* width math should use `cells` not the byte-length of the result.
|
|
1035
|
+
*/
|
|
1036
|
+
function buildAuroraWave(level: number, cells = 16): string {
|
|
1037
|
+
// Generate 2*cells audio "samples" via multi-frequency sine
|
|
1038
|
+
// + the live RMS energy. Same organic motion as the legacy
|
|
1039
|
+
// wave but at higher density.
|
|
1040
|
+
const samples = 2 * cells;
|
|
1041
|
+
const t = Date.now() / 1000;
|
|
1042
|
+
const energy = Math.pow(level, 0.7);
|
|
1043
|
+
const arr: number[] = [];
|
|
1044
|
+
for (let i = 0; i < samples; i++) {
|
|
1045
|
+
const pos = i / samples;
|
|
1046
|
+
const wave1 = Math.sin(t * 4.5 + i * 0.45) * 0.35;
|
|
1047
|
+
const wave2 = Math.sin(t * 7.2 + i * 0.7 + 2.0) * 0.15;
|
|
1048
|
+
const center = 1.0 - Math.abs(pos - 0.5) * 1.0;
|
|
1049
|
+
const base = 0.1 + energy * 0.9;
|
|
1050
|
+
const value = Math.max(0, Math.min(1, (wave1 + wave2 + 0.5) * base * center));
|
|
1051
|
+
arr.push(value);
|
|
1052
|
+
}
|
|
1053
|
+
// Lazy-import to keep voice.ts hot path fast on cold start.
|
|
1054
|
+
const { liquidBraille, auroraColor } = require("./voice/ui-aura") as typeof import("./voice/ui-aura");
|
|
1055
|
+
return liquidBraille(arr, auroraColor);
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
// ─── Warmup Widget ──────────────────────────────────────────────────
|
|
1059
|
+
function showWarmupWidget() {
|
|
1060
|
+
if (!ctx?.hasUI) return;
|
|
1061
|
+
|
|
1062
|
+
const startTime = Date.now();
|
|
1063
|
+
|
|
1064
|
+
const renderWarmup = () => {
|
|
1065
|
+
if (!ctx?.hasUI) return;
|
|
1066
|
+
const elapsed = Date.now() - startTime;
|
|
1067
|
+
const progress = Math.min(elapsed / getHoldThresholdMs(), 1);
|
|
1068
|
+
|
|
1069
|
+
ctx.ui.setWidget(
|
|
1070
|
+
"voice-recording",
|
|
1071
|
+
(_tui, theme) => {
|
|
1072
|
+
return {
|
|
1073
|
+
invalidate() {},
|
|
1074
|
+
render(width: number): string[] {
|
|
1075
|
+
// v7.2 world-class — Same Floating Island chrome
|
|
1076
|
+
// as the active recording widget, with the
|
|
1077
|
+
// progress bar inside. Establishes visual
|
|
1078
|
+
// continuity between warmup → recording (same
|
|
1079
|
+
// island, content shifts, no jump).
|
|
1080
|
+
const { island, auroraColor, titleBreathe } =
|
|
1081
|
+
require("./voice/ui-aura") as typeof import("./voice/ui-aura");
|
|
1082
|
+
const dim = (s: string) => theme.fg("dim", s);
|
|
1083
|
+
const muted = (s: string) => theme.fg("muted", s);
|
|
1084
|
+
const accent = (s: string) => theme.fg("accent", s);
|
|
1085
|
+
const islandW = Math.max(36, Math.min(46, width - 2));
|
|
1086
|
+
|
|
1087
|
+
// Aurora gradient progress: ▰ filled / ▱ empty
|
|
1088
|
+
// with truecolor across the filled portion.
|
|
1089
|
+
const innerW = islandW - 2;
|
|
1090
|
+
const fixedW = 3 /* " ○ " */ + 1; /* trail */
|
|
1091
|
+
const meterCells = Math.max(12, innerW - fixedW);
|
|
1092
|
+
const filled = Math.round(progress * meterCells);
|
|
1093
|
+
let bar = "";
|
|
1094
|
+
for (let i = 0; i < meterCells; i++) {
|
|
1095
|
+
if (i < filled) {
|
|
1096
|
+
// Color stop based on position along filled portion.
|
|
1097
|
+
const t = filled === 0 ? 0 : i / Math.max(1, meterCells - 1);
|
|
1098
|
+
bar += auroraColor(t) + "▰";
|
|
1099
|
+
} else {
|
|
1100
|
+
bar += dim("▱");
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
bar += "\x1b[0m";
|
|
1104
|
+
|
|
1105
|
+
const dot = progress < 1 ? muted("○") : accent("●");
|
|
1106
|
+
const content = ` ${dot} ${bar} `;
|
|
1107
|
+
// Breathing title — same aurora-cycle as recording widget
|
|
1108
|
+
// so the warmup → recording transition feels seamless.
|
|
1109
|
+
const titleStyled = titleBreathe(Date.now()) + "\x1b[1mVoice Mode\x1b[0m";
|
|
1110
|
+
const footer = progress < 1 ? dim("hold to record") : accent("ready");
|
|
1111
|
+
return island({ width: islandW, title: titleStyled, content, footer, dim });
|
|
1112
|
+
},
|
|
1113
|
+
};
|
|
1114
|
+
},
|
|
1115
|
+
{ placement: "belowEditor" }
|
|
1116
|
+
);
|
|
1117
|
+
};
|
|
1118
|
+
|
|
1119
|
+
renderWarmup();
|
|
1120
|
+
warmupWidgetTimer = setInterval(renderWarmup, 90);
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
// ─── Recording Widget ───────────────────────────────────────────────
|
|
1124
|
+
let _recWidgetAnimTimer: ReturnType<typeof setInterval> | null = null;
|
|
1125
|
+
|
|
1126
|
+
function showRecordingWidget() {
|
|
1127
|
+
if (!ctx?.hasUI) return;
|
|
1128
|
+
|
|
1129
|
+
// Stop warmup animation if still running — seamless takeover,
|
|
1130
|
+
// no gap between warmup and recording widgets (same widget ID).
|
|
1131
|
+
clearWarmupWidget();
|
|
1132
|
+
|
|
1133
|
+
_recWidgetAnimTimer = setInterval(() => {
|
|
1134
|
+
showRecordingWidgetFrame();
|
|
1135
|
+
}, 150);
|
|
1136
|
+
|
|
1137
|
+
showRecordingWidgetFrame();
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
function showRecordingWidgetFrame() {
|
|
1141
|
+
if (!ctx?.hasUI) return;
|
|
1142
|
+
|
|
1143
|
+
// Minimal recording indicator below editor
|
|
1144
|
+
ctx.ui.setWidget(
|
|
1145
|
+
"voice-recording",
|
|
1146
|
+
(_tui, theme) => {
|
|
1147
|
+
return {
|
|
1148
|
+
invalidate() {},
|
|
1149
|
+
render(width: number): string[] {
|
|
1150
|
+
// v7.2 world-class — Floating Island + Liquid Braille +
|
|
1151
|
+
// Aurora gradient + breathing title + activity chip
|
|
1152
|
+
// + 300 ms fade-in transition from warmup.
|
|
1153
|
+
const { island, titleBreathe, activityTag } =
|
|
1154
|
+
require("./voice/ui-aura") as typeof import("./voice/ui-aura");
|
|
1155
|
+
const now = Date.now();
|
|
1156
|
+
const elapsed = (now - recordingStart) / 1000;
|
|
1157
|
+
const mins = Math.floor(elapsed / 60);
|
|
1158
|
+
const secs = elapsed % 60;
|
|
1159
|
+
const timeStr = mins > 0 ? `${mins}:${String(Math.floor(secs)).padStart(2, "0")}` : `${secs.toFixed(1)}s`;
|
|
1160
|
+
const dim = (s: string) => theme.fg("dim", s);
|
|
1161
|
+
const muted = (s: string) => theme.fg("muted", s);
|
|
1162
|
+
const accent = (s: string) => theme.fg("accent", s);
|
|
1163
|
+
|
|
1164
|
+
// Activity chip — one-glance "is sound coming in?"
|
|
1165
|
+
const chip = activityTag(audioLevelSmoothed, dim);
|
|
1166
|
+
const chipPlain = chip.replace(/\x1b\[[\d;]*[A-Za-z]/g, "");
|
|
1167
|
+
|
|
1168
|
+
// Compact 36-48 cols. Reserved cells:
|
|
1169
|
+
// " ● "(3) + wave + " · TIME "(timer+4) + " CHIP "(chip+3) + " "(1)
|
|
1170
|
+
const islandW = Math.max(36, Math.min(48, width - 2));
|
|
1171
|
+
const innerW = islandW - 2;
|
|
1172
|
+
const fixedW = 3 + (4 + timeStr.length) + (3 + chipPlain.length) + 1;
|
|
1173
|
+
const waveCells = Math.max(8, innerW - fixedW);
|
|
1174
|
+
|
|
1175
|
+
// Fade-in over first 300 ms — wave amplitude
|
|
1176
|
+
// scales 0→1 so the recording widget grows out
|
|
1177
|
+
// of warmup rather than snapping in.
|
|
1178
|
+
const sinceStart = Math.max(0, now - recordingStart);
|
|
1179
|
+
const fade = Math.min(1, sinceStart / 300);
|
|
1180
|
+
|
|
1181
|
+
const dot = theme.fg("error", getRecordDot());
|
|
1182
|
+
const wave = buildAuroraWave(audioLevelSmoothed * fade, waveCells);
|
|
1183
|
+
|
|
1184
|
+
// Breathing title — slow aurora-color cycle.
|
|
1185
|
+
const titleStyled = titleBreathe(now) + "\x1b[1mVoice Input\x1b[0m";
|
|
1186
|
+
|
|
1187
|
+
const content = ` ${dot} ${wave} ${dim("·")} ${muted(timeStr)} ${chip} `;
|
|
1188
|
+
const footerStyled = `${dim("release")} ${accent("↑")}`;
|
|
1189
|
+
return island({ width: islandW, title: titleStyled, content, footer: footerStyled, dim });
|
|
1190
|
+
},
|
|
1191
|
+
};
|
|
1192
|
+
},
|
|
1193
|
+
{ placement: "belowEditor" }
|
|
1194
|
+
);
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
// ─── Live Transcript ────────────────────────────────────────────────────
|
|
1198
|
+
// Instead of showing transcript in a widget, put it directly in the editor
|
|
1199
|
+
// input area so users see it where they type.
|
|
1200
|
+
|
|
1201
|
+
function updateLiveTranscriptWidget(interim: string, finals: string[]) {
|
|
1202
|
+
if (!ctx?.hasUI) return;
|
|
1203
|
+
|
|
1204
|
+
// DON'T stop the waveform animation — keep it running!
|
|
1205
|
+
// We still want the ● REC waveform + timer to show.
|
|
1206
|
+
// Just update the editor text with the live transcript.
|
|
1207
|
+
|
|
1208
|
+
const finalized = finals.join(" ");
|
|
1209
|
+
const displayText = finalized + (interim ? (finalized ? " " : "") + interim : "");
|
|
1210
|
+
|
|
1211
|
+
// Show live text directly in the editor input (prepend any existing text)
|
|
1212
|
+
if (displayText.trim()) {
|
|
1213
|
+
const prefix = editorTextBeforeVoice ? editorTextBeforeVoice + " " : "";
|
|
1214
|
+
ctx.ui.setEditorText(prefix + displayText);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
// ─── Voice: Start / Stop ─────────────────────────────────────────────────
|
|
1219
|
+
|
|
1220
|
+
let _startingRecording = false; // Re-entrancy guard for startVoiceRecording
|
|
1221
|
+
|
|
1222
|
+
async function startVoiceRecording(): Promise<boolean> {
|
|
1223
|
+
voiceDebug("startVoiceRecording called", { voiceState, hasUI: !!ctx?.hasUI, starting: _startingRecording });
|
|
1224
|
+
if (!ctx?.hasUI) return false;
|
|
1225
|
+
if (_startingRecording) return false; // Prevent overlapping starts during corruption guard sleep
|
|
1226
|
+
_startingRecording = true;
|
|
1227
|
+
|
|
1228
|
+
abortActiveSpeak();
|
|
1229
|
+
|
|
1230
|
+
try {
|
|
1231
|
+
// ── SESSION CORRUPTION GUARD ──
|
|
1232
|
+
// If we're still finalizing from a previous recording, abort it first.
|
|
1233
|
+
// This prevents the "slow connection overlaps new recording" bug.
|
|
1234
|
+
if (voiceState === "finalizing" || voiceState === "recording") {
|
|
1235
|
+
abortSession(activeSession);
|
|
1236
|
+
activeSession = null;
|
|
1237
|
+
clearRecordingAnimTimer();
|
|
1238
|
+
clearWarmupWidget();
|
|
1239
|
+
hideWidget();
|
|
1240
|
+
setVoiceState("idle");
|
|
1241
|
+
// Brief pause to let resources release
|
|
1242
|
+
await new Promise((r) => setTimeout(r, CORRUPTION_GUARD_MS));
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
// ── STALE TRANSCRIPT CLEANUP ──
|
|
1246
|
+
// Don't hideWidget() here — the warmup widget is still showing and
|
|
1247
|
+
// showRecordingWidget() will seamlessly replace it using the same
|
|
1248
|
+
// widget ID. Hiding it first causes a visible gap (jitter).
|
|
1249
|
+
|
|
1250
|
+
recordingStart = Date.now();
|
|
1251
|
+
|
|
1252
|
+
// Snapshot editor text before voice overwrites it with live transcript
|
|
1253
|
+
editorTextBeforeVoice = ctx?.hasUI ? ctx.ui.getEditorText() || "" : "";
|
|
1254
|
+
|
|
1255
|
+
return startStreamingRecording();
|
|
1256
|
+
} finally {
|
|
1257
|
+
_startingRecording = false;
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
// ── Pre-recording: start capturing audio during warmup so we don't miss words ──
|
|
1262
|
+
function startPreRecording() {
|
|
1263
|
+
abortActiveSpeak();
|
|
1264
|
+
if (preRecordingSession) return; // Already started
|
|
1265
|
+
if (config.backend === "local") return; // No pre-recording for local batch mode
|
|
1266
|
+
if (!resolveDeepgramApiKey(config)) return; // No key — skip silently
|
|
1267
|
+
if (!detectAudioCaptureTool()) return; // No audio tool — skip silently
|
|
1268
|
+
|
|
1269
|
+
voiceDebug("startPreRecording → capturing audio during warmup");
|
|
1270
|
+
|
|
1271
|
+
const session = startStreamingSession(config, {
|
|
1272
|
+
onTranscript: (interim, finals) => {
|
|
1273
|
+
// During warmup, silently accumulate transcript
|
|
1274
|
+
// (don't update UI — user hasn't committed to voice yet)
|
|
1275
|
+
voiceDebug("preRecording transcript", { interim: interim.slice(0, 50), finals: finals.length });
|
|
1276
|
+
},
|
|
1277
|
+
onDone: (fullText, meta) => {
|
|
1278
|
+
// Pre-recording ended (user released during warmup) — discard
|
|
1279
|
+
voiceDebug("preRecording onDone (discarded)", { fullText: fullText.slice(0, 50) });
|
|
1280
|
+
if (preRecordingSession === session) preRecordingSession = null;
|
|
1281
|
+
},
|
|
1282
|
+
onError: (err: string) => {
|
|
1283
|
+
voiceDebug("preRecording onError (ignored)", { err });
|
|
1284
|
+
if (preRecordingSession === session) preRecordingSession = null;
|
|
1285
|
+
},
|
|
1286
|
+
});
|
|
1287
|
+
|
|
1288
|
+
if (session) {
|
|
1289
|
+
preRecordingSession = session;
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1293
|
+
function abortPreRecording() {
|
|
1294
|
+
if (preRecordingSession) {
|
|
1295
|
+
voiceDebug("abortPreRecording → discarding warmup audio");
|
|
1296
|
+
abortSession(preRecordingSession);
|
|
1297
|
+
preRecordingSession = null;
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
async function startStreamingRecording(): Promise<boolean> {
|
|
1302
|
+
voiceDebug("startStreamingRecording called", {
|
|
1303
|
+
hasKey: !!resolveDeepgramApiKey(config),
|
|
1304
|
+
hasPreRecording: !!preRecordingSession,
|
|
1305
|
+
});
|
|
1306
|
+
setVoiceState("recording");
|
|
1307
|
+
|
|
1308
|
+
// ── Callbacks for the active recording session ──
|
|
1309
|
+
const recordingCallbacks = {
|
|
1310
|
+
onTranscript: (interim: string, finals: string[]) => {
|
|
1311
|
+
// Live transcript update — this is the key UX feature
|
|
1312
|
+
updateLiveTranscriptWidget(interim, finals);
|
|
1313
|
+
updateVoiceStatus();
|
|
1314
|
+
},
|
|
1315
|
+
onDone: (fullText: string, meta: { hadAudio: boolean; hadSpeech: boolean }) => {
|
|
1316
|
+
voiceDebug("onDone callback", { fullText: fullText.slice(0, 100), meta, voiceState, spaceConsumed });
|
|
1317
|
+
activeSession = null;
|
|
1318
|
+
clearRecordingAnimTimer();
|
|
1319
|
+
if (statusTimer) {
|
|
1320
|
+
clearInterval(statusTimer);
|
|
1321
|
+
statusTimer = null;
|
|
1322
|
+
}
|
|
1323
|
+
lastStopTime = Date.now();
|
|
1324
|
+
|
|
1325
|
+
if (!fullText.trim()) {
|
|
1326
|
+
// ── DISTINGUISH SILENCE VS NO SPEECH ──
|
|
1327
|
+
hideWidget();
|
|
1328
|
+
playSound("error");
|
|
1329
|
+
// Full state reset on empty result
|
|
1330
|
+
resetHoldState({ cooldown: 3000 });
|
|
1331
|
+
if (!meta.hadAudio) {
|
|
1332
|
+
ctx?.ui.notify("Microphone captured no audio. Check mic permissions.", "error");
|
|
1333
|
+
} else if (!meta.hadSpeech) {
|
|
1334
|
+
ctx?.ui.notify("Microphone captured silence — no speech detected.", "warning");
|
|
1335
|
+
} else {
|
|
1336
|
+
ctx?.ui.notify("No speech detected.", "warning");
|
|
1337
|
+
}
|
|
1338
|
+
setVoiceState("idle");
|
|
1339
|
+
return;
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
hideWidget();
|
|
1343
|
+
|
|
1344
|
+
if (ctx?.hasUI) {
|
|
1345
|
+
const prefix = editorTextBeforeVoice ? editorTextBeforeVoice + " " : "";
|
|
1346
|
+
const isLocal = config.backend === "local";
|
|
1347
|
+
const finalText = prefix + fullText;
|
|
1348
|
+
|
|
1349
|
+
if (isLocal) {
|
|
1350
|
+
// Local backend (batch mode): no interim transcripts were sent to the editor,
|
|
1351
|
+
// so we must always insert the final text. This is the ONLY place it arrives.
|
|
1352
|
+
ctx.ui.setEditorText(finalText);
|
|
1353
|
+
} else {
|
|
1354
|
+
// Streaming backend: interim transcripts already updated the editor live.
|
|
1355
|
+
// Only set final text if the editor still has content (user didn't hit Enter).
|
|
1356
|
+
const currentEditorText = ctx.ui.getEditorText?.() ?? "";
|
|
1357
|
+
if (currentEditorText.trim()) {
|
|
1358
|
+
ctx.ui.setEditorText(finalText);
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
// v7.1.1 — auto-submit on STT (config.autoSubmitOnSpeak).
|
|
1363
|
+
// When enabled, the transcribed text is sent to the
|
|
1364
|
+
// agent immediately instead of sitting in the editor
|
|
1365
|
+
// waiting for [enter]. Defaults OFF; user toggles via
|
|
1366
|
+
// /voice-autosubmit or settings panel.
|
|
1367
|
+
if (config.autoSubmitOnSpeak === true && finalText.trim().length > 0) {
|
|
1368
|
+
// v7.2.3 — if the agent is currently mid-turn
|
|
1369
|
+
// (especially mid-retry), DON'T auto-submit.
|
|
1370
|
+
// followUp queueing during a retry pile-up
|
|
1371
|
+
// makes the agent look like it's "looping" —
|
|
1372
|
+
// each queued message gets processed only
|
|
1373
|
+
// after the broken turn finishes. Better UX:
|
|
1374
|
+
// keep transcribed text in the editor, notify
|
|
1375
|
+
// the user, and let them press [enter]
|
|
1376
|
+
// manually when ready.
|
|
1377
|
+
if (agentBusy) {
|
|
1378
|
+
voiceDebug("autoSubmitOnSpeak: agent busy — leaving text in editor");
|
|
1379
|
+
try {
|
|
1380
|
+
ctx.ui.notify("Agent is busy — voice text held in editor. Press [↵] to send when ready.", "info");
|
|
1381
|
+
} catch {
|
|
1382
|
+
/* notify may fail silently */
|
|
1383
|
+
}
|
|
1384
|
+
// Skip dispatch but DON'T early-return —
|
|
1385
|
+
// the rest of the onDone handler still
|
|
1386
|
+
// needs to run state cleanup
|
|
1387
|
+
// (resetHoldState / setVoiceState("idle")).
|
|
1388
|
+
} else {
|
|
1389
|
+
// `sendUserMessage` lives on the `pi` ExtensionAPI
|
|
1390
|
+
// surface, NOT on `ctx`. Always triggers a turn.
|
|
1391
|
+
// `deliverAs: "followUp"` queues mid-stream
|
|
1392
|
+
// messages instead of throwing "Agent is already
|
|
1393
|
+
// processing".
|
|
1394
|
+
//
|
|
1395
|
+
// v7.2.2 (godspeed architect finding) — only
|
|
1396
|
+
// clear the editor AFTER send confirms. If
|
|
1397
|
+
// send rejects synchronously OR returns a
|
|
1398
|
+
// rejected Promise, the user's dictated text
|
|
1399
|
+
// stays visible so they can re-send.
|
|
1400
|
+
const send = (pi as any).sendUserMessage as ((text: string, opts?: any) => unknown) | undefined;
|
|
1401
|
+
if (typeof send === "function") {
|
|
1402
|
+
voiceDebug("autoSubmitOnSpeak: dispatching", { len: finalText.length });
|
|
1403
|
+
// godspeed architect finding — only clear if the
|
|
1404
|
+
// editor STILL contains exactly the dispatched
|
|
1405
|
+
// transcript. If the user typed more chars or
|
|
1406
|
+
// edited it while send was pending, leave their
|
|
1407
|
+
// edits alone.
|
|
1408
|
+
const dispatchedText = finalText;
|
|
1409
|
+
const clearAfterSuccess = () => {
|
|
1410
|
+
try {
|
|
1411
|
+
const cur = ctx?.ui.getEditorText?.() ?? "";
|
|
1412
|
+
if (cur === dispatchedText) {
|
|
1413
|
+
ctx?.ui.setEditorText("");
|
|
1414
|
+
}
|
|
1415
|
+
} catch {
|
|
1416
|
+
/* ui may be gone */
|
|
1417
|
+
}
|
|
1418
|
+
editorTextBeforeVoice = "";
|
|
1419
|
+
};
|
|
1420
|
+
try {
|
|
1421
|
+
const r = send(finalText, { deliverAs: "followUp" });
|
|
1422
|
+
if (r && typeof (r as Promise<unknown>).then === "function") {
|
|
1423
|
+
(r as Promise<unknown>).then(clearAfterSuccess).catch((err) => {
|
|
1424
|
+
voiceDebug("autoSubmitOnSpeak: sendUserMessage rejected", String(err));
|
|
1425
|
+
// Editor stays populated — user can re-press [enter].
|
|
1426
|
+
});
|
|
1427
|
+
} else {
|
|
1428
|
+
// Sync return (or undefined) — assume success.
|
|
1429
|
+
clearAfterSuccess();
|
|
1430
|
+
}
|
|
1431
|
+
} catch (err) {
|
|
1432
|
+
voiceDebug("autoSubmitOnSpeak: sendUserMessage threw sync", String(err));
|
|
1433
|
+
// Editor stays populated.
|
|
1434
|
+
}
|
|
1435
|
+
} else {
|
|
1436
|
+
voiceDebug("autoSubmitOnSpeak: pi.sendUserMessage not available on this Pi version");
|
|
1437
|
+
ctx.ui.notify(
|
|
1438
|
+
"Auto-submit ON but unavailable on this Pi version (need pi.sendUserMessage). " +
|
|
1439
|
+
"Press [enter] to send, or update Pi.",
|
|
1440
|
+
"warning"
|
|
1441
|
+
);
|
|
1442
|
+
}
|
|
1443
|
+
} // end else (agent not busy)
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
const elapsed = ((Date.now() - recordingStart) / 1000).toFixed(1);
|
|
1447
|
+
addToHistory(fullText, parseFloat(elapsed));
|
|
1448
|
+
}
|
|
1449
|
+
playSound("stop");
|
|
1450
|
+
// Full state reset on successful completion
|
|
1451
|
+
resetHoldState();
|
|
1452
|
+
setVoiceState("idle");
|
|
1453
|
+
},
|
|
1454
|
+
onError: (err: string) => {
|
|
1455
|
+
activeSession = null;
|
|
1456
|
+
clearRecordingAnimTimer();
|
|
1457
|
+
if (statusTimer) {
|
|
1458
|
+
clearInterval(statusTimer);
|
|
1459
|
+
statusTimer = null;
|
|
1460
|
+
}
|
|
1461
|
+
hideWidget();
|
|
1462
|
+
|
|
1463
|
+
// ── STOP THE LOOP ──
|
|
1464
|
+
// On error, fully reset ALL hold state AND set a cooldown
|
|
1465
|
+
// so incoming key-repeat events can't re-trigger activation.
|
|
1466
|
+
resetHoldState({ cooldown: 5000 });
|
|
1467
|
+
clearWarmupWidget();
|
|
1468
|
+
|
|
1469
|
+
ctx?.ui.notify(`Voice error: ${err}`, "error");
|
|
1470
|
+
playSound("error");
|
|
1471
|
+
setVoiceState("idle");
|
|
1472
|
+
},
|
|
1473
|
+
};
|
|
1474
|
+
|
|
1475
|
+
// ── Promote pre-recording, start local, or start streaming ──
|
|
1476
|
+
let session: VoiceSession | null;
|
|
1477
|
+
|
|
1478
|
+
if (config.backend === "local") {
|
|
1479
|
+
// Local backend: buffer audio, transcribe on stop
|
|
1480
|
+
const audioTool = detectAudioCaptureTool();
|
|
1481
|
+
if (!audioTool) {
|
|
1482
|
+
recordingCallbacks.onError("No audio capture tool found. Install one of: sox, ffmpeg, or arecord (Linux)");
|
|
1483
|
+
resetHoldState();
|
|
1484
|
+
setVoiceState("idle");
|
|
1485
|
+
return false;
|
|
1486
|
+
}
|
|
1487
|
+
const recProc = spawn(audioTool.cmd, audioTool.args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
1488
|
+
recProc.stderr?.on("data", (d: Buffer) => {
|
|
1489
|
+
const msg = d.toString().trim();
|
|
1490
|
+
if (msg.includes("buffer overrun") || msg.includes("Discarding") || msg.includes("Last message repeated"))
|
|
1491
|
+
return;
|
|
1492
|
+
voiceDebug(`${audioTool.name} stderr:`, msg);
|
|
1493
|
+
});
|
|
1494
|
+
session = startLocalSession(recProc, recordingCallbacks);
|
|
1495
|
+
|
|
1496
|
+
// Feed audio level meter for waveform animation
|
|
1497
|
+
recProc.stdout?.on("data", (chunk: Buffer) => {
|
|
1498
|
+
updateAudioLevel(chunk);
|
|
1499
|
+
});
|
|
1500
|
+
} else if (preRecordingSession) {
|
|
1501
|
+
// Promote: swap callbacks so pre-recorded audio feeds into real UI
|
|
1502
|
+
voiceDebug("Promoting pre-recording session to active");
|
|
1503
|
+
session = preRecordingSession;
|
|
1504
|
+
preRecordingSession = null;
|
|
1505
|
+
session.onTranscript = recordingCallbacks.onTranscript;
|
|
1506
|
+
session.onDone = recordingCallbacks.onDone;
|
|
1507
|
+
session.onError = recordingCallbacks.onError;
|
|
1508
|
+
// Flush any transcript already accumulated during warmup
|
|
1509
|
+
if (session.finalizedParts.length > 0 || session.interimText) {
|
|
1510
|
+
updateLiveTranscriptWidget(session.interimText, session.finalizedParts);
|
|
1511
|
+
}
|
|
1512
|
+
} else {
|
|
1513
|
+
session = startStreamingSession(config, recordingCallbacks);
|
|
1514
|
+
}
|
|
1515
|
+
|
|
1516
|
+
if (!session) {
|
|
1517
|
+
// startStreamingSession returned null — reset ALL state
|
|
1518
|
+
resetHoldState();
|
|
1519
|
+
setVoiceState("idle");
|
|
1520
|
+
return false;
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
activeSession = session;
|
|
1524
|
+
|
|
1525
|
+
// Status timer for elapsed time
|
|
1526
|
+
statusTimer = setInterval(() => {
|
|
1527
|
+
if (voiceState === "recording") {
|
|
1528
|
+
updateVoiceStatus();
|
|
1529
|
+
const elapsed = (Date.now() - recordingStart) / 1000;
|
|
1530
|
+
if (elapsed >= MAX_RECORDING_SECS) {
|
|
1531
|
+
stopVoiceRecording();
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
}, 1000);
|
|
1535
|
+
|
|
1536
|
+
showRecordingWidget();
|
|
1537
|
+
playSound("start");
|
|
1538
|
+
|
|
1539
|
+
// #13: arm the gap-based release timer only once recording is ready
|
|
1540
|
+
// (voiceState flipped to recording, success exit). Only hold sessions
|
|
1541
|
+
// (spaceDownTime != null — the user is holding SPACE) and non-Kitty
|
|
1542
|
+
// terminals arm: holding re-arms on each repeat, and release (repeat
|
|
1543
|
+
// stream stops) is perceived within RELEASE_DETECT_RECORDING_MS, so a
|
|
1544
|
+
// key-up inside the startup window also stops ~250ms later. Toggle/
|
|
1545
|
+
// dictation have no repeat stream keeping the timer alive — arming
|
|
1546
|
+
// would auto-stop recording ~250ms after start — so they must be
|
|
1547
|
+
// excluded (spaceDownTime == null). Kitty relies on the real key-release
|
|
1548
|
+
// event, so it stays clear.
|
|
1549
|
+
if (decideRecordingStartTimer({ kittyReleaseDetected, isHold: spaceDownTime != null }) === "arm") {
|
|
1550
|
+
resetReleaseDetect();
|
|
1551
|
+
}
|
|
1552
|
+
return true;
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
// ── Tail recording: keep capturing for 1.5s after space release ──
|
|
1556
|
+
function scheduleDelayedStop() {
|
|
1557
|
+
cancelDelayedStop(); // Clear any existing timer
|
|
1558
|
+
voiceDebug("scheduleDelayedStop → will stop in", TAIL_RECORDING_MS, "ms");
|
|
1559
|
+
tailRecordingTimer = setTimeout(() => {
|
|
1560
|
+
tailRecordingTimer = null;
|
|
1561
|
+
voiceDebug("tailRecordingTimer fired → stopping recording");
|
|
1562
|
+
stopVoiceRecording();
|
|
1563
|
+
}, TAIL_RECORDING_MS);
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
function cancelDelayedStop() {
|
|
1567
|
+
if (tailRecordingTimer) {
|
|
1568
|
+
clearTimeout(tailRecordingTimer);
|
|
1569
|
+
tailRecordingTimer = null;
|
|
1570
|
+
voiceDebug("cancelDelayedStop → tail recording timer cleared");
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
async function stopVoiceRecording() {
|
|
1575
|
+
cancelDelayedStop(); // Safety: clear any pending delayed stop
|
|
1576
|
+
voiceDebug("stopVoiceRecording called", { voiceState, hasActiveSession: !!activeSession });
|
|
1577
|
+
if (voiceState !== "recording" || !ctx) return;
|
|
1578
|
+
if (statusTimer) {
|
|
1579
|
+
clearInterval(statusTimer);
|
|
1580
|
+
statusTimer = null;
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
if (activeSession) {
|
|
1584
|
+
setVoiceState("finalizing");
|
|
1585
|
+
clearRecordingAnimTimer();
|
|
1586
|
+
hideWidget();
|
|
1587
|
+
if (activeSession.backend === "local") {
|
|
1588
|
+
// Local: show which model is transcribing + estimated time
|
|
1589
|
+
const modelName =
|
|
1590
|
+
LOCAL_MODELS.find((m) => m.id === (config.localModel || "whisper-small"))?.name ||
|
|
1591
|
+
config.localModel ||
|
|
1592
|
+
"local model";
|
|
1593
|
+
ctx?.ui.notify(`Transcribing with ${modelName}…`, "info");
|
|
1594
|
+
await stopLocalSession(activeSession, config);
|
|
1595
|
+
} else {
|
|
1596
|
+
stopStreamingSession(activeSession);
|
|
1597
|
+
}
|
|
1598
|
+
} else {
|
|
1599
|
+
// No active session — shouldn't happen, but recover gracefully
|
|
1600
|
+
voiceDebug("stopVoiceRecording: no active session, resetting to idle");
|
|
1601
|
+
hideWidget();
|
|
1602
|
+
setVoiceState("idle");
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
// ─── Hold-to-Talk State Machine ─────────────────────────────────────────
|
|
1607
|
+
//
|
|
1608
|
+
// SPACE key handling with STRICT hold-duration detection.
|
|
1609
|
+
//
|
|
1610
|
+
// TWO TERMINAL MODES:
|
|
1611
|
+
//
|
|
1612
|
+
// A) KITTY PROTOCOL (Ghostty on Linux, Kitty, WezTerm, etc.):
|
|
1613
|
+
// True key-down/repeat/release events. On first SPACE press,
|
|
1614
|
+
// immediately enter warmup (show countdown). If released before
|
|
1615
|
+
// HOLD_THRESHOLD_MS → cancel warmup, type a space. If held past
|
|
1616
|
+
// threshold → start recording. True release event stops recording.
|
|
1617
|
+
// No timer-based release detection needed.
|
|
1618
|
+
//
|
|
1619
|
+
// B) NON-KITTY (macOS Terminal, Ghostty on macOS, etc.):
|
|
1620
|
+
// No key-release event. Holding sends rapid press events (~30-90ms apart).
|
|
1621
|
+
// A single tap sends exactly ONE press.
|
|
1622
|
+
// Algorithm:
|
|
1623
|
+
// 1. First SPACE press → record time, start release-detect timer.
|
|
1624
|
+
// 2. No more presses within RELEASE_DETECT_MS (500ms) → TAP → type space.
|
|
1625
|
+
// 3. Rapid presses arrive → user is HOLDING. After REPEAT_CONFIRM_COUNT
|
|
1626
|
+
// rapid presses → enter warmup, show countdown.
|
|
1627
|
+
// 4. After HOLD_THRESHOLD_MS (1200ms) from first press → start recording.
|
|
1628
|
+
// 5. Recording continues while key-repeat events arrive.
|
|
1629
|
+
// Gap > RELEASE_DETECT_MS after RECORDING_GRACE_MS → stop.
|
|
1630
|
+
//
|
|
1631
|
+
// The RECORDING_GRACE_MS prevents the state transition at recording start
|
|
1632
|
+
// from being mistaken for a key release (brief gap in events).
|
|
1633
|
+
|
|
1634
|
+
function onSpaceReleaseDetected() {
|
|
1635
|
+
// GapTimer has already self-cleared (single-shot); nothing to reset here.
|
|
1636
|
+
voiceDebug("onSpaceReleaseDetected", {
|
|
1637
|
+
voiceState,
|
|
1638
|
+
holdConfirmed,
|
|
1639
|
+
spaceConsumed,
|
|
1640
|
+
spaceDownTime,
|
|
1641
|
+
spacePressCount,
|
|
1642
|
+
timeSinceRecStart: spaceConsumed ? Date.now() - recordingStartedAt : null,
|
|
1643
|
+
});
|
|
1644
|
+
|
|
1645
|
+
// If we never confirmed this was a hold (< REPEAT_CONFIRM_COUNT rapid presses),
|
|
1646
|
+
// then it was a TAP → space already passed through naturally (not consumed)
|
|
1647
|
+
if (!holdConfirmed && voiceState === "idle") {
|
|
1648
|
+
resetHoldState();
|
|
1649
|
+
clearWarmupWidget();
|
|
1650
|
+
hideWidget();
|
|
1651
|
+
// No need to type a space — the first press was NOT consumed,
|
|
1652
|
+
// so it already reached the focused UI component naturally.
|
|
1653
|
+
return;
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
// Released during warmup — cancel (user held but not long enough)
|
|
1657
|
+
if (voiceState === "warmup") {
|
|
1658
|
+
resetHoldState();
|
|
1659
|
+
abortPreRecording();
|
|
1660
|
+
clearWarmupWidget();
|
|
1661
|
+
hideWidget();
|
|
1662
|
+
setVoiceState("idle");
|
|
1663
|
+
spaceDownTime = null;
|
|
1664
|
+
spaceConsumed = false;
|
|
1665
|
+
spacePressCount = 0;
|
|
1666
|
+
holdConfirmed = false;
|
|
1667
|
+
// Don't type a space — user clearly intended to trigger voice but let go too early
|
|
1668
|
+
ctx?.ui.notify("Hold SPACE longer to activate voice.", "info");
|
|
1669
|
+
return;
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
// Released during recording — but ONLY if grace period has passed.
|
|
1673
|
+
// The grace period prevents the recording-start transition from being
|
|
1674
|
+
// mistaken for a key release.
|
|
1675
|
+
if (spaceConsumed && voiceState === "recording") {
|
|
1676
|
+
const timeSinceRecordingStart = Date.now() - recordingStartedAt;
|
|
1677
|
+
voiceDebug("release detected during recording", { timeSinceRecordingStart, RECORDING_GRACE_MS });
|
|
1678
|
+
if (timeSinceRecordingStart < RECORDING_GRACE_MS) {
|
|
1679
|
+
// Too soon after recording started — this is likely a false release
|
|
1680
|
+
// caused by the state transition. Re-arm the release detector.
|
|
1681
|
+
voiceDebug(" → too soon, re-arming (grace period)");
|
|
1682
|
+
resetReleaseDetect();
|
|
1683
|
+
return;
|
|
1684
|
+
}
|
|
1685
|
+
voiceDebug(" → scheduling delayed stop (tail recording)");
|
|
1686
|
+
resetHoldState();
|
|
1687
|
+
scheduleDelayedStop();
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
function resetReleaseDetect() {
|
|
1692
|
+
clearReleaseTimer();
|
|
1693
|
+
if (voiceState === "warmup" || voiceState === "recording" || spaceDownTime || spaceConsumed || holdConfirmed) {
|
|
1694
|
+
// Use longer timeout during active recording — key repeats can be
|
|
1695
|
+
// irregular when the system is under load (Deepgram streaming, etc.)
|
|
1696
|
+
const timeout = voiceState === "recording" || spaceConsumed ? RELEASE_DETECT_RECORDING_MS : RELEASE_DETECT_MS;
|
|
1697
|
+
voiceDebug("resetReleaseDetect", { timeout, voiceState, spaceConsumed });
|
|
1698
|
+
releaseGapTimer = new GapTimer(realTimerPort, { ms: timeout, onFire: onSpaceReleaseDetected });
|
|
1699
|
+
releaseGapTimer.arm();
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
function setupHoldToTalk() {
|
|
1704
|
+
if (!ctx?.hasUI) return;
|
|
1705
|
+
|
|
1706
|
+
if (terminalInputUnsub) {
|
|
1707
|
+
terminalInputUnsub();
|
|
1708
|
+
terminalInputUnsub = null;
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
terminalInputUnsub = ctx.ui.onTerminalInput((data: string) => {
|
|
1712
|
+
if (!config.enabled) return undefined;
|
|
1713
|
+
|
|
1714
|
+
// v7.1 §4 — escape priority routing for v7.1 widgets. When
|
|
1715
|
+
// no overlay (panel/help/picker) is in front (those run
|
|
1716
|
+
// inside ctx.ui.custom() and consume their own input), the
|
|
1717
|
+
// fallthrough order is: install widget → playback indicator
|
|
1718
|
+
// → editor. Most recent install wins precedence.
|
|
1719
|
+
if (matchesKey(data, Key.escape)) {
|
|
1720
|
+
if (activeInstallWidgets.size > 0) {
|
|
1721
|
+
// Most recent install — Maps preserve insertion order.
|
|
1722
|
+
const ids = Array.from(activeInstallWidgets.keys());
|
|
1723
|
+
const lastId = ids[ids.length - 1]!;
|
|
1724
|
+
const w = activeInstallWidgets.get(lastId);
|
|
1725
|
+
if (w) {
|
|
1726
|
+
w.cancel();
|
|
1727
|
+
return { consume: true };
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
if (activePlaybackIndicator) {
|
|
1731
|
+
activePlaybackIndicator.stop();
|
|
1732
|
+
return { consume: true };
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
// v7.1 §11 — F1 always opens help, regardless of context.
|
|
1737
|
+
// `?` is intentionally NOT bound here because the user is
|
|
1738
|
+
// in the editor and ? is a literal character. Help is
|
|
1739
|
+
// reachable by F1 or /voice-help instead.
|
|
1740
|
+
if (matchesKey(data, Key.f1) && ctx?.hasUI) {
|
|
1741
|
+
openHelpOverlay(ctx as unknown as ExtensionCommandContext).catch(() => {});
|
|
1742
|
+
return { consume: true };
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
// ── Track non-space keypresses for typing cooldown ──
|
|
1746
|
+
// If user was just typing (non-space key within TYPING_COOLDOWN_MS),
|
|
1747
|
+
// don't let space holds activate voice — they're just typing.
|
|
1748
|
+
if (!matchesKey(data, "space") && !isKeyRelease(data) && !isKeyRepeat(data)) {
|
|
1749
|
+
// Regular keypress that isn't space — user is typing
|
|
1750
|
+
if (data.length > 0 && data.charCodeAt(0) >= 32) {
|
|
1751
|
+
lastNonSpaceKeyTime = Date.now();
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
// ── SPACE handling ──
|
|
1756
|
+
if (matchesKey(data, "space")) {
|
|
1757
|
+
// ── ERROR COOLDOWN: block all voice activation for 5s after an error ──
|
|
1758
|
+
if (errorCooldownUntil > Date.now()) {
|
|
1759
|
+
// During cooldown, let space through as a normal character
|
|
1760
|
+
return undefined;
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
// ── TYPING COOLDOWN: if user was just typing, let space through ──
|
|
1764
|
+
// Apple-style: if a non-space key was pressed recently, this space
|
|
1765
|
+
// is part of typing (e.g., "hello world"), not a voice activation.
|
|
1766
|
+
// Only applies to NEW activations — don't interrupt active recording.
|
|
1767
|
+
if (
|
|
1768
|
+
voiceState === "idle" &&
|
|
1769
|
+
!spaceConsumed &&
|
|
1770
|
+
lastNonSpaceKeyTime > 0 &&
|
|
1771
|
+
Date.now() - lastNonSpaceKeyTime < TYPING_COOLDOWN_MS
|
|
1772
|
+
) {
|
|
1773
|
+
return undefined;
|
|
1774
|
+
}
|
|
1775
|
+
|
|
1776
|
+
voiceDebug("SPACE event", {
|
|
1777
|
+
isRelease: isKeyRelease(data),
|
|
1778
|
+
isRepeat: isKeyRepeat(data),
|
|
1779
|
+
voiceState,
|
|
1780
|
+
kittyReleaseDetected,
|
|
1781
|
+
holdConfirmed,
|
|
1782
|
+
spaceConsumed,
|
|
1783
|
+
spacePressCount,
|
|
1784
|
+
spaceDownTime: spaceDownTime ? Date.now() - spaceDownTime : null,
|
|
1785
|
+
dataHex: Buffer.from(data).toString("hex"),
|
|
1786
|
+
});
|
|
1787
|
+
|
|
1788
|
+
// ── Kitty key-release (true release event) ──
|
|
1789
|
+
if (isKeyRelease(data)) {
|
|
1790
|
+
kittyReleaseDetected = true;
|
|
1791
|
+
clearReleaseTimer();
|
|
1792
|
+
|
|
1793
|
+
// Released during warmup → cancel
|
|
1794
|
+
// If released very quickly (< 300ms), it was a tap → type a space
|
|
1795
|
+
// If released after 300ms+, user was trying voice → show hint
|
|
1796
|
+
if (voiceState === "warmup") {
|
|
1797
|
+
const holdDuration = spaceDownTime ? Date.now() - spaceDownTime : 0;
|
|
1798
|
+
resetHoldState();
|
|
1799
|
+
abortPreRecording();
|
|
1800
|
+
clearWarmupWidget();
|
|
1801
|
+
hideWidget();
|
|
1802
|
+
setVoiceState("idle");
|
|
1803
|
+
if (holdDuration < 300) {
|
|
1804
|
+
// Quick tap — just type a space
|
|
1805
|
+
if (ctx?.hasUI) ctx.ui.setEditorText((ctx.ui.getEditorText() || "") + " ");
|
|
1806
|
+
} else {
|
|
1807
|
+
// Held long enough to see warmup but let go → show hint
|
|
1808
|
+
ctx?.ui.notify("Hold SPACE longer to activate voice.", "info");
|
|
1809
|
+
}
|
|
1810
|
+
return { consume: true };
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
// Tap: released before warmup even started (shouldn't happen in
|
|
1814
|
+
// Kitty path since we enter warmup on first press, but handle anyway)
|
|
1815
|
+
if (spaceDownTime && !holdConfirmed && voiceState === "idle") {
|
|
1816
|
+
resetHoldState();
|
|
1817
|
+
if (ctx?.hasUI) ctx.ui.setEditorText((ctx.ui.getEditorText() || "") + " ");
|
|
1818
|
+
return { consume: true };
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
// Released during recording → schedule delayed stop (tail recording)
|
|
1822
|
+
if (spaceConsumed && voiceState === "recording") {
|
|
1823
|
+
resetHoldState();
|
|
1824
|
+
scheduleDelayedStop();
|
|
1825
|
+
return { consume: true };
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
spaceDownTime = null;
|
|
1829
|
+
spaceConsumed = false;
|
|
1830
|
+
spacePressCount = 0;
|
|
1831
|
+
holdConfirmed = false;
|
|
1832
|
+
return undefined;
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1835
|
+
// ── Kitty key-repeat ──
|
|
1836
|
+
if (isKeyRepeat(data)) {
|
|
1837
|
+
// #13: terminals without key-release events (e.g. Ghostty on
|
|
1838
|
+
// macOS, non-kitty modes) signal "released" purely by a GAP in
|
|
1839
|
+
// repeat events. Every repeat must re-arm the release-detect
|
|
1840
|
+
// timer — recording start clears it, and if nothing re-arms it
|
|
1841
|
+
// the release is never detected → hold-to-talk locks up forever.
|
|
1842
|
+
if (shouldArmReleaseDetectOnRepeat({ voiceState, kittyReleaseDetected })) {
|
|
1843
|
+
resetReleaseDetect();
|
|
1844
|
+
}
|
|
1845
|
+
// Already in recording/finalizing — consume (release timer kept alive above)
|
|
1846
|
+
if (voiceState === "recording" || voiceState === "finalizing" || spaceConsumed) {
|
|
1847
|
+
return { consume: true };
|
|
1848
|
+
}
|
|
1849
|
+
// Already in warmup — consume (hold timer is running)
|
|
1850
|
+
if (voiceState === "warmup") {
|
|
1851
|
+
return { consume: true };
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
// During initial hold detection: if we took PATH B on first
|
|
1855
|
+
// press (because kittyReleaseDetected was false), we need to
|
|
1856
|
+
// count these repeats to confirm the hold. Update state so
|
|
1857
|
+
// onSpaceReleaseDetected won't fire a false tap.
|
|
1858
|
+
if (spaceDownTime && !holdConfirmed) {
|
|
1859
|
+
// NOTE: Do NOT set kittyReleaseDetected here!
|
|
1860
|
+
// Ghostty on macOS sends repeat events but NO release events.
|
|
1861
|
+
// Only a true isKeyRelease() should flip the Kitty flag.
|
|
1862
|
+
|
|
1863
|
+
const now = Date.now();
|
|
1864
|
+
spacePressCount++;
|
|
1865
|
+
lastSpacePressTime = now;
|
|
1866
|
+
|
|
1867
|
+
// Enough repeats to confirm hold — enter warmup
|
|
1868
|
+
if (spacePressCount >= REPEAT_CONFIRM_COUNT) {
|
|
1869
|
+
holdConfirmed = true;
|
|
1870
|
+
setVoiceState("warmup");
|
|
1871
|
+
showWarmupWidget();
|
|
1872
|
+
startPreRecording();
|
|
1873
|
+
|
|
1874
|
+
const alreadyElapsed = now - (spaceDownTime || now);
|
|
1875
|
+
const remaining = Math.max(0, getHoldThresholdMs() - alreadyElapsed);
|
|
1876
|
+
|
|
1877
|
+
holdActivationTimer = setTimeout(() => {
|
|
1878
|
+
holdActivationTimer = null;
|
|
1879
|
+
if (voiceState === "warmup") {
|
|
1880
|
+
// Don't clearWarmupWidget() here — showRecordingWidget()
|
|
1881
|
+
// seamlessly replaces it using the same widget ID.
|
|
1882
|
+
spaceConsumed = true;
|
|
1883
|
+
recordingStartedAt = Date.now();
|
|
1884
|
+
// Clear release timer during async recording startup to
|
|
1885
|
+
// prevent a false stop. The 250ms gap timer is re-armed
|
|
1886
|
+
// once recording is actually ready — see startStreamingRecording (#13).
|
|
1887
|
+
clearReleaseTimer();
|
|
1888
|
+
voiceDebug("holdActivationTimer fired → starting recording (Kitty repeat path)");
|
|
1889
|
+
startVoiceRecording()
|
|
1890
|
+
.then((ok) => {
|
|
1891
|
+
if (!ok) {
|
|
1892
|
+
resetHoldState();
|
|
1893
|
+
setVoiceState("idle");
|
|
1894
|
+
}
|
|
1895
|
+
})
|
|
1896
|
+
.catch((err) => {
|
|
1897
|
+
voiceDebug("startVoiceRecording THREW", { error: String(err) });
|
|
1898
|
+
resetHoldState({ cooldown: 5000 });
|
|
1899
|
+
setVoiceState("idle");
|
|
1900
|
+
});
|
|
1901
|
+
} else {
|
|
1902
|
+
spaceDownTime = null;
|
|
1903
|
+
spaceConsumed = false;
|
|
1904
|
+
spacePressCount = 0;
|
|
1905
|
+
holdConfirmed = false;
|
|
1906
|
+
}
|
|
1907
|
+
}, remaining);
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
// Re-arm gap-based release detection — this is how
|
|
1911
|
+
// Ghostty-on-macOS (repeats but no release) detects
|
|
1912
|
+
// the key being released. (Preserved from the original
|
|
1913
|
+
// ordering — the recording-path re-arm above is separate.)
|
|
1914
|
+
resetReleaseDetect();
|
|
1915
|
+
return { consume: true };
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
return { consume: true };
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1921
|
+
// === Key PRESS (not repeat, not release) ===
|
|
1922
|
+
//
|
|
1923
|
+
// TWO TERMINAL MODES:
|
|
1924
|
+
//
|
|
1925
|
+
// A) Kitty protocol (kittyReleaseDetected = true):
|
|
1926
|
+
// Press fires ONCE on key-down. Repeats come as isKeyRepeat().
|
|
1927
|
+
// Release comes as isKeyRelease(). NO timer-based release detection
|
|
1928
|
+
// needed — the true release event handles everything.
|
|
1929
|
+
// On first press: enter warmup immediately and start hold timer.
|
|
1930
|
+
// (No need to wait for repeats to confirm hold.)
|
|
1931
|
+
//
|
|
1932
|
+
// B) Non-Kitty (macOS Terminal, etc.):
|
|
1933
|
+
// Holding a key sends rapid "press" events (~30-90ms apart).
|
|
1934
|
+
// A single tap sends exactly ONE press. There is NO release event.
|
|
1935
|
+
// We detect "tap vs hold" by counting rapid presses, and detect
|
|
1936
|
+
// "release" when no press arrives within RELEASE_DETECT_MS.
|
|
1937
|
+
|
|
1938
|
+
// If finalizing → ignore
|
|
1939
|
+
if (voiceState === "finalizing") {
|
|
1940
|
+
return { consume: true };
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
// If already recording → cancel any pending delayed stop and keep going
|
|
1944
|
+
if (voiceState === "recording") {
|
|
1945
|
+
cancelDelayedStop(); // User re-pressed — they want to keep recording
|
|
1946
|
+
spaceConsumed = true; // Re-arm hold state for the continued recording
|
|
1947
|
+
spaceDownTime = Date.now();
|
|
1948
|
+
holdConfirmed = true;
|
|
1949
|
+
if (!kittyReleaseDetected) {
|
|
1950
|
+
voiceDebug("SPACE during recording → cancel delayed stop, re-arm release detect");
|
|
1951
|
+
resetReleaseDetect();
|
|
1952
|
+
} else {
|
|
1953
|
+
voiceDebug("SPACE during recording → cancel delayed stop (Kitty)");
|
|
1954
|
+
}
|
|
1955
|
+
return { consume: true };
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
// If already in warmup → consume
|
|
1959
|
+
if (voiceState === "warmup") {
|
|
1960
|
+
if (!kittyReleaseDetected) {
|
|
1961
|
+
voiceDebug("SPACE during warmup → re-arm release detect");
|
|
1962
|
+
resetReleaseDetect();
|
|
1963
|
+
}
|
|
1964
|
+
return { consume: true };
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1967
|
+
// If we've already consumed space for this hold → consume
|
|
1968
|
+
// This handles the gap between holdActivationTimer firing and
|
|
1969
|
+
// voiceState transitioning to "recording" (async gap)
|
|
1970
|
+
if (spaceConsumed) {
|
|
1971
|
+
if (!kittyReleaseDetected) {
|
|
1972
|
+
voiceDebug("SPACE while spaceConsumed (async gap) → re-arm release detect");
|
|
1973
|
+
resetReleaseDetect();
|
|
1974
|
+
}
|
|
1975
|
+
return { consume: true };
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
// ──────────────────────────────────────────────────────────
|
|
1979
|
+
// PATH A: Kitty protocol — true key events available
|
|
1980
|
+
// ──────────────────────────────────────────────────────────
|
|
1981
|
+
if (kittyReleaseDetected) {
|
|
1982
|
+
// First press → immediately enter warmup (release event
|
|
1983
|
+
// will cancel if it was a tap)
|
|
1984
|
+
if (voiceState === "idle") {
|
|
1985
|
+
spaceDownTime = Date.now();
|
|
1986
|
+
spaceConsumed = false;
|
|
1987
|
+
spacePressCount = 1;
|
|
1988
|
+
lastSpacePressTime = Date.now();
|
|
1989
|
+
holdConfirmed = true; // Kitty: trust the press, release cancels
|
|
1990
|
+
|
|
1991
|
+
setVoiceState("warmup");
|
|
1992
|
+
showWarmupWidget();
|
|
1993
|
+
startPreRecording();
|
|
1994
|
+
|
|
1995
|
+
holdActivationTimer = setTimeout(() => {
|
|
1996
|
+
holdActivationTimer = null;
|
|
1997
|
+
if (voiceState === "warmup") {
|
|
1998
|
+
// Don't clearWarmupWidget() here — showRecordingWidget()
|
|
1999
|
+
// seamlessly replaces it using the same widget ID.
|
|
2000
|
+
spaceConsumed = true;
|
|
2001
|
+
recordingStartedAt = Date.now();
|
|
2002
|
+
voiceDebug("holdActivationTimer fired → starting recording (Kitty path)");
|
|
2003
|
+
startVoiceRecording()
|
|
2004
|
+
.then((ok) => {
|
|
2005
|
+
if (!ok) {
|
|
2006
|
+
resetHoldState();
|
|
2007
|
+
setVoiceState("idle");
|
|
2008
|
+
}
|
|
2009
|
+
})
|
|
2010
|
+
.catch((err) => {
|
|
2011
|
+
voiceDebug("startVoiceRecording THREW", { error: String(err) });
|
|
2012
|
+
resetHoldState({ cooldown: 5000 });
|
|
2013
|
+
setVoiceState("idle");
|
|
2014
|
+
});
|
|
2015
|
+
} else {
|
|
2016
|
+
spaceDownTime = null;
|
|
2017
|
+
spaceConsumed = false;
|
|
2018
|
+
spacePressCount = 0;
|
|
2019
|
+
holdConfirmed = false;
|
|
2020
|
+
}
|
|
2021
|
+
}, getHoldThresholdMs());
|
|
2022
|
+
|
|
2023
|
+
return { consume: true };
|
|
2024
|
+
}
|
|
2025
|
+
return { consume: true };
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
// ──────────────────────────────────────────────────────────
|
|
2029
|
+
// PATH B: Non-Kitty — gap-based hold/release detection
|
|
2030
|
+
// ──────────────────────────────────────────────────────────
|
|
2031
|
+
// Holding a key sends rapid press events.
|
|
2032
|
+
// We count presses and measure gaps to detect holds vs taps.
|
|
2033
|
+
if (spaceDownTime) {
|
|
2034
|
+
const now = Date.now();
|
|
2035
|
+
const gap = now - lastSpacePressTime;
|
|
2036
|
+
|
|
2037
|
+
if (gap < REPEAT_CONFIRM_MS) {
|
|
2038
|
+
// Rapid press = user is holding
|
|
2039
|
+
spacePressCount++;
|
|
2040
|
+
lastSpacePressTime = now;
|
|
2041
|
+
|
|
2042
|
+
if (spacePressCount >= REPEAT_CONFIRM_COUNT && !holdConfirmed) {
|
|
2043
|
+
holdConfirmed = true;
|
|
2044
|
+
setVoiceState("warmup");
|
|
2045
|
+
showWarmupWidget();
|
|
2046
|
+
startPreRecording();
|
|
2047
|
+
|
|
2048
|
+
const alreadyElapsed = now - spaceDownTime;
|
|
2049
|
+
const remaining = Math.max(0, getHoldThresholdMs() - alreadyElapsed);
|
|
2050
|
+
|
|
2051
|
+
holdActivationTimer = setTimeout(() => {
|
|
2052
|
+
holdActivationTimer = null;
|
|
2053
|
+
if (voiceState === "warmup") {
|
|
2054
|
+
// Don't clearWarmupWidget() here — showRecordingWidget()
|
|
2055
|
+
// seamlessly replaces it using the same widget ID.
|
|
2056
|
+
spaceConsumed = true;
|
|
2057
|
+
recordingStartedAt = Date.now();
|
|
2058
|
+
// CRITICAL: Clear release timer and DO NOT re-arm.
|
|
2059
|
+
// The next key-repeat press event will re-arm it.
|
|
2060
|
+
// Without this, the async startVoiceRecording creates
|
|
2061
|
+
// a gap where the release timer fires falsely.
|
|
2062
|
+
clearReleaseTimer();
|
|
2063
|
+
voiceDebug("holdActivationTimer fired → starting recording (non-Kitty)");
|
|
2064
|
+
startVoiceRecording()
|
|
2065
|
+
.then((ok) => {
|
|
2066
|
+
if (!ok) {
|
|
2067
|
+
resetHoldState();
|
|
2068
|
+
setVoiceState("idle");
|
|
2069
|
+
}
|
|
2070
|
+
// Do NOT re-arm release detect here!
|
|
2071
|
+
// The next SPACE key-repeat event will do it.
|
|
2072
|
+
// Re-arming here causes false stops because
|
|
2073
|
+
// the timer fires during the async gap.
|
|
2074
|
+
})
|
|
2075
|
+
.catch((err) => {
|
|
2076
|
+
voiceDebug("startVoiceRecording THREW", { error: String(err) });
|
|
2077
|
+
resetHoldState({ cooldown: 5000 });
|
|
2078
|
+
setVoiceState("idle");
|
|
2079
|
+
});
|
|
2080
|
+
} else {
|
|
2081
|
+
spaceDownTime = null;
|
|
2082
|
+
spaceConsumed = false;
|
|
2083
|
+
spacePressCount = 0;
|
|
2084
|
+
holdConfirmed = false;
|
|
2085
|
+
}
|
|
2086
|
+
}, remaining);
|
|
2087
|
+
}
|
|
2088
|
+
|
|
2089
|
+
resetReleaseDetect();
|
|
2090
|
+
return { consume: true };
|
|
2091
|
+
} else {
|
|
2092
|
+
// Gap too large → previous hold abandoned, new tap
|
|
2093
|
+
const wasInWarmup = (voiceState as VoiceState) === "warmup";
|
|
2094
|
+
resetHoldState();
|
|
2095
|
+
abortPreRecording();
|
|
2096
|
+
clearWarmupWidget();
|
|
2097
|
+
hideWidget();
|
|
2098
|
+
if (wasInWarmup) setVoiceState("idle");
|
|
2099
|
+
// Only type a space if we weren't already in warmup
|
|
2100
|
+
// (if we were in warmup, user was trying to activate voice, not type)
|
|
2101
|
+
// Note: first space already passed through naturally (not consumed)
|
|
2102
|
+
// so we don't need to manually type it here
|
|
2103
|
+
// Fall through to treat this as a new first press
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
// IDLE — first SPACE press (non-Kitty path)
|
|
2108
|
+
// Do NOT consume — let it pass through to whatever UI is focused
|
|
2109
|
+
// (editor, search box, picker, etc.). Only start consuming after
|
|
2110
|
+
// we confirm it's a hold via REPEAT_CONFIRM_COUNT rapid presses.
|
|
2111
|
+
if (voiceState === "idle") {
|
|
2112
|
+
spaceDownTime = Date.now();
|
|
2113
|
+
spaceConsumed = false;
|
|
2114
|
+
spacePressCount = 1;
|
|
2115
|
+
lastSpacePressTime = Date.now();
|
|
2116
|
+
holdConfirmed = false;
|
|
2117
|
+
|
|
2118
|
+
resetReleaseDetect();
|
|
2119
|
+
|
|
2120
|
+
// Don't consume — let the space reach the focused UI component
|
|
2121
|
+
return undefined;
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
if (spaceConsumed) return { consume: true };
|
|
2125
|
+
return undefined;
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
// ── Any other key pressed → cancel potential hold ──
|
|
2129
|
+
if (spaceDownTime && !holdConfirmed && voiceState === "idle") {
|
|
2130
|
+
resetHoldState();
|
|
2131
|
+
// No need to insert a space manually — the first space press was
|
|
2132
|
+
// already allowed to pass through to the focused UI component.
|
|
2133
|
+
return undefined;
|
|
2134
|
+
}
|
|
2135
|
+
|
|
2136
|
+
if (voiceState === "warmup" && holdConfirmed && !spaceConsumed) {
|
|
2137
|
+
clearWarmupWidget();
|
|
2138
|
+
hideWidget();
|
|
2139
|
+
resetHoldState();
|
|
2140
|
+
setVoiceState("idle");
|
|
2141
|
+
return undefined;
|
|
2142
|
+
}
|
|
2143
|
+
|
|
2144
|
+
// ── Escape key — cancel voice / double-escape clears editor ──
|
|
2145
|
+
// Skip release/repeat events — only act on actual presses
|
|
2146
|
+
if (matchesKey(data, "escape") && !isKeyRelease(data) && !isKeyRepeat(data)) {
|
|
2147
|
+
// During recording: cancel recording and clear transcript
|
|
2148
|
+
if (voiceState === "recording" || voiceState === "warmup" || voiceState === "finalizing") {
|
|
2149
|
+
voiceDebug("Escape pressed → canceling voice");
|
|
2150
|
+
abortPreRecording();
|
|
2151
|
+
if (activeSession) {
|
|
2152
|
+
abortSession(activeSession);
|
|
2153
|
+
activeSession = null;
|
|
2154
|
+
}
|
|
2155
|
+
clearRecordingAnimTimer();
|
|
2156
|
+
clearWarmupWidget();
|
|
2157
|
+
hideWidget();
|
|
2158
|
+
if (statusTimer) {
|
|
2159
|
+
clearInterval(statusTimer);
|
|
2160
|
+
statusTimer = null;
|
|
2161
|
+
}
|
|
2162
|
+
// Restore editor text to what it was before recording
|
|
2163
|
+
if (ctx?.hasUI) ctx.ui.setEditorText(editorTextBeforeVoice);
|
|
2164
|
+
resetHoldState();
|
|
2165
|
+
playSound("error");
|
|
2166
|
+
setVoiceState("idle");
|
|
2167
|
+
lastEscapeTime = Date.now();
|
|
2168
|
+
return { consume: true };
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
// In idle: double-escape (two presses within 500ms) clears editor
|
|
2172
|
+
if (voiceState === "idle") {
|
|
2173
|
+
const now = Date.now();
|
|
2174
|
+
if (lastEscapeTime > 0 && now - lastEscapeTime < 500) {
|
|
2175
|
+
if (ctx?.hasUI) {
|
|
2176
|
+
const currentText = ctx.ui.getEditorText() || "";
|
|
2177
|
+
if (currentText.trim()) {
|
|
2178
|
+
ctx.ui.setEditorText("");
|
|
2179
|
+
lastEscapeTime = 0;
|
|
2180
|
+
return { consume: true };
|
|
2181
|
+
}
|
|
2182
|
+
}
|
|
2183
|
+
}
|
|
2184
|
+
lastEscapeTime = now;
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
return undefined;
|
|
2189
|
+
});
|
|
2190
|
+
}
|
|
2191
|
+
|
|
2192
|
+
// ─── Shortcuts ───────────────────────────────────────────────────────────
|
|
2193
|
+
|
|
2194
|
+
// resolvedToggleShortcut is a runtime config string, but pi.registerShortcut
|
|
2195
|
+
// wants the literal-union KeyId type. The string was validated by
|
|
2196
|
+
// isValidShortcut() in loadGlobalToggleShortcut() (modifier+key shape with
|
|
2197
|
+
// a known modifier set), so asserting `as KeyId` is honest — `as any` was
|
|
2198
|
+
// strictly worse because it hid the intent and would have masked a real
|
|
2199
|
+
// signature change. If pi-tui later promotes KeyId to `string`, this
|
|
2200
|
+
// assertion just becomes a no-op.
|
|
2201
|
+
pi.registerShortcut(resolvedToggleShortcut as KeyId, {
|
|
2202
|
+
description: "Toggle voice recording (start/stop)",
|
|
2203
|
+
handler: async (handlerCtx) => {
|
|
2204
|
+
ctx = handlerCtx;
|
|
2205
|
+
if (!config.enabled) {
|
|
2206
|
+
handlerCtx.ui.notify("Voice disabled. Use /voice on", "warning");
|
|
2207
|
+
return;
|
|
2208
|
+
}
|
|
2209
|
+
if (dictationMode) {
|
|
2210
|
+
// The configured toggle shortcut stops dictation mode
|
|
2211
|
+
dictationMode = false;
|
|
2212
|
+
if (voiceState === "recording") {
|
|
2213
|
+
await stopVoiceRecording();
|
|
2214
|
+
}
|
|
2215
|
+
handlerCtx.ui.notify("Dictation mode stopped.", "info");
|
|
2216
|
+
return;
|
|
2217
|
+
}
|
|
2218
|
+
if (voiceState === "idle") {
|
|
2219
|
+
spaceConsumed = true;
|
|
2220
|
+
const ok = await startVoiceRecording();
|
|
2221
|
+
if (!ok) {
|
|
2222
|
+
spaceConsumed = false;
|
|
2223
|
+
}
|
|
2224
|
+
} else if (voiceState === "recording") {
|
|
2225
|
+
resetHoldState();
|
|
2226
|
+
await stopVoiceRecording();
|
|
2227
|
+
} else if (voiceState === "warmup") {
|
|
2228
|
+
// Cancel warmup
|
|
2229
|
+
abortPreRecording();
|
|
2230
|
+
clearWarmupWidget();
|
|
2231
|
+
hideWidget();
|
|
2232
|
+
resetHoldState();
|
|
2233
|
+
setVoiceState("idle");
|
|
2234
|
+
}
|
|
2235
|
+
// voiceState === "finalizing" → ignore (wait for transcript)
|
|
2236
|
+
},
|
|
2237
|
+
});
|
|
2238
|
+
|
|
2239
|
+
// ─── Lifecycle ───────────────────────────────────────────────────────────
|
|
2240
|
+
|
|
2241
|
+
pi.on("session_start", async (event, startCtx) => {
|
|
2242
|
+
// `event.reason` was added in pi-mono 0.65.0 ("startup" | "reload" | "new" |
|
|
2243
|
+
// "resume" | "fork"). Older Pi versions don't include it. Narrow defensively
|
|
2244
|
+
// so the same code typechecks against both 0.57-era and 0.65+ types.
|
|
2245
|
+
const reason = (event as { reason?: string } | undefined)?.reason ?? "startup";
|
|
2246
|
+
const isStartup = reason === "startup";
|
|
2247
|
+
|
|
2248
|
+
// Non-startup transitions: pi-mono >= 0.68.0 fires session_shutdown first
|
|
2249
|
+
// (and awaits it), so voiceCleanup() has already run. This is just belt-
|
|
2250
|
+
// and-suspenders for older Pi versions that may skip session_shutdown on
|
|
2251
|
+
// session replacement. The try/catch matches session_shutdown — a throw
|
|
2252
|
+
// inside voiceCleanup (e.g. a child process kill EPERM under load) must
|
|
2253
|
+
// not abort handler execution and leave ctx unassigned.
|
|
2254
|
+
if (!isStartup) {
|
|
2255
|
+
try {
|
|
2256
|
+
voiceCleanup();
|
|
2257
|
+
} catch (err) {
|
|
2258
|
+
voiceDebug("voiceCleanup threw during session_start", { error: String(err) });
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
ctx = startCtx;
|
|
2263
|
+
currentCwd = startCtx.cwd;
|
|
2264
|
+
const loaded = loadConfigWithSource(startCtx.cwd);
|
|
2265
|
+
config = loaded.config;
|
|
2266
|
+
configSource = loaded.source;
|
|
2267
|
+
|
|
2268
|
+
// v7.1.3 — version banner emitted to debug log on every session
|
|
2269
|
+
// start. Lets users / support verify which extension build is
|
|
2270
|
+
// actually loaded after a `pi install .` (path-installed
|
|
2271
|
+
// extensions cache modules across `pi install` reinstalls — only
|
|
2272
|
+
// a fresh `pi` process picks up source changes).
|
|
2273
|
+
voiceDebug("pi-listen v7.1.3 loaded", { reason });
|
|
2274
|
+
|
|
2275
|
+
// Migration / setup runs on EVERY session_start, regardless of reason.
|
|
2276
|
+
// Only the first-run notification is gated on isStartup.
|
|
2277
|
+
if (config.onboarding.completed) {
|
|
2278
|
+
// Always refresh the status bar — when voice is disabled,
|
|
2279
|
+
// updateVoiceStatus() clears the entry so users don't see stale
|
|
2280
|
+
// "MIC STREAM" text from a prior session. Hold-to-talk wiring
|
|
2281
|
+
// only runs when enabled.
|
|
2282
|
+
updateVoiceStatus();
|
|
2283
|
+
if (config.enabled) {
|
|
2284
|
+
setupHoldToTalk();
|
|
2285
|
+
}
|
|
2286
|
+
return;
|
|
2287
|
+
}
|
|
2288
|
+
|
|
2289
|
+
// Onboarding not complete. Bail before the migration / hint UI work if the
|
|
2290
|
+
// session has no UI surface — non-interactive sessions can't display
|
|
2291
|
+
// notifications anyway, and migration wiring (setupHoldToTalk) requires UI.
|
|
2292
|
+
if (!startCtx.hasUI) return;
|
|
2293
|
+
|
|
2294
|
+
// Try migration if a backend is already configured.
|
|
2295
|
+
const hasKey = !!resolveDeepgramApiKey(config);
|
|
2296
|
+
const hasLocalModel = config.backend === "local" && !!config.localModel;
|
|
2297
|
+
const audioTool = detectAudioCaptureTool();
|
|
2298
|
+
if (hasKey || hasLocalModel) {
|
|
2299
|
+
// Backend configured (Deepgram key or local model) — auto-activate.
|
|
2300
|
+
// Migration runs every transition; the welcome notification is gated
|
|
2301
|
+
// on isStartup so /new and /resume don't re-spam the hint.
|
|
2302
|
+
config.onboarding.completed = true;
|
|
2303
|
+
config.onboarding.completedAt = new Date().toISOString();
|
|
2304
|
+
config.onboarding.source = "migration";
|
|
2305
|
+
const configToSave = getSessionStartPersistedConfig({
|
|
2306
|
+
config,
|
|
2307
|
+
envDeepgramApiKey: process.env.DEEPGRAM_API_KEY,
|
|
2308
|
+
});
|
|
2309
|
+
saveConfig(configToSave, config.scope === "project" ? "project" : "global", currentCwd);
|
|
2310
|
+
updateVoiceStatus();
|
|
2311
|
+
setupHoldToTalk();
|
|
2312
|
+
if (!isStartup) return;
|
|
2313
|
+
const backendLabel = hasLocalModel
|
|
2314
|
+
? `Local model: ${LOCAL_MODELS.find((m) => m.id === config.localModel)?.name || config.localModel} (offline, batch mode)`
|
|
2315
|
+
: "Deepgram Nova-3 (cloud, live streaming)";
|
|
2316
|
+
const lines = [
|
|
2317
|
+
"pi-listen ready!",
|
|
2318
|
+
"",
|
|
2319
|
+
" Hold SPACE to record → release to transcribe",
|
|
2320
|
+
` ${toggleShortcutLabel} to toggle recording`,
|
|
2321
|
+
` Backend: ${backendLabel}`,
|
|
2322
|
+
` Audio: ${audioTool ? `${audioTool.name}` : "NONE — install sox or ffmpeg"}`,
|
|
2323
|
+
"",
|
|
2324
|
+
" /voice-settings to change backend, model, or language",
|
|
2325
|
+
];
|
|
2326
|
+
startCtx.ui.notify(lines.join("\n"), audioTool ? "info" : "warning");
|
|
2327
|
+
return;
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2330
|
+
// No backend configured — show install hint only on actual startup.
|
|
2331
|
+
if (!isStartup) return;
|
|
2332
|
+
const lines = [
|
|
2333
|
+
"pi-listen installed — voice input for Pi",
|
|
2334
|
+
"",
|
|
2335
|
+
" Two backends available:",
|
|
2336
|
+
" • Deepgram — cloud, live streaming, $200 free credit (6–12 months of use)",
|
|
2337
|
+
" • Local models — fully offline, no API key, auto-downloads on first use",
|
|
2338
|
+
"",
|
|
2339
|
+
` Audio capture: ${audioTool ? `${audioTool.name} ✓` : "not found — install sox or ffmpeg"}`,
|
|
2340
|
+
"",
|
|
2341
|
+
" Run /voice-settings to choose your backend and get started.",
|
|
2342
|
+
];
|
|
2343
|
+
startCtx.ui.notify(lines.join("\n"), "info");
|
|
2344
|
+
});
|
|
2345
|
+
|
|
2346
|
+
pi.on("session_shutdown", async (event) => {
|
|
2347
|
+
// Synchronous teardown FIRST. Pi-mono >= 0.65.0 awaits the handler promise
|
|
2348
|
+
// before firing the replacement session_start, so the late-ctx-null race
|
|
2349
|
+
// the audit flagged is not present on current Pi. Keeping the order
|
|
2350
|
+
// (cleanup → null → await import) is still cheap insurance for older Pi
|
|
2351
|
+
// versions whose replacement path may not await. The try/catch is so a
|
|
2352
|
+
// throw inside voiceCleanup (e.g. a child process kill EPERM under load)
|
|
2353
|
+
// can't leak ctx or skip the recognizer cache clear.
|
|
2354
|
+
try {
|
|
2355
|
+
voiceCleanup();
|
|
2356
|
+
} catch (err) {
|
|
2357
|
+
voiceDebug("voiceCleanup threw during shutdown", { error: String(err) });
|
|
2358
|
+
}
|
|
2359
|
+
ctx = null;
|
|
2360
|
+
|
|
2361
|
+
// Clear the sherpa recognizer cache ONLY on terminal quit. On older Pi
|
|
2362
|
+
// versions (< 0.65.0) shutdown handlers are not awaited before the
|
|
2363
|
+
// replacement session_start, so an `await import()` here can race with
|
|
2364
|
+
// the new session re-initializing the recognizer — and our late
|
|
2365
|
+
// clearRecognizerCache() would wipe the recognizer the new session just
|
|
2366
|
+
// created. Keeping the cache across non-quit transitions is also faster:
|
|
2367
|
+
// /reload, /new, /fork, /resume typically reuse the same model+language,
|
|
2368
|
+
// so the recognizer is still hot. Per-session language/model changes are
|
|
2369
|
+
// already invalidated in voice/settings-panel.ts when the user picks a
|
|
2370
|
+
// different model. Reason is undefined on pre-0.65 Pi (where shutdown
|
|
2371
|
+
// only ever fired on quit), so we treat undefined the same as "quit".
|
|
2372
|
+
const reason = (event as { reason?: string } | undefined)?.reason;
|
|
2373
|
+
if (reason === "quit" || reason === undefined) {
|
|
2374
|
+
try {
|
|
2375
|
+
const { clearRecognizerCache } = await import("./voice/sherpa-engine");
|
|
2376
|
+
clearRecognizerCache();
|
|
2377
|
+
} catch {}
|
|
2378
|
+
}
|
|
2379
|
+
});
|
|
2380
|
+
|
|
2381
|
+
// Note: pi-mono < 0.65.0 fired a discrete "session_switch" event for
|
|
2382
|
+
// /new, /resume, /fork. That event was removed in 0.65.0 in favor of the
|
|
2383
|
+
// session_shutdown → session_start (with reason) flow handled above.
|
|
2384
|
+
// We don't register a shim here because package.json:peerDependencies
|
|
2385
|
+
// requires "@earendil-works/pi-coding-agent": "*", so a host without
|
|
2386
|
+
// the new flow can't install this extension in the first place.
|
|
2387
|
+
|
|
2388
|
+
// ─── Auto-speak (TTS after assistant turn ends) ─────────────────────
|
|
2389
|
+
//
|
|
2390
|
+
// When ttsAutoSpeak is true AND ttsEnabled is true, subscribe to
|
|
2391
|
+
// `turn_end` and pipe the assistant's response through speak() with
|
|
2392
|
+
// the same text-filter and length-cap logic the manual command uses.
|
|
2393
|
+
//
|
|
2394
|
+
// The handler is always registered; it short-circuits when the flags
|
|
2395
|
+
// are off. This way toggling `ttsAutoSpeak` via /voice-speak-toggle
|
|
2396
|
+
// doesn't require a session restart.
|
|
2397
|
+
//
|
|
2398
|
+
// Rate limit: track the last auto-speak timestamp and skip if a new
|
|
2399
|
+
// turn ends within ~3 seconds. Prevents the agent's rapid-fire
|
|
2400
|
+
// short responses from queueing up unread audio.
|
|
2401
|
+
let lastAutoSpeakAt = 0;
|
|
2402
|
+
const AUTO_SPEAK_RATE_LIMIT_MS = 3000;
|
|
2403
|
+
|
|
2404
|
+
// v7.2.3 — track agent busy state so autoSubmit can skip dispatch
|
|
2405
|
+
// when the agent is mid-turn (and especially mid-retry). Otherwise
|
|
2406
|
+
// holding space during a "Retrying (n/3) in Xs..." cycle queues
|
|
2407
|
+
// followUp messages that pile onto the failing turn — feels like
|
|
2408
|
+
// the voice extension is "looping" because the agent never recovers.
|
|
2409
|
+
let agentBusy = false;
|
|
2410
|
+
pi.on("agent_start", async () => {
|
|
2411
|
+
agentBusy = true;
|
|
2412
|
+
});
|
|
2413
|
+
pi.on("agent_end", async () => {
|
|
2414
|
+
agentBusy = false;
|
|
2415
|
+
});
|
|
2416
|
+
|
|
2417
|
+
// v7.2.1 — streaming auto-speak. Instead of waiting for `turn_end`
|
|
2418
|
+
// (which only fires AFTER the full response is generated), subscribe
|
|
2419
|
+
// to `message_update` and `message_end`:
|
|
2420
|
+
// - message_update fires as the LLM streams tokens. We extract
|
|
2421
|
+
// the accumulated text, find new sentence boundaries since
|
|
2422
|
+
// the last speak, and queue those sentences for synthesis.
|
|
2423
|
+
// - message_end flushes any remaining buffer (one final speak
|
|
2424
|
+
// for trailing text without a sentence terminator).
|
|
2425
|
+
// Result: TTS starts speaking the FIRST sentence within ~1 second
|
|
2426
|
+
// of the agent producing it, instead of after the entire response
|
|
2427
|
+
// completes. Sentence chunking + pipelined synth (already in
|
|
2428
|
+
// speak.ts) keeps audio flowing continuously.
|
|
2429
|
+
//
|
|
2430
|
+
// Per-message tracking: each assistant message has its own
|
|
2431
|
+
// `spokenLen` cursor so concurrent messages (compaction, sub-turns)
|
|
2432
|
+
// don't cross-talk. Map keyed by message id.
|
|
2433
|
+
|
|
2434
|
+
interface MessageStreamState {
|
|
2435
|
+
spokenLen: number; // chars of accumulated text already queued for speech
|
|
2436
|
+
pending: Promise<void>; // chain of in-flight speak() calls — serialize per message
|
|
2437
|
+
}
|
|
2438
|
+
const messageStreams = new Map<string, MessageStreamState>();
|
|
2439
|
+
|
|
2440
|
+
const SENTENCE_TERMINATORS = /[.!?](?=\s|$)|[\n]{1,}/g;
|
|
2441
|
+
|
|
2442
|
+
function extractAccumulatedText(message: any): string {
|
|
2443
|
+
if (!message || !Array.isArray(message.content)) return "";
|
|
2444
|
+
return (message.content as any[])
|
|
2445
|
+
.filter((c) => c?.type === "text" && typeof c.text === "string")
|
|
2446
|
+
.map((c) => c.text as string)
|
|
2447
|
+
.join("");
|
|
2448
|
+
}
|
|
2449
|
+
|
|
2450
|
+
/** Detect last sentence boundary at or before `maxIdx` in `text`.
|
|
2451
|
+
* Returns the index AFTER the terminator (so [0..idx] is a complete
|
|
2452
|
+
* block). Returns -1 if none found. */
|
|
2453
|
+
function lastSentenceEnd(text: string, fromIdx: number): number {
|
|
2454
|
+
let lastEnd = -1;
|
|
2455
|
+
const re = /[.!?](?=\s|$)|\n+/g;
|
|
2456
|
+
let m: RegExpExecArray | null;
|
|
2457
|
+
while ((m = re.exec(text)) !== null) {
|
|
2458
|
+
const end = m.index + m[0].length;
|
|
2459
|
+
if (end > fromIdx) lastEnd = end;
|
|
2460
|
+
}
|
|
2461
|
+
return lastEnd;
|
|
2462
|
+
}
|
|
2463
|
+
|
|
2464
|
+
// Eager-speak threshold — if the buffered new text grows past this
|
|
2465
|
+
// many chars without a sentence terminator (e.g. very long
|
|
2466
|
+
// sentence mid-stream), speak what we have at the next clause
|
|
2467
|
+
// boundary (`,` `;` ` — `) or flush as-is. Keeps latency low for
|
|
2468
|
+
// long-winded responses.
|
|
2469
|
+
const EAGER_SPEAK_CHARS = 80;
|
|
2470
|
+
const EAGER_CLAUSE_RE = /[,;:—](?=\s)/g;
|
|
2471
|
+
|
|
2472
|
+
function lastClauseEnd(text: string): number {
|
|
2473
|
+
let last = -1;
|
|
2474
|
+
EAGER_CLAUSE_RE.lastIndex = 0;
|
|
2475
|
+
let m: RegExpExecArray | null;
|
|
2476
|
+
while ((m = EAGER_CLAUSE_RE.exec(text)) !== null) {
|
|
2477
|
+
last = m.index + m[0].length;
|
|
2478
|
+
}
|
|
2479
|
+
return last;
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2482
|
+
async function maybeSpeakNew(messageId: string, fullText: string, isFinal: boolean): Promise<void> {
|
|
2483
|
+
if (!config.ttsEnabled || !config.ttsAutoSpeak) return;
|
|
2484
|
+
// Don't speak while STT is hot — feedback loop hazard.
|
|
2485
|
+
if (voiceState === "warmup" || voiceState === "recording" || voiceState === "finalizing") return;
|
|
2486
|
+
if (!fullText) return;
|
|
2487
|
+
|
|
2488
|
+
let state = messageStreams.get(messageId);
|
|
2489
|
+
if (!state) {
|
|
2490
|
+
state = { spokenLen: 0, pending: Promise.resolve() };
|
|
2491
|
+
messageStreams.set(messageId, state);
|
|
2492
|
+
}
|
|
2493
|
+
|
|
2494
|
+
const newText = fullText.slice(state.spokenLen);
|
|
2495
|
+
if (!newText) return;
|
|
2496
|
+
|
|
2497
|
+
let speakUpTo: number;
|
|
2498
|
+
if (isFinal) {
|
|
2499
|
+
// Flush everything remaining.
|
|
2500
|
+
speakUpTo = newText.length;
|
|
2501
|
+
} else {
|
|
2502
|
+
// Prefer sentence boundary; fall back to clause boundary
|
|
2503
|
+
// if we've buffered > EAGER_SPEAK_CHARS without one.
|
|
2504
|
+
const boundary = lastSentenceEnd(newText, 0);
|
|
2505
|
+
if (boundary > 0) {
|
|
2506
|
+
speakUpTo = boundary;
|
|
2507
|
+
} else if (newText.length >= EAGER_SPEAK_CHARS) {
|
|
2508
|
+
const clause = lastClauseEnd(newText);
|
|
2509
|
+
if (clause <= 0) return; // not even a clause yet — wait
|
|
2510
|
+
speakUpTo = clause;
|
|
2511
|
+
} else {
|
|
2512
|
+
return; // wait for more text
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
|
|
2516
|
+
const chunk = newText.slice(0, speakUpTo).trim();
|
|
2517
|
+
state.spokenLen += speakUpTo;
|
|
2518
|
+
if (!chunk) return;
|
|
2519
|
+
voiceDebug("autoSpeak.streaming", { id: messageId, chars: chunk.length, isFinal });
|
|
2520
|
+
|
|
2521
|
+
// Strip code blocks / links / emojis / abbreviations to spoken form.
|
|
2522
|
+
const { prepareForSpeech } = await import("./voice/tts-text-filter");
|
|
2523
|
+
const prepared = prepareForSpeech(chunk, {
|
|
2524
|
+
maxChars: 2000,
|
|
2525
|
+
stripCodeBlocks: true,
|
|
2526
|
+
collapseLinks: true,
|
|
2527
|
+
});
|
|
2528
|
+
if (prepared.skipped || !prepared.text.trim()) return;
|
|
2529
|
+
|
|
2530
|
+
// Serialize per-message: chain onto pending so chunks play in
|
|
2531
|
+
// order, never overlapping for the same message.
|
|
2532
|
+
const text = prepared.text;
|
|
2533
|
+
state.pending = state.pending.then(async () => {
|
|
2534
|
+
try {
|
|
2535
|
+
if (ctx) await runSpeak(ctx, text);
|
|
2536
|
+
} catch {
|
|
2537
|
+
// Auto-speak failures are non-blocking.
|
|
2538
|
+
}
|
|
2539
|
+
});
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
// Diagnostic: stream log shows when message_update / message_end
|
|
2543
|
+
// actually fire (some Pi versions don't emit message_update
|
|
2544
|
+
// per-token). Lets us verify the streaming path is alive.
|
|
2545
|
+
const streamDiag = (s: string) => {
|
|
2546
|
+
try {
|
|
2547
|
+
const fs2 = require("node:fs") as typeof import("node:fs");
|
|
2548
|
+
fs2.appendFileSync("/tmp/pi-listen-stream.log", `[${new Date().toISOString()}] ${s}\n`);
|
|
2549
|
+
} catch {
|
|
2550
|
+
/* best-effort */
|
|
2551
|
+
}
|
|
2552
|
+
};
|
|
2553
|
+
let mu_count = 0;
|
|
2554
|
+
pi.on("message_update", async (event) => {
|
|
2555
|
+
const msg = (event as any)?.message;
|
|
2556
|
+
if (!msg || msg.role !== "assistant") return;
|
|
2557
|
+
const id = (msg.id as string) || "current";
|
|
2558
|
+
const fullText = extractAccumulatedText(msg);
|
|
2559
|
+
mu_count++;
|
|
2560
|
+
if (mu_count <= 5 || mu_count % 10 === 0) {
|
|
2561
|
+
streamDiag(`message_update #${mu_count} id=${id} chars=${fullText.length}`);
|
|
2562
|
+
}
|
|
2563
|
+
await maybeSpeakNew(id, fullText, false);
|
|
2564
|
+
});
|
|
2565
|
+
|
|
2566
|
+
pi.on("message_end", async (event) => {
|
|
2567
|
+
const msg = (event as any)?.message;
|
|
2568
|
+
if (!msg || msg.role !== "assistant") return;
|
|
2569
|
+
const id = (msg.id as string) || "current";
|
|
2570
|
+
const fullText = extractAccumulatedText(msg);
|
|
2571
|
+
streamDiag(`message_end id=${id} chars=${fullText.length} updates_seen=${mu_count}`);
|
|
2572
|
+
mu_count = 0;
|
|
2573
|
+
await maybeSpeakNew(id, fullText, true);
|
|
2574
|
+
// Drop the stream state once flushed — prevents unbounded growth.
|
|
2575
|
+
const state = messageStreams.get(id);
|
|
2576
|
+
if (state) {
|
|
2577
|
+
state.pending.finally(() => {
|
|
2578
|
+
if (messageStreams.get(id) === state) messageStreams.delete(id);
|
|
2579
|
+
});
|
|
2580
|
+
}
|
|
2581
|
+
});
|
|
2582
|
+
|
|
2583
|
+
// Legacy turn_end fallback — only relevant on Pi versions that
|
|
2584
|
+
// don't fire message_update yet. Tracked via lastAutoSpeakAt so
|
|
2585
|
+
// it doesn't double-speak when message_update already covered the
|
|
2586
|
+
// content.
|
|
2587
|
+
pi.on("turn_end", async (event, _evtCtx) => {
|
|
2588
|
+
if (!config.ttsEnabled || !config.ttsAutoSpeak) return;
|
|
2589
|
+
const message = (event as any)?.message;
|
|
2590
|
+
if (!message || message.role !== "assistant") return;
|
|
2591
|
+
const id = (message.id as string) || "current";
|
|
2592
|
+
// If message_update already drained this message's stream
|
|
2593
|
+
// state, the streaming path handled it — skip.
|
|
2594
|
+
const state = messageStreams.get(id);
|
|
2595
|
+
if (state) return;
|
|
2596
|
+
if (voiceState === "warmup" || voiceState === "recording" || voiceState === "finalizing") return;
|
|
2597
|
+
const now = Date.now();
|
|
2598
|
+
if (now - lastAutoSpeakAt < AUTO_SPEAK_RATE_LIMIT_MS) return;
|
|
2599
|
+
const text = extractAccumulatedText(message).trim();
|
|
2600
|
+
if (!text) return;
|
|
2601
|
+
const { prepareForSpeech } = await import("./voice/tts-text-filter");
|
|
2602
|
+
const prepared = prepareForSpeech(text, { maxChars: 2000, stripCodeBlocks: true, collapseLinks: true });
|
|
2603
|
+
if (prepared.skipped) return;
|
|
2604
|
+
lastAutoSpeakAt = now;
|
|
2605
|
+
try {
|
|
2606
|
+
if (ctx) await runSpeak(ctx, prepared.text);
|
|
2607
|
+
} catch {
|
|
2608
|
+
/* non-blocking */
|
|
2609
|
+
}
|
|
2610
|
+
});
|
|
2611
|
+
|
|
2612
|
+
// ─── /voice command ──────────────────────────────────────────────────────
|
|
2613
|
+
|
|
2614
|
+
pi.registerCommand("voice", {
|
|
2615
|
+
description: "Voice: /voice [on|off|stop|dictate|history|test|info|setup]",
|
|
2616
|
+
handler: async (args, cmdCtx) => {
|
|
2617
|
+
ctx = cmdCtx;
|
|
2618
|
+
const sub = (args || "").trim().toLowerCase();
|
|
2619
|
+
|
|
2620
|
+
if (sub === "on") {
|
|
2621
|
+
config.enabled = true;
|
|
2622
|
+
updateVoiceStatus();
|
|
2623
|
+
setupHoldToTalk();
|
|
2624
|
+
const backendInfo =
|
|
2625
|
+
config.backend === "local"
|
|
2626
|
+
? `Voice enabled (local model: ${config.localModel || "whisper-small"}).`
|
|
2627
|
+
: "Voice enabled (Deepgram streaming).";
|
|
2628
|
+
cmdCtx.ui.notify(
|
|
2629
|
+
[
|
|
2630
|
+
backendInfo,
|
|
2631
|
+
"",
|
|
2632
|
+
" Hold SPACE → release to transcribe",
|
|
2633
|
+
` ${toggleShortcutLabel} → toggle recording on/off`,
|
|
2634
|
+
" Quick SPACE tap → types a space (no voice)",
|
|
2635
|
+
" Escape × 2 → clear editor",
|
|
2636
|
+
"",
|
|
2637
|
+
" /voice-settings → open settings panel",
|
|
2638
|
+
" /voice dictate → continuous mode (no hold)",
|
|
2639
|
+
" /voice test → verify setup",
|
|
2640
|
+
"",
|
|
2641
|
+
" Say 'undo', 'clear', 'new line', 'period' during dictation",
|
|
2642
|
+
].join("\n"),
|
|
2643
|
+
"info"
|
|
2644
|
+
);
|
|
2645
|
+
return;
|
|
2646
|
+
}
|
|
2647
|
+
|
|
2648
|
+
if (sub === "off") {
|
|
2649
|
+
config.enabled = false;
|
|
2650
|
+
voiceCleanup();
|
|
2651
|
+
ctx.ui.setStatus("voice", undefined);
|
|
2652
|
+
cmdCtx.ui.notify("Voice disabled.", "info");
|
|
2653
|
+
return;
|
|
2654
|
+
}
|
|
2655
|
+
|
|
2656
|
+
if (sub === "stop") {
|
|
2657
|
+
if (dictationMode) {
|
|
2658
|
+
dictationMode = false;
|
|
2659
|
+
if (voiceState === "recording") {
|
|
2660
|
+
await stopVoiceRecording();
|
|
2661
|
+
}
|
|
2662
|
+
cmdCtx.ui.notify("Dictation mode stopped.", "info");
|
|
2663
|
+
} else if (voiceState === "recording") {
|
|
2664
|
+
await stopVoiceRecording();
|
|
2665
|
+
cmdCtx.ui.notify("Recording stopped and transcribed.", "info");
|
|
2666
|
+
} else if (voiceState === "warmup") {
|
|
2667
|
+
abortPreRecording();
|
|
2668
|
+
clearWarmupWidget();
|
|
2669
|
+
hideWidget();
|
|
2670
|
+
resetHoldState();
|
|
2671
|
+
setVoiceState("idle");
|
|
2672
|
+
cmdCtx.ui.notify("Warmup cancelled.", "info");
|
|
2673
|
+
} else {
|
|
2674
|
+
cmdCtx.ui.notify("No recording in progress.", "info");
|
|
2675
|
+
}
|
|
2676
|
+
return;
|
|
2677
|
+
}
|
|
2678
|
+
|
|
2679
|
+
// /voice dictate — continuous dictation mode
|
|
2680
|
+
|
|
2681
|
+
if (sub === "dictate") {
|
|
2682
|
+
if (!config.enabled) {
|
|
2683
|
+
cmdCtx.ui.notify("Voice disabled. Use /voice on", "warning");
|
|
2684
|
+
return;
|
|
2685
|
+
}
|
|
2686
|
+
if (dictationMode) {
|
|
2687
|
+
cmdCtx.ui.notify("Already in dictation mode. /voice stop to end.", "info");
|
|
2688
|
+
return;
|
|
2689
|
+
}
|
|
2690
|
+
dictationMode = true;
|
|
2691
|
+
editorTextBeforeVoice = ctx?.hasUI ? ctx.ui.getEditorText() || "" : "";
|
|
2692
|
+
const ok = await startVoiceRecording();
|
|
2693
|
+
if (ok) {
|
|
2694
|
+
cmdCtx.ui.notify(
|
|
2695
|
+
[
|
|
2696
|
+
"🎤 Continuous dictation mode active.",
|
|
2697
|
+
"",
|
|
2698
|
+
" Speak freely — no need to hold SPACE.",
|
|
2699
|
+
" /voice stop → finalize and stop",
|
|
2700
|
+
` ${toggleShortcutLabel} → also stops dictation`,
|
|
2701
|
+
].join("\n"),
|
|
2702
|
+
"info"
|
|
2703
|
+
);
|
|
2704
|
+
} else {
|
|
2705
|
+
dictationMode = false;
|
|
2706
|
+
cmdCtx.ui.notify("Failed to start dictation.", "error");
|
|
2707
|
+
}
|
|
2708
|
+
return;
|
|
2709
|
+
}
|
|
2710
|
+
|
|
2711
|
+
// /voice history — show recent transcriptions
|
|
2712
|
+
if (sub === "history") {
|
|
2713
|
+
if (recordingHistory.length === 0) {
|
|
2714
|
+
cmdCtx.ui.notify("No recording history yet.", "info");
|
|
2715
|
+
return;
|
|
2716
|
+
}
|
|
2717
|
+
const lines = ["📜 Recent transcriptions:", ""];
|
|
2718
|
+
const show = recordingHistory.slice(0, 20);
|
|
2719
|
+
for (const entry of show) {
|
|
2720
|
+
const time = new Date(entry.timestamp).toLocaleTimeString();
|
|
2721
|
+
const dur = entry.duration.toFixed(1);
|
|
2722
|
+
const preview = entry.text.slice(0, 60) + (entry.text.length > 60 ? "…" : "");
|
|
2723
|
+
lines.push(` ${time} (${dur}s): ${preview}`);
|
|
2724
|
+
}
|
|
2725
|
+
if (recordingHistory.length > 20) {
|
|
2726
|
+
lines.push(` … and ${recordingHistory.length - 20} more`);
|
|
2727
|
+
}
|
|
2728
|
+
cmdCtx.ui.notify(lines.join("\n"), "info");
|
|
2729
|
+
return;
|
|
2730
|
+
}
|
|
2731
|
+
|
|
2732
|
+
if (sub === "test") {
|
|
2733
|
+
cmdCtx.ui.notify("Testing voice setup…", "info");
|
|
2734
|
+
const isLocal = config.backend === "local";
|
|
2735
|
+
const dgKey = resolveDeepgramApiKey(config);
|
|
2736
|
+
const tool = detectAudioCaptureTool();
|
|
2737
|
+
|
|
2738
|
+
const lines = [
|
|
2739
|
+
"Voice diagnostics:",
|
|
2740
|
+
"",
|
|
2741
|
+
` Backend: ${isLocal ? "local" : "deepgram"}`,
|
|
2742
|
+
"",
|
|
2743
|
+
" Audio capture:",
|
|
2744
|
+
` tool: ${tool ? `${tool.name} (${tool.cmd})` : "NONE FOUND"}`,
|
|
2745
|
+
];
|
|
2746
|
+
if (!tool) {
|
|
2747
|
+
lines.push(" available: sox ✗ ffmpeg ✗ arecord ✗");
|
|
2748
|
+
lines.push(" install one: brew install sox (or ffmpeg)");
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
if (isLocal) {
|
|
2752
|
+
lines.push(` local model: ${config.localModel || "whisper-small"}`);
|
|
2753
|
+
lines.push(` local endpoint: ${config.localEndpoint || DEFAULT_LOCAL_ENDPOINT}`);
|
|
2754
|
+
} else {
|
|
2755
|
+
lines.push(` DEEPGRAM_API_KEY: ${dgKey ? "set (" + dgKey.slice(0, 8) + "…)" : "NOT SET"}`);
|
|
2756
|
+
}
|
|
2757
|
+
lines.push("");
|
|
2758
|
+
lines.push(" Config:");
|
|
2759
|
+
lines.push(` language: ${config.language}`);
|
|
2760
|
+
lines.push(` onboarding: ${config.onboarding.completed ? "complete" : "incomplete"}`);
|
|
2761
|
+
lines.push(` hold threshold: ${getHoldThresholdMs()}ms`);
|
|
2762
|
+
lines.push(` toggle shortcut: ${resolvedToggleShortcut}`);
|
|
2763
|
+
lines.push(` kitty protocol: ${kittyReleaseDetected ? "detected" : "not detected"}`);
|
|
2764
|
+
lines.push(` state: ${voiceState}`);
|
|
2765
|
+
|
|
2766
|
+
// Mic capture test using detected tool
|
|
2767
|
+
if (tool) {
|
|
2768
|
+
const testFile = path.join(os.tmpdir(), "pi-voice-test.wav");
|
|
2769
|
+
let testProc;
|
|
2770
|
+
if (tool.name === "sox") {
|
|
2771
|
+
testProc = spawn("rec", ["-q", "-r", "16000", "-c", "1", "-b", "16", "-d", "1", testFile], {
|
|
2772
|
+
stdio: "pipe",
|
|
2773
|
+
});
|
|
2774
|
+
} else if (tool.name === "ffmpeg") {
|
|
2775
|
+
const isMac = process.platform === "darwin";
|
|
2776
|
+
const isLinux = process.platform === "linux";
|
|
2777
|
+
let testInputArgs: string[];
|
|
2778
|
+
if (isMac) testInputArgs = ["-f", "avfoundation", "-i", ":default"];
|
|
2779
|
+
else if (isLinux) testInputArgs = ["-f", "pulse", "-i", "default"];
|
|
2780
|
+
else {
|
|
2781
|
+
const dshowDev = detectWindowsAudioDevice();
|
|
2782
|
+
testInputArgs = dshowDev
|
|
2783
|
+
? ["-f", "dshow", "-i", `audio=${dshowDev}`]
|
|
2784
|
+
: ["-f", "dshow", "-i", "audio=Microphone"];
|
|
2785
|
+
}
|
|
2786
|
+
const inputArgs = testInputArgs;
|
|
2787
|
+
testProc = spawn(
|
|
2788
|
+
"ffmpeg",
|
|
2789
|
+
[...inputArgs, "-t", "1", "-ar", "16000", "-ac", "1", "-y", "-loglevel", "error", testFile],
|
|
2790
|
+
{ stdio: "pipe" }
|
|
2791
|
+
);
|
|
2792
|
+
} else {
|
|
2793
|
+
testProc = spawn("arecord", ["-q", "-f", "S16_LE", "-r", "16000", "-c", "1", "-d", "1", testFile], {
|
|
2794
|
+
stdio: "pipe",
|
|
2795
|
+
});
|
|
2796
|
+
}
|
|
2797
|
+
testProc.on("error", () => {});
|
|
2798
|
+
await new Promise<void>((resolve) => {
|
|
2799
|
+
let resolved = false;
|
|
2800
|
+
const done = () => {
|
|
2801
|
+
if (!resolved) {
|
|
2802
|
+
resolved = true;
|
|
2803
|
+
resolve();
|
|
2804
|
+
}
|
|
2805
|
+
};
|
|
2806
|
+
testProc.on("close", done);
|
|
2807
|
+
setTimeout(() => {
|
|
2808
|
+
try {
|
|
2809
|
+
testProc.kill();
|
|
2810
|
+
} catch {}
|
|
2811
|
+
done();
|
|
2812
|
+
}, 3000);
|
|
2813
|
+
});
|
|
2814
|
+
if (fs.existsSync(testFile)) {
|
|
2815
|
+
const size = fs.statSync(testFile).size;
|
|
2816
|
+
lines.push(` mic capture: OK (${size} bytes via ${tool.name})`);
|
|
2817
|
+
try {
|
|
2818
|
+
fs.unlinkSync(testFile);
|
|
2819
|
+
} catch {}
|
|
2820
|
+
} else {
|
|
2821
|
+
lines.push(` mic capture: FAILED — ${tool.name} ran but no audio captured`);
|
|
2822
|
+
}
|
|
2823
|
+
} else {
|
|
2824
|
+
lines.push(" mic capture: skipped (no audio tool)");
|
|
2825
|
+
}
|
|
2826
|
+
|
|
2827
|
+
if (isLocal && config.localEndpoint) {
|
|
2828
|
+
// External local server connectivity check
|
|
2829
|
+
const serverCheck = await checkLocalServer(config.localEndpoint);
|
|
2830
|
+
if (serverCheck.ok) {
|
|
2831
|
+
lines.push(" local server: OK (reachable)");
|
|
2832
|
+
} else {
|
|
2833
|
+
lines.push(` local server: NOT REACHABLE — ${serverCheck.error || "connection refused"}`);
|
|
2834
|
+
}
|
|
2835
|
+
} else if (isLocal) {
|
|
2836
|
+
// In-process sherpa-onnx mode — check module availability
|
|
2837
|
+
try {
|
|
2838
|
+
const { initSherpa, isSherpaAvailable } = await import("./voice/sherpa-engine");
|
|
2839
|
+
if (!isSherpaAvailable()) await initSherpa();
|
|
2840
|
+
const { isSherpaAvailable: checkAgain, getSherpaError } = await import("./voice/sherpa-engine");
|
|
2841
|
+
if (checkAgain()) {
|
|
2842
|
+
lines.push(" sherpa-onnx: OK (in-process mode)");
|
|
2843
|
+
} else {
|
|
2844
|
+
lines.push(` sherpa-onnx: NOT AVAILABLE — ${getSherpaError() || "unknown"}`);
|
|
2845
|
+
}
|
|
2846
|
+
} catch (e: any) {
|
|
2847
|
+
lines.push(` sherpa-onnx: NOT AVAILABLE — ${e?.message || e}`);
|
|
2848
|
+
}
|
|
2849
|
+
} else if (dgKey) {
|
|
2850
|
+
// Deepgram API key validation
|
|
2851
|
+
try {
|
|
2852
|
+
const res = await fetch("https://api.deepgram.com/v1/projects", {
|
|
2853
|
+
method: "GET",
|
|
2854
|
+
headers: { Authorization: `Token ${dgKey}` },
|
|
2855
|
+
signal: AbortSignal.timeout(5000),
|
|
2856
|
+
});
|
|
2857
|
+
if (res.ok) {
|
|
2858
|
+
lines.push(" Deepgram API: OK (key validated)");
|
|
2859
|
+
} else if (res.status === 401 || res.status === 403) {
|
|
2860
|
+
lines.push(" Deepgram API: INVALID KEY — check your API key");
|
|
2861
|
+
} else {
|
|
2862
|
+
lines.push(` Deepgram API: ERROR (HTTP ${res.status})`);
|
|
2863
|
+
}
|
|
2864
|
+
} catch (err) {
|
|
2865
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2866
|
+
lines.push(` Deepgram API: UNREACHABLE — ${msg}`);
|
|
2867
|
+
}
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
// Summary
|
|
2871
|
+
lines.push("");
|
|
2872
|
+
let ready: boolean;
|
|
2873
|
+
if (isLocal && config.localEndpoint) {
|
|
2874
|
+
const serverOk = (await checkLocalServer(config.localEndpoint)).ok;
|
|
2875
|
+
ready = !!tool && serverOk;
|
|
2876
|
+
if (!tool) {
|
|
2877
|
+
lines.push(" Setup needed — install any one of:");
|
|
2878
|
+
lines.push(" brew install sox # macOS (recommended)");
|
|
2879
|
+
lines.push(" apt install sox # Linux");
|
|
2880
|
+
} else if (!serverOk) {
|
|
2881
|
+
lines.push(" Setup needed — start a local transcription server:");
|
|
2882
|
+
lines.push(" whisper.cpp: ./build/bin/whisper-server -m models/ggml-small.bin --port 8080");
|
|
2883
|
+
lines.push(" Or any OpenAI-compatible transcription server");
|
|
2884
|
+
} else {
|
|
2885
|
+
lines.push(" All checks passed — voice is ready!");
|
|
2886
|
+
lines.push(` Hold SPACE to record, or use ${toggleShortcutLabel} to toggle.`);
|
|
2887
|
+
}
|
|
2888
|
+
} else if (isLocal) {
|
|
2889
|
+
// In-process sherpa-onnx mode — no server needed
|
|
2890
|
+
ready = !!tool;
|
|
2891
|
+
if (!tool) {
|
|
2892
|
+
lines.push(" Setup needed — install any one of:");
|
|
2893
|
+
lines.push(" brew install sox # macOS (recommended)");
|
|
2894
|
+
lines.push(" apt install sox # Linux");
|
|
2895
|
+
} else {
|
|
2896
|
+
lines.push(" All checks passed — voice is ready (in-process sherpa-onnx)!");
|
|
2897
|
+
lines.push(` Hold SPACE to record, or use ${toggleShortcutLabel} to toggle.`);
|
|
2898
|
+
}
|
|
2899
|
+
} else {
|
|
2900
|
+
ready = !!dgKey && !!tool;
|
|
2901
|
+
if (!dgKey) {
|
|
2902
|
+
lines.push(" Setup needed:");
|
|
2903
|
+
lines.push(" 1. Get a free key → https://dpgr.am/pi-voice ($200 free credit)");
|
|
2904
|
+
lines.push(' 2. export DEEPGRAM_API_KEY="your-key" (add to ~/.zshrc)');
|
|
2905
|
+
lines.push(" 3. Or run /voice-settings to configure");
|
|
2906
|
+
} else if (!tool) {
|
|
2907
|
+
lines.push(" Setup needed — install any one of:");
|
|
2908
|
+
lines.push(" brew install sox # macOS (recommended)");
|
|
2909
|
+
lines.push(" brew install ffmpeg # macOS (alternative)");
|
|
2910
|
+
lines.push(" apt install sox # Linux");
|
|
2911
|
+
lines.push(" apt install ffmpeg # Linux (alternative)");
|
|
2912
|
+
lines.push(" choco install sox # Windows");
|
|
2913
|
+
} else {
|
|
2914
|
+
lines.push(" All checks passed — voice is ready!");
|
|
2915
|
+
lines.push(` Hold SPACE to record, or use ${toggleShortcutLabel} to toggle.`);
|
|
2916
|
+
}
|
|
2917
|
+
}
|
|
2918
|
+
|
|
2919
|
+
cmdCtx.ui.notify(lines.join("\n"), ready ? "info" : "warning");
|
|
2920
|
+
return;
|
|
2921
|
+
}
|
|
2922
|
+
|
|
2923
|
+
// /voice language, /voice setup, /voice info → open settings panel
|
|
2924
|
+
if (sub === "language" || sub === "lang" || sub.startsWith("language ") || sub.startsWith("lang ")) {
|
|
2925
|
+
await openSettingsPanel(cmdCtx);
|
|
2926
|
+
return;
|
|
2927
|
+
}
|
|
2928
|
+
|
|
2929
|
+
if (sub === "info" || sub === "setup" || sub === "reconfigure" || sub === "settings" || sub === "config") {
|
|
2930
|
+
await openSettingsPanel(cmdCtx);
|
|
2931
|
+
return;
|
|
2932
|
+
}
|
|
2933
|
+
|
|
2934
|
+
// Default: toggle
|
|
2935
|
+
config.enabled = !config.enabled;
|
|
2936
|
+
if (!config.enabled) {
|
|
2937
|
+
voiceCleanup();
|
|
2938
|
+
} else {
|
|
2939
|
+
setupHoldToTalk();
|
|
2940
|
+
}
|
|
2941
|
+
updateVoiceStatus();
|
|
2942
|
+
cmdCtx.ui.notify(`Voice ${config.enabled ? "enabled" : "disabled"}.`, "info");
|
|
2943
|
+
},
|
|
2944
|
+
});
|
|
2945
|
+
|
|
2946
|
+
// ─── /voice-setup → redirects to settings panel ─────────────────────────
|
|
2947
|
+
|
|
2948
|
+
pi.registerCommand("voice-setup", {
|
|
2949
|
+
description: "Open pi-listen settings panel",
|
|
2950
|
+
handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx),
|
|
2951
|
+
});
|
|
2952
|
+
|
|
2953
|
+
// ─── /voice-language → redirects to settings panel ───────────────────────
|
|
2954
|
+
|
|
2955
|
+
pi.registerCommand("voice-language", {
|
|
2956
|
+
description: "Open pi-listen settings to change language",
|
|
2957
|
+
handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx),
|
|
2958
|
+
});
|
|
2959
|
+
|
|
2960
|
+
// ─── /voice-help → v7.1 §11 keyboard / command reference ────────────────
|
|
2961
|
+
|
|
2962
|
+
pi.registerCommand("voice-help", {
|
|
2963
|
+
description: "Show pi-listen keyboard + command reference",
|
|
2964
|
+
handler: async (_args, cmdCtx) => openHelpOverlay(cmdCtx),
|
|
2965
|
+
});
|
|
2966
|
+
|
|
2967
|
+
async function openHelpOverlay(cmdCtx: ExtensionCommandContext): Promise<void> {
|
|
2968
|
+
if (!cmdCtx.hasUI) {
|
|
2969
|
+
cmdCtx.ui.notify("pi-listen: hold space=record · /voice-speak <text> · /voice-settings · /voice-help", "info");
|
|
2970
|
+
return;
|
|
2971
|
+
}
|
|
2972
|
+
const { HelpOverlay } = await import("./voice/ui-help-overlay");
|
|
2973
|
+
await cmdCtx.ui.custom<void>((_tui, theme, _kb, done) => new HelpOverlay({ theme }, done), {
|
|
2974
|
+
overlay: true,
|
|
2975
|
+
overlayOptions: { width: "70%", minWidth: 60, maxHeight: "80%", anchor: "center" },
|
|
2976
|
+
});
|
|
2977
|
+
}
|
|
2978
|
+
|
|
2979
|
+
// ─── Settings panel (shared handler) ────────────────────────────────────
|
|
2980
|
+
|
|
2981
|
+
async function openSettingsPanel(cmdCtx: ExtensionCommandContext, initialTab?: number) {
|
|
2982
|
+
ctx = cmdCtx;
|
|
2983
|
+
|
|
2984
|
+
const { detectDevice, getModelFitness, formatDeviceSummary } = await import("./voice/device");
|
|
2985
|
+
const { getDownloadedModels, deleteModel, ensureModelDownloaded } = await import("./voice/model-download");
|
|
2986
|
+
const { isSherpaAvailable, clearRecognizerCache } = await import("./voice/sherpa-engine");
|
|
2987
|
+
const { VoiceSettingsPanel } = await import("./voice/settings-panel");
|
|
2988
|
+
type PanelAction = import("./voice/settings-panel").PanelAction;
|
|
2989
|
+
const { LANGUAGES } = await import("./voice/onboarding");
|
|
2990
|
+
const { resolveDeepgramApiKey } = await import("./voice/deepgram");
|
|
2991
|
+
|
|
2992
|
+
const device = detectDevice();
|
|
2993
|
+
// Construct the panel inside the custom() callback so the host theme
|
|
2994
|
+
// is in scope. Without this the panel falls back to raw ANSI which
|
|
2995
|
+
// clashes with non-default themes (Catppuccin Mocha etc.).
|
|
2996
|
+
const panelDeps = {
|
|
2997
|
+
config,
|
|
2998
|
+
device,
|
|
2999
|
+
cwd: currentCwd,
|
|
3000
|
+
getModelFitness,
|
|
3001
|
+
getDownloadedModels,
|
|
3002
|
+
deleteModel,
|
|
3003
|
+
isSherpaAvailable,
|
|
3004
|
+
formatDeviceSummary,
|
|
3005
|
+
saveConfig: (cfg: VoiceConfig, scope: VoiceSettingsScope, cwd: string) => saveConfig(cfg, scope, cwd),
|
|
3006
|
+
clearRecognizerCache: () => {
|
|
3007
|
+
try {
|
|
3008
|
+
clearRecognizerCache();
|
|
3009
|
+
} catch {}
|
|
3010
|
+
},
|
|
3011
|
+
resolveApiKey: () => resolveDeepgramApiKey(config) ?? undefined,
|
|
3012
|
+
deepgramLanguages: LANGUAGES.map((l) => ({ name: l.name, code: l.code, popular: l.popular })),
|
|
3013
|
+
};
|
|
3014
|
+
|
|
3015
|
+
let panel!: InstanceType<typeof VoiceSettingsPanel>;
|
|
3016
|
+
const result = await cmdCtx.ui.custom<PanelAction>(
|
|
3017
|
+
(_tui, theme, _kb, done) => {
|
|
3018
|
+
panel = new VoiceSettingsPanel({ ...panelDeps, theme }, initialTab);
|
|
3019
|
+
panel.onClose = (action) => done(action);
|
|
3020
|
+
return panel;
|
|
3021
|
+
},
|
|
3022
|
+
{
|
|
3023
|
+
overlay: true,
|
|
3024
|
+
overlayOptions: {
|
|
3025
|
+
width: "70%",
|
|
3026
|
+
minWidth: 44,
|
|
3027
|
+
maxHeight: "80%",
|
|
3028
|
+
anchor: "center",
|
|
3029
|
+
},
|
|
3030
|
+
}
|
|
3031
|
+
);
|
|
3032
|
+
|
|
3033
|
+
// Post-close: handle the speak-test action by re-using the
|
|
3034
|
+
// /voice-speak-test command path. We do this AFTER the panel has
|
|
3035
|
+
// closed so the test sample plays without the picker overlay
|
|
3036
|
+
// interfering with the audio cue (Pi's terminal renderer paints
|
|
3037
|
+
// the panel as an overlay; closing it first gives a clean
|
|
3038
|
+
// playback experience).
|
|
3039
|
+
if (result?.type === "speak-test") {
|
|
3040
|
+
await runSpeak(cmdCtx, "The quick brown fox jumps over the lazy dog.", { forceEnabled: true });
|
|
3041
|
+
return;
|
|
3042
|
+
}
|
|
3043
|
+
|
|
3044
|
+
// Post-close: handle the TTS install action triggered by selecting
|
|
3045
|
+
// a not-yet-installed model in the Speak tab Model picker.
|
|
3046
|
+
// v7.1: surfaces progress through the new sticky install widget
|
|
3047
|
+
// (`tts-install-progress.ts`) keyed by modelId so concurrent
|
|
3048
|
+
// installs of different models coexist without slot collision.
|
|
3049
|
+
if (result?.type === "tts-install" && result.modelId) {
|
|
3050
|
+
const { ensureTtsModelInstalled, getTtsModel } = await import("./voice/tts-local-models");
|
|
3051
|
+
const model = getTtsModel(result.modelId);
|
|
3052
|
+
// runInstallWithWidget rethrows on failure (Codex v6 #2) so
|
|
3053
|
+
// callers like runSpeak can short-circuit. This call site is
|
|
3054
|
+
// terminal — no further work — so swallow the rethrow after
|
|
3055
|
+
// notifications were already emitted by the helper itself.
|
|
3056
|
+
try {
|
|
3057
|
+
await runInstallWithWidget(cmdCtx, model.id, model.name, model.sizeBytes ?? 0, ensureTtsModelInstalled);
|
|
3058
|
+
} catch {
|
|
3059
|
+
/* notify already emitted in runInstallWithWidget */
|
|
3060
|
+
}
|
|
3061
|
+
return;
|
|
3062
|
+
}
|
|
3063
|
+
|
|
3064
|
+
// Post-close: handle download action with full pre-checks + progress
|
|
3065
|
+
if (result?.type === "download" && result.modelId) {
|
|
3066
|
+
const model = LOCAL_MODELS.find((m) => m.id === result.modelId);
|
|
3067
|
+
if (model) {
|
|
3068
|
+
const { checkDownloadPrereqs, createProgressTracker, verifyDownload, formatBytes } = await import(
|
|
3069
|
+
"./voice/model-download"
|
|
3070
|
+
);
|
|
3071
|
+
const { initSherpa, isSherpaAvailable, getSherpaError } = await import("./voice/sherpa-engine");
|
|
3072
|
+
|
|
3073
|
+
// ── Step 1: Check sherpa-onnx dependency ──
|
|
3074
|
+
if (!isSherpaAvailable()) {
|
|
3075
|
+
cmdCtx.ui.notify("Initializing sherpa-onnx runtime…", "info");
|
|
3076
|
+
const ok = await initSherpa();
|
|
3077
|
+
if (!ok) {
|
|
3078
|
+
cmdCtx.ui.notify(
|
|
3079
|
+
[
|
|
3080
|
+
"sherpa-onnx is required for local models but failed to initialize.",
|
|
3081
|
+
`Error: ${getSherpaError() || "unknown"}`,
|
|
3082
|
+
"",
|
|
3083
|
+
"To fix:",
|
|
3084
|
+
" 1. Ensure sherpa-onnx-node is installed: bun add sherpa-onnx-node",
|
|
3085
|
+
" 2. Check platform compatibility (macOS/Linux x64/arm64)",
|
|
3086
|
+
" 3. Or switch to Deepgram (cloud) backend in /voice-settings",
|
|
3087
|
+
].join("\n"),
|
|
3088
|
+
"error"
|
|
3089
|
+
);
|
|
3090
|
+
return;
|
|
3091
|
+
}
|
|
3092
|
+
}
|
|
3093
|
+
|
|
3094
|
+
// ── Step 2: Pre-download checks (disk, network, permissions) ──
|
|
3095
|
+
cmdCtx.ui.notify(`Checking prerequisites for ${model.name} (${model.size})…`, "info");
|
|
3096
|
+
const preCheck = await checkDownloadPrereqs(model.sherpaModel.downloadUrls, model.sizeBytes);
|
|
3097
|
+
if (!preCheck.ok) {
|
|
3098
|
+
cmdCtx.ui.notify(
|
|
3099
|
+
[
|
|
3100
|
+
`Cannot download ${model.name}:`,
|
|
3101
|
+
"",
|
|
3102
|
+
...preCheck.issues.map((i) => ` • ${i}`),
|
|
3103
|
+
"",
|
|
3104
|
+
"Resolve the above and try again via /voice-models.",
|
|
3105
|
+
].join("\n"),
|
|
3106
|
+
"error"
|
|
3107
|
+
);
|
|
3108
|
+
return;
|
|
3109
|
+
}
|
|
3110
|
+
|
|
3111
|
+
// ── Step 3: Download with real-time progress ──
|
|
3112
|
+
const tracker = createProgressTracker(model.name);
|
|
3113
|
+
cmdCtx.ui.notify(`Starting download: ${model.name} (${model.size})…`, "info");
|
|
3114
|
+
|
|
3115
|
+
try {
|
|
3116
|
+
await ensureModelDownloaded(model.id, model.sherpaModel.downloadUrls, model.sizeBytes, (raw) => {
|
|
3117
|
+
const rich = tracker(raw);
|
|
3118
|
+
if (rich) cmdCtx.ui.notify(rich.line, "info");
|
|
3119
|
+
});
|
|
3120
|
+
} catch (err: any) {
|
|
3121
|
+
const msg = err?.message || String(err);
|
|
3122
|
+
const lines = [`Download failed: ${model.name}`];
|
|
3123
|
+
if (msg.includes("timed out") || msg.includes("Timeout")) {
|
|
3124
|
+
lines.push("The download timed out. Check your internet speed and try again.");
|
|
3125
|
+
} else if (msg.includes("ENOSPC") || msg.includes("no space")) {
|
|
3126
|
+
lines.push("Disk is full. Free up space and try again.");
|
|
3127
|
+
} else if (msg.includes("HTTP 4") || msg.includes("HTTP 5")) {
|
|
3128
|
+
lines.push(`Server error: ${msg}`);
|
|
3129
|
+
lines.push("The model server may be temporarily down. Try again in a few minutes.");
|
|
3130
|
+
} else {
|
|
3131
|
+
lines.push(`Error: ${msg}`);
|
|
3132
|
+
}
|
|
3133
|
+
lines.push("", "Partial downloads are auto-resumed on next attempt.");
|
|
3134
|
+
cmdCtx.ui.notify(lines.join("\n"), "error");
|
|
3135
|
+
return;
|
|
3136
|
+
}
|
|
3137
|
+
|
|
3138
|
+
// ── Step 4: Post-download verification ──
|
|
3139
|
+
const verification = verifyDownload(model.id, model.sherpaModel.downloadUrls, model.sizeBytes);
|
|
3140
|
+
if (!verification.ok) {
|
|
3141
|
+
cmdCtx.ui.notify(
|
|
3142
|
+
[
|
|
3143
|
+
`${model.name} downloaded but verification failed:`,
|
|
3144
|
+
"",
|
|
3145
|
+
...verification.issues.map((i) => ` • ${i}`),
|
|
3146
|
+
"",
|
|
3147
|
+
"Try: /voice-models → Downloaded tab → delete and re-download.",
|
|
3148
|
+
].join("\n"),
|
|
3149
|
+
"warning"
|
|
3150
|
+
);
|
|
3151
|
+
return;
|
|
3152
|
+
}
|
|
3153
|
+
|
|
3154
|
+
cmdCtx.ui.notify(`${model.name} downloaded and verified (${model.size}). Ready to use.`, "info");
|
|
3155
|
+
}
|
|
3156
|
+
}
|
|
3157
|
+
|
|
3158
|
+
// Sync voice state after panel changes
|
|
3159
|
+
if (config.enabled) {
|
|
3160
|
+
setupHoldToTalk();
|
|
3161
|
+
} else {
|
|
3162
|
+
voiceCleanup();
|
|
3163
|
+
}
|
|
3164
|
+
updateVoiceStatus();
|
|
3165
|
+
}
|
|
3166
|
+
|
|
3167
|
+
// ─── TTS commands (v6.0.0+) ─────────────────────────────────────────
|
|
3168
|
+
//
|
|
3169
|
+
// The active speech AbortController lives at extension scope so
|
|
3170
|
+
// `/voice-speak-stop` can cancel whatever is currently playing.
|
|
3171
|
+
// Re-entrant `/voice-speak` calls abort the prior one before starting
|
|
3172
|
+
// (no overlapping audio); `null` means nothing is in-flight.
|
|
3173
|
+
let activeSpeak: AbortController | null = null;
|
|
3174
|
+
|
|
3175
|
+
function abortActiveSpeak(): boolean {
|
|
3176
|
+
if (!activeSpeak) return false;
|
|
3177
|
+
try {
|
|
3178
|
+
activeSpeak.abort();
|
|
3179
|
+
} catch {}
|
|
3180
|
+
activeSpeak = null;
|
|
3181
|
+
return true;
|
|
3182
|
+
}
|
|
3183
|
+
|
|
3184
|
+
/**
|
|
3185
|
+
* v7.1: run an install with the new sticky `TtsInstallProgressWidget`.
|
|
3186
|
+
* Replaces the v7.0.x notify-spam loop. Mounts a per-model-id slot
|
|
3187
|
+
* (`installWidgetKey(modelId)`) so two concurrent installs for
|
|
3188
|
+
* different models coexist without clobbering. The widget owns its
|
|
3189
|
+
* own AbortController (currently abort-only via cancel()); the
|
|
3190
|
+
* existing in-flight Map in `ensureTtsModelInstalled` serializes
|
|
3191
|
+
* same-id calls.
|
|
3192
|
+
*/
|
|
3193
|
+
async function runInstallWithWidget(
|
|
3194
|
+
cmdCtx: ExtensionCommandContext | ExtensionContext,
|
|
3195
|
+
modelId: string,
|
|
3196
|
+
modelName: string,
|
|
3197
|
+
totalBytesEstimate: number,
|
|
3198
|
+
ensureTtsModelInstalled: (
|
|
3199
|
+
id: string,
|
|
3200
|
+
opts: { signal?: AbortSignal; onProgress?: (info: any) => void }
|
|
3201
|
+
) => Promise<unknown>,
|
|
3202
|
+
// godspeed architect finding: accept caller signal so
|
|
3203
|
+
// /voice-speak-stop or TTS-disable propagates into the install.
|
|
3204
|
+
// Caller's signal cascades: aborting it triggers our own
|
|
3205
|
+
// AbortController and tears down the widget cleanly.
|
|
3206
|
+
callerSignal?: AbortSignal
|
|
3207
|
+
): Promise<void> {
|
|
3208
|
+
if (!cmdCtx.hasUI) {
|
|
3209
|
+
// Headless / scripted mode — fall back to a single notify so
|
|
3210
|
+
// users running pi without a TUI still get progress feedback.
|
|
3211
|
+
cmdCtx.ui.notify(`Installing ${modelName}…`, "info");
|
|
3212
|
+
try {
|
|
3213
|
+
await ensureTtsModelInstalled(modelId, {});
|
|
3214
|
+
cmdCtx.ui.notify(`${modelName} ready.`, "info");
|
|
3215
|
+
} catch (err: any) {
|
|
3216
|
+
// Codex v6.5: rethrow so callers like runSpeak short-
|
|
3217
|
+
// circuit instead of proceeding with a model that
|
|
3218
|
+
// isn't installed. Match the TUI branch's
|
|
3219
|
+
// `__alreadyNotified` contract so the outer catch
|
|
3220
|
+
// doesn't emit a duplicate notify.
|
|
3221
|
+
if (err?.name === "AbortError") {
|
|
3222
|
+
cmdCtx.ui.notify(`Install cancelled: ${modelName}`, "warning");
|
|
3223
|
+
} else {
|
|
3224
|
+
cmdCtx.ui.notify(`Install failed: ${err?.message ?? err}`, "error");
|
|
3225
|
+
}
|
|
3226
|
+
if (err && typeof err === "object") {
|
|
3227
|
+
try {
|
|
3228
|
+
(err as any).__alreadyNotified = true;
|
|
3229
|
+
} catch {
|
|
3230
|
+
/* frozen errors */
|
|
3231
|
+
}
|
|
3232
|
+
}
|
|
3233
|
+
throw err;
|
|
3234
|
+
}
|
|
3235
|
+
return;
|
|
3236
|
+
}
|
|
3237
|
+
|
|
3238
|
+
const { registry, ticker } = getOrInitVoiceUi();
|
|
3239
|
+
const controller = new AbortController();
|
|
3240
|
+
// Cascade caller signal → controller. If runSpeak's activeSpeak
|
|
3241
|
+
// signal aborts (user runs /voice-speak-stop or disables TTS),
|
|
3242
|
+
// the install cancels too. Listener removed in finally.
|
|
3243
|
+
let callerAbortListener: (() => void) | null = null;
|
|
3244
|
+
if (callerSignal) {
|
|
3245
|
+
if (callerSignal.aborted) {
|
|
3246
|
+
try {
|
|
3247
|
+
controller.abort();
|
|
3248
|
+
} catch {}
|
|
3249
|
+
} else {
|
|
3250
|
+
callerAbortListener = () => {
|
|
3251
|
+
try {
|
|
3252
|
+
controller.abort();
|
|
3253
|
+
} catch {}
|
|
3254
|
+
};
|
|
3255
|
+
callerSignal.addEventListener("abort", callerAbortListener);
|
|
3256
|
+
}
|
|
3257
|
+
}
|
|
3258
|
+
const widget = new TtsInstallProgressWidget({
|
|
3259
|
+
ui: cmdCtx.ui,
|
|
3260
|
+
modelId,
|
|
3261
|
+
modelName,
|
|
3262
|
+
totalBytesEstimate,
|
|
3263
|
+
registry,
|
|
3264
|
+
ticker,
|
|
3265
|
+
controller,
|
|
3266
|
+
});
|
|
3267
|
+
activeInstallWidgets.set(modelId, widget);
|
|
3268
|
+
try {
|
|
3269
|
+
await ensureTtsModelInstalled(modelId, {
|
|
3270
|
+
signal: controller.signal,
|
|
3271
|
+
onProgress: (info) => widget.onProgress(info),
|
|
3272
|
+
});
|
|
3273
|
+
// Widget self-disposes on phase=done; if ensure resolved
|
|
3274
|
+
// without firing done (very unlikely), make sure cleanup
|
|
3275
|
+
// runs anyway.
|
|
3276
|
+
widget.dispose();
|
|
3277
|
+
cmdCtx.ui.notify(`${modelName} ready.`, "info");
|
|
3278
|
+
} catch (err: any) {
|
|
3279
|
+
widget.dispose();
|
|
3280
|
+
// Codex v6 finding #2: rethrow so the caller (e.g. runSpeak)
|
|
3281
|
+
// can short-circuit instead of proceeding into speak() with
|
|
3282
|
+
// a model that isn't installed. We notify once here with the
|
|
3283
|
+
// install context; the rethrown error will be caught by the
|
|
3284
|
+
// caller's outer try/catch but tagged with `__alreadyNotified`
|
|
3285
|
+
// so the caller can skip its generic notify.
|
|
3286
|
+
if (err?.name === "AbortError") {
|
|
3287
|
+
cmdCtx.ui.notify(`Install cancelled: ${modelName}`, "warning");
|
|
3288
|
+
} else {
|
|
3289
|
+
cmdCtx.ui.notify(`Install failed: ${err?.message ?? err}`, "error");
|
|
3290
|
+
}
|
|
3291
|
+
if (err && typeof err === "object") {
|
|
3292
|
+
try {
|
|
3293
|
+
(err as any).__alreadyNotified = true;
|
|
3294
|
+
} catch {
|
|
3295
|
+
/* frozen errors */
|
|
3296
|
+
}
|
|
3297
|
+
}
|
|
3298
|
+
throw err;
|
|
3299
|
+
} finally {
|
|
3300
|
+
// Codex v6 finding #4: owner-checked delete so an older
|
|
3301
|
+
// finally cannot evict a newer same-id widget. (The
|
|
3302
|
+
// in-flight Map in ensureTtsModelInstalled already serializes
|
|
3303
|
+
// same-id installs, but the side-table here doesn't piggy-
|
|
3304
|
+
// back on that guarantee — defensive owner check.)
|
|
3305
|
+
if (activeInstallWidgets.get(modelId) === widget) {
|
|
3306
|
+
activeInstallWidgets.delete(modelId);
|
|
3307
|
+
}
|
|
3308
|
+
// Always remove the caller-signal listener.
|
|
3309
|
+
if (callerAbortListener && callerSignal) {
|
|
3310
|
+
try {
|
|
3311
|
+
callerSignal.removeEventListener("abort", callerAbortListener);
|
|
3312
|
+
} catch {}
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
}
|
|
3316
|
+
|
|
3317
|
+
async function runSpeak(
|
|
3318
|
+
cmdCtx: ExtensionCommandContext | ExtensionContext,
|
|
3319
|
+
text: string,
|
|
3320
|
+
opts: { forceEnabled?: boolean } = {}
|
|
3321
|
+
): Promise<void> {
|
|
3322
|
+
// `forceEnabled` lets /voice-speak-test bypass the gate without
|
|
3323
|
+
// mutating shared config. The previous mutate-snapshot-restore
|
|
3324
|
+
// pattern raced against /voice-speak-toggle and could clobber the
|
|
3325
|
+
// user's explicit toggle.
|
|
3326
|
+
if (!config.ttsEnabled && !opts.forceEnabled) {
|
|
3327
|
+
cmdCtx.ui.notify("TTS is disabled. Enable in /voice-settings.", "warning");
|
|
3328
|
+
return;
|
|
3329
|
+
}
|
|
3330
|
+
if (voiceState === "recording" || voiceState === "finalizing") {
|
|
3331
|
+
// Speaking while the mic is hot would feedback into STT.
|
|
3332
|
+
cmdCtx.ui.notify("Cannot speak while recording. Stop recording first.", "warning");
|
|
3333
|
+
return;
|
|
3334
|
+
}
|
|
3335
|
+
|
|
3336
|
+
// Cancel any in-flight speech so the new request takes the floor.
|
|
3337
|
+
abortActiveSpeak();
|
|
3338
|
+
const controller = new AbortController();
|
|
3339
|
+
activeSpeak = controller;
|
|
3340
|
+
|
|
3341
|
+
try {
|
|
3342
|
+
const { speak } = await import("./voice/speak");
|
|
3343
|
+
const { getInstalledTtsModelDir, ensureTtsModelInstalled, getTtsModel } = await import(
|
|
3344
|
+
"./voice/tts-local-models"
|
|
3345
|
+
);
|
|
3346
|
+
|
|
3347
|
+
// On the local backend, fetch the model on-demand if missing.
|
|
3348
|
+
// Deepgram backend skips this branch entirely. v7.1: surface
|
|
3349
|
+
// progress through the sticky install widget instead of the
|
|
3350
|
+
// v7.0.x notify-spam loop.
|
|
3351
|
+
if ((config.ttsBackend ?? "local") === "local") {
|
|
3352
|
+
const modelId = config.ttsLocalModel || "kitten-nano-en-v0_2";
|
|
3353
|
+
try {
|
|
3354
|
+
getInstalledTtsModelDir(modelId);
|
|
3355
|
+
} catch {
|
|
3356
|
+
const model = getTtsModel(modelId);
|
|
3357
|
+
await runInstallWithWidget(
|
|
3358
|
+
cmdCtx,
|
|
3359
|
+
modelId,
|
|
3360
|
+
model.name,
|
|
3361
|
+
model.sizeBytes ?? 0,
|
|
3362
|
+
ensureTtsModelInstalled,
|
|
3363
|
+
controller.signal
|
|
3364
|
+
);
|
|
3365
|
+
}
|
|
3366
|
+
}
|
|
3367
|
+
|
|
3368
|
+
// v7.1: mount the honest playback indicator (§6 of plan).
|
|
3369
|
+
// Spinner + state word with no fake amplitude meter. Until
|
|
3370
|
+
// `speak()` exposes a phase callback (v7.2), the indicator
|
|
3371
|
+
// stays on "playing" for the whole synth+play cycle —
|
|
3372
|
+
// honest because audio IS in flight throughout. Disposed
|
|
3373
|
+
// in finally regardless of success/abort/error. Tracked
|
|
3374
|
+
// on `activePlaybackIndicator` so the [esc] router can
|
|
3375
|
+
// stop playback when no install widget owns escape.
|
|
3376
|
+
let indicator: TtsPlaybackIndicator | null = null;
|
|
3377
|
+
if (cmdCtx.hasUI) {
|
|
3378
|
+
const { registry, ticker } = getOrInitVoiceUi();
|
|
3379
|
+
indicator = new TtsPlaybackIndicator({
|
|
3380
|
+
ui: cmdCtx.ui,
|
|
3381
|
+
registry,
|
|
3382
|
+
ticker,
|
|
3383
|
+
onStop: () => abortActiveSpeak(),
|
|
3384
|
+
});
|
|
3385
|
+
indicator.setState("playing");
|
|
3386
|
+
activePlaybackIndicator = indicator;
|
|
3387
|
+
}
|
|
3388
|
+
try {
|
|
3389
|
+
await speak({
|
|
3390
|
+
text,
|
|
3391
|
+
config,
|
|
3392
|
+
signal: controller.signal,
|
|
3393
|
+
resolveModelDir: (id) => getInstalledTtsModelDir(id),
|
|
3394
|
+
});
|
|
3395
|
+
} finally {
|
|
3396
|
+
indicator?.setState("idle"); // self-disposes
|
|
3397
|
+
// Owner-checked clear, mirroring runInstallWithWidget's
|
|
3398
|
+
// Codex v6 #4 fix.
|
|
3399
|
+
if (activePlaybackIndicator === indicator) activePlaybackIndicator = null;
|
|
3400
|
+
}
|
|
3401
|
+
} catch (err: any) {
|
|
3402
|
+
if (err?.name === "AbortError") return;
|
|
3403
|
+
// If the install widget already notified the user with a
|
|
3404
|
+
// scoped message ("Install failed: <model>"), don't emit a
|
|
3405
|
+
// duplicate generic "Speak failed:" notify. The install
|
|
3406
|
+
// widget's notify is more informative for that path.
|
|
3407
|
+
if (!err?.__alreadyNotified) {
|
|
3408
|
+
cmdCtx.ui.notify(`Speak failed: ${err?.message ?? err}`, "error");
|
|
3409
|
+
}
|
|
3410
|
+
} finally {
|
|
3411
|
+
if (activeSpeak === controller) activeSpeak = null;
|
|
3412
|
+
}
|
|
3413
|
+
}
|
|
3414
|
+
|
|
3415
|
+
pi.registerCommand("voice-speak", {
|
|
3416
|
+
description: "Speak the given text (text-to-speech)",
|
|
3417
|
+
handler: async (args, cmdCtx) => {
|
|
3418
|
+
ctx = cmdCtx;
|
|
3419
|
+
const text = (args || "").trim();
|
|
3420
|
+
if (!text) {
|
|
3421
|
+
cmdCtx.ui.notify("Usage: /voice-speak <text>", "warning");
|
|
3422
|
+
return;
|
|
3423
|
+
}
|
|
3424
|
+
await runSpeak(cmdCtx, text);
|
|
3425
|
+
},
|
|
3426
|
+
});
|
|
3427
|
+
|
|
3428
|
+
// v7.1.3 — toggle Deepgram WebSocket streaming TTS (cloud backend).
|
|
3429
|
+
// When ON, /voice-speak uses wss://api.deepgram.com/v1/speak so audio
|
|
3430
|
+
// frames stream into the local player as they arrive (sub-200ms
|
|
3431
|
+
// TTFA in good network conditions). When OFF, the REST `/v1/speak`
|
|
3432
|
+
// path returns a complete WAV.
|
|
3433
|
+
pi.registerCommand("voice-stream", {
|
|
3434
|
+
description: "Toggle Deepgram WebSocket streaming TTS (cloud)",
|
|
3435
|
+
handler: async (args, cmdCtx) => {
|
|
3436
|
+
ctx = cmdCtx;
|
|
3437
|
+
const trimmed = (args || "").trim().toLowerCase();
|
|
3438
|
+
let next: boolean;
|
|
3439
|
+
if (trimmed === "on") next = true;
|
|
3440
|
+
else if (trimmed === "off") next = false;
|
|
3441
|
+
else next = !(config.ttsDeepgramStreaming === true);
|
|
3442
|
+
config.ttsDeepgramStreaming = next;
|
|
3443
|
+
saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
|
|
3444
|
+
cmdCtx.ui.notify(`Deepgram WebSocket streaming TTS: ${next ? "ON" : "OFF"}`, "info");
|
|
3445
|
+
},
|
|
3446
|
+
});
|
|
3447
|
+
|
|
3448
|
+
// v7.1.3 — tune the hold-to-talk activation delay.
|
|
3449
|
+
pi.registerCommand("voice-hold-delay", {
|
|
3450
|
+
description: "Set hold-to-talk delay in ms (200-3000, default 700)",
|
|
3451
|
+
handler: async (args, cmdCtx) => {
|
|
3452
|
+
ctx = cmdCtx;
|
|
3453
|
+
const trimmed = (args || "").trim();
|
|
3454
|
+
if (!trimmed) {
|
|
3455
|
+
cmdCtx.ui.notify(`Hold delay: ${getHoldThresholdMs()}ms (default 700)`, "info");
|
|
3456
|
+
return;
|
|
3457
|
+
}
|
|
3458
|
+
const ms = parseInt(trimmed, 10);
|
|
3459
|
+
if (!Number.isFinite(ms) || ms < 200 || ms > 3000) {
|
|
3460
|
+
cmdCtx.ui.notify(`Invalid value: ${trimmed}. Must be 200-3000 ms.`, "warning");
|
|
3461
|
+
return;
|
|
3462
|
+
}
|
|
3463
|
+
(config as any).holdThresholdMs = ms;
|
|
3464
|
+
saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
|
|
3465
|
+
cmdCtx.ui.notify(`Hold delay set to ${ms}ms.`, "info");
|
|
3466
|
+
},
|
|
3467
|
+
});
|
|
3468
|
+
|
|
3469
|
+
// v7.1.1 — toggle auto-submit on STT (sends transcribed text
|
|
3470
|
+
// directly to the agent instead of just placing it in the editor).
|
|
3471
|
+
pi.registerCommand("voice-autosubmit", {
|
|
3472
|
+
description: "Toggle auto-submit on STT — sends spoken text to the agent immediately",
|
|
3473
|
+
handler: async (args, cmdCtx) => {
|
|
3474
|
+
ctx = cmdCtx;
|
|
3475
|
+
const trimmed = (args || "").trim().toLowerCase();
|
|
3476
|
+
let next: boolean;
|
|
3477
|
+
if (trimmed === "on") next = true;
|
|
3478
|
+
else if (trimmed === "off") next = false;
|
|
3479
|
+
else next = !(config.autoSubmitOnSpeak === true);
|
|
3480
|
+
config.autoSubmitOnSpeak = next;
|
|
3481
|
+
saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
|
|
3482
|
+
cmdCtx.ui.notify(`Auto-submit on speak: ${next ? "ON" : "OFF"}`, "info");
|
|
3483
|
+
},
|
|
3484
|
+
});
|
|
3485
|
+
|
|
3486
|
+
pi.registerCommand("voice-speak-stop", {
|
|
3487
|
+
description: "Stop in-flight TTS playback",
|
|
3488
|
+
handler: async (_args, cmdCtx) => {
|
|
3489
|
+
ctx = cmdCtx;
|
|
3490
|
+
if (abortActiveSpeak()) {
|
|
3491
|
+
cmdCtx.ui.notify("Speech stopped.", "info");
|
|
3492
|
+
} else {
|
|
3493
|
+
cmdCtx.ui.notify("No active speech.", "info");
|
|
3494
|
+
}
|
|
3495
|
+
},
|
|
3496
|
+
});
|
|
3497
|
+
|
|
3498
|
+
pi.registerCommand("voice-speak-toggle", {
|
|
3499
|
+
description: "Toggle TTS on/off (master switch)",
|
|
3500
|
+
handler: async (_args, cmdCtx) => {
|
|
3501
|
+
ctx = cmdCtx;
|
|
3502
|
+
const nowEnabling = !config.ttsEnabled;
|
|
3503
|
+
config.ttsEnabled = nowEnabling;
|
|
3504
|
+
saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
|
|
3505
|
+
if (!nowEnabling) abortActiveSpeak();
|
|
3506
|
+
cmdCtx.ui.notify(`TTS ${nowEnabling ? "enabled" : "disabled"}.`, "info");
|
|
3507
|
+
// First-time enable → v7.1 §9 rich onboarding overlay with
|
|
3508
|
+
// three explicit actions (try / pick model / skip).
|
|
3509
|
+
// Subsequent toggles are silent.
|
|
3510
|
+
if (nowEnabling && !(config as any).ttsOnboardingShown && cmdCtx.hasUI) {
|
|
3511
|
+
try {
|
|
3512
|
+
const { detectDevice } = await import("./voice/device");
|
|
3513
|
+
const { TtsOnboardingOverlay } = await import("./voice/tts-onboarding-overlay");
|
|
3514
|
+
const device = detectDevice();
|
|
3515
|
+
// §9: persist `ttsOnboardingShown = true` BEFORE any
|
|
3516
|
+
// async work so a failed install/cancel never re-prompts.
|
|
3517
|
+
(config as any).ttsOnboardingShown = true;
|
|
3518
|
+
saveConfig(config, config.scope === "project" ? "project" : "global", currentCwd);
|
|
3519
|
+
|
|
3520
|
+
const result = await cmdCtx.ui.custom<import("./voice/tts-onboarding-overlay").OnboardingResult>(
|
|
3521
|
+
(_tui, theme, _kb, done) => {
|
|
3522
|
+
return new TtsOnboardingOverlay({ systemLocale: device.systemLocale, theme }, done);
|
|
3523
|
+
},
|
|
3524
|
+
{
|
|
3525
|
+
overlay: true,
|
|
3526
|
+
overlayOptions: { width: "70%", minWidth: 60, maxHeight: "60%", anchor: "center" },
|
|
3527
|
+
}
|
|
3528
|
+
);
|
|
3529
|
+
if (result?.kind === "test") {
|
|
3530
|
+
await runSpeak(cmdCtx, "The quick brown fox jumps over the lazy dog.", { forceEnabled: true });
|
|
3531
|
+
} else if (result?.kind === "pickModel") {
|
|
3532
|
+
// Open the settings panel directly on the Speak tab.
|
|
3533
|
+
// Tab order is general/models/downloaded/speak/device → idx 3.
|
|
3534
|
+
await openSettingsPanel(cmdCtx, 3);
|
|
3535
|
+
}
|
|
3536
|
+
} catch (err) {
|
|
3537
|
+
voiceDebug("onboarding overlay threw", String(err));
|
|
3538
|
+
}
|
|
3539
|
+
} else if (nowEnabling && !cmdCtx.hasUI) {
|
|
3540
|
+
// Headless mode — fall back to the v7.0 notify-based hint.
|
|
3541
|
+
try {
|
|
3542
|
+
const { detectDevice } = await import("./voice/device");
|
|
3543
|
+
const { maybeShowTtsOnboarding } = await import("./voice/tts-onboarding");
|
|
3544
|
+
maybeShowTtsOnboarding({
|
|
3545
|
+
ctx: cmdCtx,
|
|
3546
|
+
config,
|
|
3547
|
+
device: detectDevice(),
|
|
3548
|
+
cwd: currentCwd,
|
|
3549
|
+
saveConfig: (cfg, scope, cwd) => saveConfig(cfg, scope, cwd),
|
|
3550
|
+
});
|
|
3551
|
+
} catch {
|
|
3552
|
+
/* onboarding hint is best-effort */
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
},
|
|
3556
|
+
});
|
|
3557
|
+
|
|
3558
|
+
pi.registerCommand("voice-speak-test", {
|
|
3559
|
+
description: "Synthesize a sample sentence in the current voice",
|
|
3560
|
+
handler: async (_args, cmdCtx) => {
|
|
3561
|
+
ctx = cmdCtx;
|
|
3562
|
+
// Pass forceEnabled so the test runs even when ttsEnabled is
|
|
3563
|
+
// false — without mutating shared config (a concurrent
|
|
3564
|
+
// /voice-speak-toggle would otherwise be clobbered when the
|
|
3565
|
+
// test's finally restored its snapshot).
|
|
3566
|
+
await runSpeak(cmdCtx, "The quick brown fox jumps over the lazy dog.", { forceEnabled: true });
|
|
3567
|
+
},
|
|
3568
|
+
});
|
|
3569
|
+
|
|
3570
|
+
pi.registerCommand("voice-speak-info", {
|
|
3571
|
+
description: "Show TTS configuration: backend, model, voice, install state",
|
|
3572
|
+
handler: async (_args, cmdCtx) => {
|
|
3573
|
+
ctx = cmdCtx;
|
|
3574
|
+
const { getTtsModel, isTtsModelInstalled, TTS_LOCAL_MODELS } = await import("./voice/tts-local-models");
|
|
3575
|
+
const { DEEPGRAM_TTS_VOICES } = await import("./voice/tts-deepgram");
|
|
3576
|
+
const { resolveDeepgramApiKey } = await import("./voice/deepgram");
|
|
3577
|
+
|
|
3578
|
+
const isLocal = (config.ttsBackend ?? "local") === "local";
|
|
3579
|
+
const lines: string[] = ["TTS configuration:", ""];
|
|
3580
|
+
lines.push(` Enabled: ${config.ttsEnabled ? "yes" : "no"}`);
|
|
3581
|
+
lines.push(` Backend: ${isLocal ? "local (sherpa-onnx)" : "deepgram (cloud REST)"}`);
|
|
3582
|
+
lines.push(` Language: ${config.ttsLanguage ?? config.language ?? "en"}`);
|
|
3583
|
+
lines.push(` Speed: ${(config.ttsSpeed ?? 1.0).toFixed(2)}x`);
|
|
3584
|
+
lines.push(` Auto-speak: ${config.ttsAutoSpeak ? "yes" : "no"}`);
|
|
3585
|
+
lines.push("");
|
|
3586
|
+
|
|
3587
|
+
if (isLocal) {
|
|
3588
|
+
const modelId = config.ttsLocalModel ?? "kitten-nano-en-v0_2";
|
|
3589
|
+
let model;
|
|
3590
|
+
try {
|
|
3591
|
+
model = getTtsModel(modelId);
|
|
3592
|
+
} catch {
|
|
3593
|
+
model = undefined;
|
|
3594
|
+
}
|
|
3595
|
+
const installed = isTtsModelInstalled(modelId);
|
|
3596
|
+
lines.push(" Local backend:");
|
|
3597
|
+
lines.push(` Model: ${modelId}${model ? ` (${model.name}, ${model.size})` : " — unknown id"}`);
|
|
3598
|
+
lines.push(` Installed: ${installed ? "yes" : "NO — first speak will download"}`);
|
|
3599
|
+
if (model) {
|
|
3600
|
+
const sid = typeof config.ttsLocalVoiceId === "number" ? config.ttsLocalVoiceId : model.defaultSid;
|
|
3601
|
+
const voice = model.voices.find((v) => v.sid === sid);
|
|
3602
|
+
lines.push(` Voice sid: ${sid}${voice ? ` (${voice.name})` : ""}`);
|
|
3603
|
+
lines.push(` Languages: ${model.languages.join(", ")}`);
|
|
3604
|
+
lines.push(` Sample rate: ${model.sampleRate} Hz`);
|
|
3605
|
+
}
|
|
3606
|
+
lines.push("");
|
|
3607
|
+
lines.push(` Catalog: ${TTS_LOCAL_MODELS.length} models available — /voice-speak-models to browse`);
|
|
3608
|
+
} else {
|
|
3609
|
+
const voiceId = config.ttsDeepgramVoiceId ?? "aura-asteria-en";
|
|
3610
|
+
const voice = DEEPGRAM_TTS_VOICES.find((v) => v.id === voiceId);
|
|
3611
|
+
const apiKey = resolveDeepgramApiKey(config);
|
|
3612
|
+
lines.push(" Deepgram backend:");
|
|
3613
|
+
lines.push(` Voice: ${voiceId}${voice ? ` (${voice.name})` : ""}`);
|
|
3614
|
+
lines.push(` API key: ${apiKey ? `set (${apiKey.slice(0, 8)}…)` : "NOT SET — set DEEPGRAM_API_KEY"}`);
|
|
3615
|
+
lines.push(` Catalog: ${DEEPGRAM_TTS_VOICES.length} Aura voices surfaced`);
|
|
3616
|
+
}
|
|
3617
|
+
|
|
3618
|
+
lines.push("");
|
|
3619
|
+
lines.push(" Commands: /voice-speak <text> · /voice-speak-stop · /voice-speak-toggle");
|
|
3620
|
+
lines.push(" /voice-speak-test · /voice-speak-models · /voice-settings");
|
|
3621
|
+
cmdCtx.ui.notify(lines.join("\n"), "info");
|
|
3622
|
+
},
|
|
3623
|
+
});
|
|
3624
|
+
|
|
3625
|
+
pi.registerCommand("voice-speak-models", {
|
|
3626
|
+
description: "Browse and install TTS models (opens settings panel on Speak tab)",
|
|
3627
|
+
// initialTab=3 → Speak tab (index 0=General, 1=Models, 2=Downloaded, 3=Speak, 4=Device)
|
|
3628
|
+
handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx, 3),
|
|
3629
|
+
});
|
|
3630
|
+
|
|
3631
|
+
// ─── /voice-settings — unified pi-listen settings panel ─────────────
|
|
3632
|
+
|
|
3633
|
+
pi.registerCommand("voice-settings", {
|
|
3634
|
+
description: "Open pi-listen settings — backend, models, language, device",
|
|
3635
|
+
handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx),
|
|
3636
|
+
});
|
|
3637
|
+
|
|
3638
|
+
// ─── /voice-models — opens settings panel on Models tab ─────────────
|
|
3639
|
+
|
|
3640
|
+
pi.registerCommand("voice-models", {
|
|
3641
|
+
description: "Manage local voice models (opens settings panel)",
|
|
3642
|
+
handler: async (_args, cmdCtx) => openSettingsPanel(cmdCtx, 1),
|
|
3643
|
+
});
|
|
3644
|
+
}
|