pi-voicekit 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +341 -0
  3. package/extensions/voice/config.ts +395 -0
  4. package/extensions/voice/deepgram.ts +33 -0
  5. package/extensions/voice/device.ts +382 -0
  6. package/extensions/voice/hold-to-talk.ts +69 -0
  7. package/extensions/voice/local.ts +1143 -0
  8. package/extensions/voice/model-download.ts +636 -0
  9. package/extensions/voice/onboarding.ts +739 -0
  10. package/extensions/voice/release-controller.ts +55 -0
  11. package/extensions/voice/settings-panel.ts +1602 -0
  12. package/extensions/voice/sherpa-engine.ts +464 -0
  13. package/extensions/voice/sherpa-loader.ts +143 -0
  14. package/extensions/voice/sherpa-onnx-node.d.ts +4 -0
  15. package/extensions/voice/speak.ts +430 -0
  16. package/extensions/voice/tts-deepgram.ts +454 -0
  17. package/extensions/voice/tts-engine.ts +653 -0
  18. package/extensions/voice/tts-install-progress.ts +257 -0
  19. package/extensions/voice/tts-local-models.ts +1255 -0
  20. package/extensions/voice/tts-onboarding-overlay.ts +186 -0
  21. package/extensions/voice/tts-onboarding.ts +87 -0
  22. package/extensions/voice/tts-playback-indicator.ts +127 -0
  23. package/extensions/voice/tts-playback.ts +675 -0
  24. package/extensions/voice/tts-text-filter.ts +404 -0
  25. package/extensions/voice/ui-aura.ts +272 -0
  26. package/extensions/voice/ui-help-overlay.ts +161 -0
  27. package/extensions/voice/ui-icons.ts +124 -0
  28. package/extensions/voice/ui-locale-labels.ts +110 -0
  29. package/extensions/voice/ui-picker.ts +209 -0
  30. package/extensions/voice/ui-render-ticker.ts +171 -0
  31. package/extensions/voice/ui-widget-base.ts +219 -0
  32. package/extensions/voice/ui-width.ts +112 -0
  33. package/extensions/voice.ts +3644 -0
  34. package/package.json +75 -0
@@ -0,0 +1,653 @@
1
+ /**
2
+ * Local TTS engine — wraps sherpa-onnx-node OfflineTts.
3
+ *
4
+ * Public API:
5
+ * - initTts() — load sherpa via the shared loader
6
+ * - synthesize(opts) — produce {samples, sampleRate} + abort
7
+ * - synthesizeById(opts) — synthesize using a model id from the catalog
8
+ * - clearTtsCache() — drop cached instance (called on model switch)
9
+ * - resolveLanguageForLocal — validate language ↔ local model match
10
+ * - validateTtsLanguageInput — non-empty/string language guard (both backends)
11
+ *
12
+ * Internal helpers:
13
+ * - getOrCreateTts(model, dir) — module-private cached OfflineTts instance
14
+ *
15
+ * ─── Concurrency contract (read this before adding "race condition" notes) ───
16
+ *
17
+ * This module assumes the standard Node.js / Bun single-threaded execution
18
+ * model. Synchronous statements (including `Map.get`/`Map.set`, property
19
+ * reads/writes, the `&&` short-circuit, and `return` of an already-resolved
20
+ * value) run to completion before any other code executes. There is NO
21
+ * preemption between two synchronous operations, no parallel JS threads,
22
+ * and no shared-memory data races on plain JS values.
23
+ *
24
+ * Async interleaving CAN happen at any `await` point. Every `await` in this
25
+ * module is followed by an identity-guard check (`inFlight.get(key) === …`)
26
+ * to detect "another caller took over while I was suspended" and to skip
27
+ * stale cache writes. Callers also pass an `AbortSignal` through
28
+ * `synthesize()` for cooperative cancellation.
29
+ *
30
+ * Sherpa-onnx-node native handles are reference-counted via N-API
31
+ * finalizers — they are released only when the JS reference count drops
32
+ * to zero. Since we keep a strong reference in `cachedTts` (or in a
33
+ * caller's local `cached` after `await getOrCreateTts(...)`), there is no
34
+ * use-after-free on the JS side. `clearTtsCache()` only drops the cache
35
+ * slot; any caller that already has a `CachedTts` reference still owns
36
+ * the underlying native handle until they drop it.
37
+ *
38
+ * Why a separate cache from STT: a user can have STT (Parakeet) and TTS
39
+ * (Kitten Nano) loaded simultaneously — they're different sherpa engine
40
+ * objects with different memory footprints, and the model-switch logic
41
+ * differs (STT swaps on language change; TTS swaps on model change).
42
+ */
43
+
44
+ import * as path from "node:path";
45
+ import * as os from "node:os";
46
+ import * as fs from "node:fs";
47
+ import { loadSherpa, getSherpaModule, getSherpaError, isSherpaAvailable } from "./sherpa-loader";
48
+ import { type TtsLocalModelInfo, modelSupportsLanguage, getTtsModel } from "./tts-local-models";
49
+
50
+ // ─── Types ────────────────────────────────────────────────────────────────────
51
+
52
+ /** Opaque OfflineTts handle from sherpa-onnx-node. */
53
+ type OfflineTts = any;
54
+
55
+ interface CachedTts {
56
+ modelId: string;
57
+ tts: OfflineTts;
58
+ sampleRate: number;
59
+ numSpeakers: number;
60
+ /**
61
+ * Per-instance synthesis serialization tail. Two concurrent
62
+ * `synthesize()` calls against the same OfflineTts handle would (a)
63
+ * produce overlapping audio at the playback layer (bad UX) and
64
+ * (b) traverse sherpa-onnx-node's native generate path twice in
65
+ * parallel — the JS binding likely serializes internally but we
66
+ * shouldn't depend on undocumented behavior. Each synthesize() awaits
67
+ * the previous one before starting; the chain unwinds in order.
68
+ *
69
+ * Failed/aborted runs do not block subsequent runs — the chain
70
+ * advances on settlement (resolve OR reject), so a single bad
71
+ * synthesize doesn't deadlock the queue.
72
+ */
73
+ generateChain: Promise<void>;
74
+ }
75
+
76
+ /** Result of a synthesize call. */
77
+ export interface TtsAudio {
78
+ /** PCM float samples in [-1, 1] interleaved mono. */
79
+ samples: Float32Array;
80
+ /** Sample rate in Hz (typically 22050 or 24000). */
81
+ sampleRate: number;
82
+ }
83
+
84
+ /** Synthesis input. `opts.signal` aborts via the OfflineTts onProgress callback. */
85
+ export interface SynthesizeOpts {
86
+ text: string;
87
+ model: TtsLocalModelInfo;
88
+ modelDir: string;
89
+ language: string;
90
+ /** Speaker id within the model. Default: model.defaultSid. */
91
+ sid?: number;
92
+ /** Speed multiplier; 1.0 = normal. Range 0.5–2.0. */
93
+ speed?: number;
94
+ /** Trailing silence between sentences (sherpa internal default = 0.5). */
95
+ silenceScale?: number;
96
+ /** Cancellation token. Triggering this returns 0 from onProgress and aborts the generate. */
97
+ signal?: AbortSignal;
98
+ /**
99
+ * Optional progress callback. Fires once per sentence with the partial
100
+ * float samples and progress ∈ [0,1]. Useful if the caller wants to
101
+ * stream playback in v6.1.
102
+ */
103
+ onProgress?: (chunk: { samples: Float32Array; progress: number }) => void;
104
+ }
105
+
106
+ // ─── State ────────────────────────────────────────────────────────────────────
107
+
108
+ /**
109
+ * Composite cache key. Two callers passing different `modelDir` for the
110
+ * same `modelId` (e.g. after a re-download to a versioned path) MUST get
111
+ * different OfflineTts instances — sherpa-onnx-node holds open file
112
+ * handles to the model files, and pointing at the wrong directory is
113
+ * undefined behaviour. Including `modelDir` in the key forces a rebuild
114
+ * when the on-disk location changes.
115
+ *
116
+ * Length-prefix the modelId so two distinct (modelId, modelDir) pairs
117
+ * can't collide via concatenation when the separator happens to appear
118
+ * inside modelId. Catalog ids are kebab-case today but the prefix is
119
+ * cheap belt-and-suspenders against future ids that include `|`.
120
+ */
121
+ function cacheKey(modelId: string, modelDir: string): string {
122
+ return `${modelId.length}|${modelId}|${modelDir}`;
123
+ }
124
+
125
+ /**
126
+ * Single source of truth for cached TTS instances. Stores resolved values
127
+ * for fast lookups AND in-flight construction promises for single-flight
128
+ * deduplication. Keys are `cacheKey(modelId, modelDir)`.
129
+ *
130
+ * Map values:
131
+ * - `Promise<CachedTts>` while construction is in-flight
132
+ * - `CachedTts` once construction has resolved
133
+ *
134
+ * Callers always wrap the lookup result in `Promise.resolve(hit)` —
135
+ * `Promise.resolve` returns a thenable's same promise unchanged and wraps
136
+ * a plain value in a one-microtask resolved promise, so a single return
137
+ * statement covers both states without an instanceof check.
138
+ *
139
+ * `clearTtsCache()` wipes the entire map. In-flight builds remain pending;
140
+ * their `.then(...)` handlers no-op against an empty map and the resulting
141
+ * native handles are dropped on the floor (GC reclaims them). This is the
142
+ * "fire and forget the in-flight build" trade-off — clearing the cache
143
+ * mid-build wastes the build's CPU but is otherwise harmless.
144
+ */
145
+ const ttsCache = new Map<string, CachedTts | Promise<CachedTts>>();
146
+
147
+ // ─── Initialization ──────────────────────────────────────────────────────────
148
+
149
+ /**
150
+ * Initialize TTS support. Routes through the shared sherpa loader so the
151
+ * native module is loaded exactly once per process even if STT was already
152
+ * initialized.
153
+ */
154
+ export async function initTts(): Promise<boolean> {
155
+ return loadSherpa();
156
+ }
157
+
158
+ export { isSherpaAvailable, getSherpaError };
159
+
160
+ // ─── Recognizer (TTS instance) management ────────────────────────────────────
161
+
162
+ /**
163
+ * Get or create an OfflineTts for `model`. Internal — public callers go
164
+ * through `synthesize()` / `synthesizeById()` instead, which keeps the
165
+ * sherpa-specific native handle behind the module boundary. Pulling the
166
+ * handle out of this module would couple every caller to sherpa lifecycle
167
+ * semantics and make swapping engines (or adding pooling) require a
168
+ * cross-package refactor.
169
+ *
170
+ * Concurrent calls share the same in-flight construction promise. After
171
+ * construction the cached instance is reused until either:
172
+ * - The user picks a different model (call clearTtsCache() then re-call)
173
+ * - The settings panel deletes a downloaded model (cache is cleared)
174
+ * - The session shuts down (clearTtsCache() runs in voiceCleanup)
175
+ *
176
+ * @throws if loadSherpa() failed (platform incompat) — caller should already
177
+ * have checked initTts() ok before calling.
178
+ */
179
+ function getOrCreateTts(model: TtsLocalModelInfo, modelDir: string): Promise<CachedTts> {
180
+ const key = cacheKey(model.id, modelDir);
181
+ const hit = ttsCache.get(key);
182
+ // Both branches return a Promise:
183
+ // - resolved CachedTts → wrapped in Promise.resolve (cheap, ~1 microtask)
184
+ // - in-flight Promise<CachedTts> → returned as-is
185
+ if (hit) return Promise.resolve(hit);
186
+
187
+ // Construct, store the in-flight promise immediately so concurrent
188
+ // same-key callers see-through it, and replace with the resolved
189
+ // CachedTts in the same map slot when construction settles. On
190
+ // failure, evict so the next caller gets a fresh attempt.
191
+ const pending = doCreateTts(model, modelDir).then(
192
+ (cached) => {
193
+ // Only swap in the resolved value if our pending promise is
194
+ // still the entry — clearTtsCache() may have wiped the map
195
+ // while we were building. In that case the freshly built
196
+ // instance is dropped on the floor (GC reclaims the native
197
+ // handle via the N-API finalizer).
198
+ if (ttsCache.get(key) === pending) {
199
+ ttsCache.set(key, cached);
200
+ }
201
+ return cached;
202
+ },
203
+ (err) => {
204
+ // Don't leave a permanently rejected promise stuck in the
205
+ // cache — the next caller should be allowed to retry.
206
+ if (ttsCache.get(key) === pending) {
207
+ ttsCache.delete(key);
208
+ }
209
+ throw err;
210
+ }
211
+ );
212
+ ttsCache.set(key, pending);
213
+ return pending;
214
+ }
215
+
216
+ async function doCreateTts(model: TtsLocalModelInfo, modelDir: string): Promise<CachedTts> {
217
+ const sherpa = getSherpaModule();
218
+ const config = buildTtsConfig(model, modelDir);
219
+ const tts = await sherpa.OfflineTts.createAsync(config);
220
+ return {
221
+ modelId: model.id,
222
+ tts,
223
+ // Some sherpa builds expose `sampleRate` directly on the handle;
224
+ // others require reading from the model. Prefer the handle and fall
225
+ // back to the catalog entry to keep this resilient across versions.
226
+ sampleRate: typeof tts.sampleRate === "number" ? tts.sampleRate : model.sampleRate,
227
+ numSpeakers: typeof tts.numSpeakers === "number" ? tts.numSpeakers : model.voices.length,
228
+ generateChain: Promise.resolve(),
229
+ };
230
+ }
231
+
232
+ /**
233
+ * Drop all cached OfflineTts instances. Same garbage-collection contract
234
+ * as the STT recognizer cache: sherpa-onnx-node has no `.dispose()` API;
235
+ * native resources are released via N-API finalizers when the JS reference
236
+ * count drops to zero. Clearing the map drops our last reference; any
237
+ * caller still holding a CachedTts via a previously-resolved promise keeps
238
+ * its native handle alive until they drop it too.
239
+ *
240
+ * In-flight construction promises check `ttsCache.get(key) === pending`
241
+ * before writing back, so a clear during build correctly causes the
242
+ * freshly-built instance to be dropped on the floor.
243
+ */
244
+ export function clearTtsCache(): void {
245
+ ttsCache.clear();
246
+ }
247
+
248
+ // ─── Warmup ──────────────────────────────────────────────────────────────────
249
+
250
+ /**
251
+ * Pre-load the sherpa-onnx module AND construct the OfflineTts for `model`
252
+ * in the background, so the user's first `/voice-speak` doesn't pay the
253
+ * 600-900ms cold-start init cost.
254
+ *
255
+ * Idempotent: subsequent calls for the same (model, modelDir) await the
256
+ * same in-flight promise via the existing `ttsCache` machinery — the only
257
+ * difference vs a real synthesize is that warmup discards the result.
258
+ *
259
+ * Cancellation: `signal` aborts the load. If the user toggles TTS off
260
+ * before warmup completes, the construction continues and the resulting
261
+ * instance lands in the cache (cheap memory cost), but no UI flicker
262
+ * happens — the cache is simply unused. Cleaner alternative would be to
263
+ * abort native createAsync but sherpa-onnx-node doesn't expose that.
264
+ *
265
+ * Errors: returns `false` on any failure (logged to debug output, not
266
+ * rethrown). Callers treat this as a best-effort optimization — failure
267
+ * here is not a user-facing error because the next /voice-speak will
268
+ * surface the same error anyway through synthesize().
269
+ */
270
+ export async function warmupTts(
271
+ model: TtsLocalModelInfo,
272
+ modelDir: string,
273
+ opts: { signal?: AbortSignal } = {}
274
+ ): Promise<boolean> {
275
+ if (opts.signal?.aborted) return false;
276
+ try {
277
+ const ok = await loadSherpa();
278
+ if (!ok) return false;
279
+ if (opts.signal?.aborted) return false;
280
+ await getOrCreateTts(model, modelDir);
281
+ return true;
282
+ } catch {
283
+ // Warmup is best-effort; swallow errors so callers never have to
284
+ // worry about a backgrounded promise rejection.
285
+ return false;
286
+ }
287
+ }
288
+
289
+ // ─── Synthesis ───────────────────────────────────────────────────────────────
290
+
291
+ /**
292
+ * Synthesize `text` to PCM samples using the given model.
293
+ *
294
+ * Validation order:
295
+ * 1. Language ↔ model compatibility (resolveLanguageForLocal)
296
+ * 2. Engine availability (loadSherpa must have succeeded)
297
+ * 3. Generate the audio (with optional abort + progress)
298
+ *
299
+ * The signal is wired through OfflineTts.generateAsync's onProgress: when
300
+ * AbortSignal fires we return 0 from the callback, which sherpa-onnx-node
301
+ * treats as "stop generating". The pending promise resolves with whatever
302
+ * samples were produced so the caller can still play the partial result.
303
+ */
304
+ export async function synthesize(opts: SynthesizeOpts): Promise<TtsAudio> {
305
+ const { text, model, modelDir, language, signal } = opts;
306
+
307
+ // 1. Language check first — synchronous, gives clear error before any
308
+ // model load or network work happens.
309
+ resolveLanguageForLocal(model, language);
310
+
311
+ // 2. Attach the abort listener BEFORE any await so a signal that fires
312
+ // during model load / engine construction is observed. If the signal
313
+ // is already aborted at entry, `signal.aborted` is true and the
314
+ // onProgress callback returns 0 on its first invocation.
315
+ let aborted = signal?.aborted === true;
316
+ const onAbort = () => {
317
+ aborted = true;
318
+ };
319
+ if (signal && !aborted) {
320
+ signal.addEventListener("abort", onAbort, { once: true });
321
+ }
322
+
323
+ try {
324
+ // Fast-path: caller passed an already-aborted signal. Bail without
325
+ // loading the engine — there's nothing to do.
326
+ if (aborted) {
327
+ throw makeAbortError();
328
+ }
329
+
330
+ // 3. Make sure the engine is loaded.
331
+ const ok = await loadSherpa();
332
+ if (aborted) throw makeAbortError();
333
+ if (!ok) {
334
+ throw new Error(`sherpa-onnx not available: ${getSherpaError() ?? "unknown error"}`);
335
+ }
336
+
337
+ // 4. Get or create the OfflineTts instance.
338
+ const cached = await getOrCreateTts(model, modelDir);
339
+ if (aborted) throw makeAbortError();
340
+
341
+ // v7.1.2 — refuse to use models we know are incompatible with
342
+ // the installed sherpa-onnx runtime. Prevents the user from
343
+ // hearing silence and wondering why; surfaces the incompat
344
+ // reason from the catalog so they can switch voices.
345
+ if (model.incompatible) {
346
+ throw new Error(
347
+ `TTS model ${model.id} is incompatible with the installed sherpa-onnx runtime: ${model.incompatible}`
348
+ );
349
+ }
350
+
351
+ // 5. Compute generate() args.
352
+ // We use the LEGACY generateAsync({ text, sid, speed }) shape
353
+ // because it works on every sherpa-onnx-node release we've
354
+ // tested (1.12.29 + 1.13.0). Two upstream bugs informed this:
355
+ // (a) generateAsync({ generationConfig }) threw
356
+ // "Not implemented yet. Only some models support this"
357
+ // on 1.12.29 (offline-tts-impl.h:Generate:38).
358
+ // FIXED in 1.13.0 (PRs #3362-#3365 in k2-fsa/sherpa-onnx).
359
+ // (b) generateAsync({ onProgress: cb }) crashes the Node
360
+ // process in napi_create_arraybuffer when the callback
361
+ // fires. Still broken on 1.13.0 — root cause is the
362
+ // binding invoking JS from a background C++ thread
363
+ // without napi_threadsafe_function.
364
+ // The legacy shape sidesteps both. Cost: we lose `silenceScale`
365
+ // (sherpa default is reasonable) and intra-synthesis progress
366
+ // callbacks (synthesis is fast enough — <1s for typical
367
+ // sentences — that the missing progress is invisible).
368
+ // Future: when bug (b) is fixed upstream, re-enable
369
+ // onProgress + GenerationConfig together for streaming UI.
370
+ const sid = clampSid(opts.sid ?? model.defaultSid, model);
371
+ const speed = clampSpeed(opts.speed ?? 1.0);
372
+
373
+ // 6. Serialize generates against this CachedTts instance via the
374
+ // chain extension below — see the `generateChain` doc on CachedTts.
375
+ const { previousTail, release: resolveOurTail } = extendChain(cached);
376
+ await previousTail.catch(() => {}); // ignore prior failure
377
+ if (aborted) {
378
+ resolveOurTail();
379
+ throw makeAbortError();
380
+ }
381
+
382
+ let audio: { samples: Float32Array; sampleRate?: number };
383
+ try {
384
+ audio = await cached.tts.generateAsync({ text, sid, speed });
385
+ } finally {
386
+ // Always release the chain so subsequent synthesize() calls
387
+ // can proceed, even if generateAsync threw.
388
+ resolveOurTail();
389
+ }
390
+ // v7.1.2 — guard against silent NaN-sample playback. Some
391
+ // sherpa-onnx-node + voices.bin combinations (notably kokoro
392
+ // multilingual sid=0/1 with sherpa 1.12.29) "succeed" but
393
+ // return all-NaN audio. Encoded WAV plays as silence, so users
394
+ // see "Playing 19s" with no sound. Fail loudly instead.
395
+ if (audio.samples.length > 0 && Number.isNaN(audio.samples[0])) {
396
+ throw new Error(
397
+ `TTS synthesis returned NaN samples for ${model.id} sid=${sid}. ` +
398
+ `This voice is incompatible with the installed sherpa-onnx-node ` +
399
+ `runtime — pick a different voice via /voice-speak-models or ` +
400
+ `/voice-settings → Speak tab.`
401
+ );
402
+ }
403
+ // If the run was aborted mid-generate, surface that to the caller
404
+ // rather than silently returning a partial buffer.
405
+ if (aborted) throw makeAbortError();
406
+ return {
407
+ samples: audio.samples,
408
+ sampleRate: audio.sampleRate ?? cached.sampleRate,
409
+ };
410
+ } finally {
411
+ signal?.removeEventListener("abort", onAbort);
412
+ }
413
+ }
414
+
415
+ function makeAbortError(): Error {
416
+ // Use a DOMException-shaped error so callers using AbortController.signal
417
+ // can pattern-match on `err.name === "AbortError"`. Falls back to a plain
418
+ // Error in environments without DOMException.
419
+ if (typeof DOMException === "function") {
420
+ return new DOMException("TTS synthesis aborted", "AbortError");
421
+ }
422
+ const e = new Error("TTS synthesis aborted");
423
+ (e as any).name = "AbortError";
424
+ return e;
425
+ }
426
+
427
+ /**
428
+ * Atomically extend a CachedTts's `generateChain` with a new tail.
429
+ *
430
+ * **All three operations run synchronously in a single function call.**
431
+ * JavaScript run-to-completion semantics guarantee no other code (no
432
+ * other synthesize() call, no microtask, no I/O callback) executes
433
+ * between the read of `cached.generateChain` and the assignment of its
434
+ * replacement. Two concurrent synthesize() callers therefore observe
435
+ * distinct previousTails and link into the chain in arrival order.
436
+ *
437
+ * Returns:
438
+ * - previousTail: the chain at function entry — caller awaits this
439
+ * before doing its own generate work
440
+ * - release: caller MUST call this when its generate settles (success
441
+ * or failure) so the next link can advance
442
+ */
443
+ function extendChain(cached: CachedTts): { previousTail: Promise<void>; release: () => void } {
444
+ const previousTail = cached.generateChain;
445
+ let release!: () => void;
446
+ cached.generateChain = new Promise<void>((res) => {
447
+ release = res;
448
+ });
449
+ return { previousTail, release };
450
+ }
451
+
452
+ // ─── Validation ──────────────────────────────────────────────────────────────
453
+
454
+ /**
455
+ * Validate that `language` is non-empty and a string. Both backends call
456
+ * this; the deepgram path additionally relies on the voice id encoding the
457
+ * language (see `tts-deepgram.ts`), so this is the only check it needs.
458
+ */
459
+ export function validateTtsLanguageInput(language: unknown): asserts language is string {
460
+ if (!language || typeof language !== "string") {
461
+ throw new Error(`TTS language is required (got: ${language})`);
462
+ }
463
+ }
464
+
465
+ /**
466
+ * Validate that `language` is supported by the local `model`. Throws a
467
+ * user-facing error describing how to fix the mismatch.
468
+ *
469
+ * Local-only by signature — deepgram callers don't have a TtsLocalModelInfo
470
+ * and use the (separate) deepgram voice catalog instead. Splitting the
471
+ * validation by backend keeps the cloud TTS module independent of the
472
+ * local catalog so the two engines can evolve independently.
473
+ */
474
+ export function resolveLanguageForLocal(model: TtsLocalModelInfo, language: string): void {
475
+ validateTtsLanguageInput(language);
476
+ if (!modelSupportsLanguage(model, language)) {
477
+ const supported = model.languages.join(", ");
478
+ throw new Error(
479
+ `Active local TTS model ${model.name} only supports ${supported}. ` +
480
+ `Switch model in /voice-settings or change ttsLanguage to a supported value.`
481
+ );
482
+ }
483
+ }
484
+
485
+ // ─── Helpers ─────────────────────────────────────────────────────────────────
486
+
487
+ /**
488
+ * Build the OfflineTtsConfig for sherpa-onnx based on the model's slot.
489
+ * Each slot has its own field names — kitten/vits/kokoro all live in
490
+ * separate sub-objects of OfflineTtsModelConfig.
491
+ */
492
+ function buildTtsConfig(model: TtsLocalModelInfo, modelDir: string): any {
493
+ const dataDir = path.join(modelDir, "espeak-ng-data");
494
+ const tokens = path.join(modelDir, "tokens.txt");
495
+ const numThreads = getTtsThreads(model.sherpaSlot);
496
+
497
+ switch (model.sherpaSlot) {
498
+ case "kitten": {
499
+ return {
500
+ model: {
501
+ kitten: {
502
+ model: findFirstOnnx(modelDir, ["model.fp16.onnx", "model.onnx"]),
503
+ voices: path.join(modelDir, "voices.bin"),
504
+ tokens,
505
+ dataDir,
506
+ lengthScale: 1.0,
507
+ },
508
+ },
509
+ numThreads,
510
+ provider: "cpu",
511
+ };
512
+ }
513
+ case "vits": {
514
+ // Piper voices ship with one file ending in `.onnx` — naming varies
515
+ // (`en_US-lessac-medium.onnx`, etc.). The downloader records the
516
+ // extracted file name so we don't have to grep for it here.
517
+ const onnx = findPiperOnnx(modelDir);
518
+ return {
519
+ model: {
520
+ vits: {
521
+ model: onnx,
522
+ tokens,
523
+ dataDir,
524
+ // Defaults from sherpa-onnx upstream; surface only if we add knobs
525
+ noiseScale: 0.667,
526
+ noiseScaleW: 0.8,
527
+ lengthScale: 1.0,
528
+ },
529
+ },
530
+ numThreads,
531
+ provider: "cpu",
532
+ };
533
+ }
534
+ case "kokoro": {
535
+ // Kokoro multilingual ships lexicon-* files for non-English languages;
536
+ // pass them comma-separated so the engine can handle code-switching.
537
+ // Filename varies between releases — int8 quantization (kokoro v1.0
538
+ // multilingual: `model.int8.onnx`) vs fp32 (`model.onnx`). Find
539
+ // whichever ships in this model dir.
540
+ const lexicon = findKokoroLexicons(modelDir);
541
+ return {
542
+ model: {
543
+ kokoro: {
544
+ model: findFirstOnnx(modelDir, ["model.int8.onnx", "model.onnx", "model.fp16.onnx"]),
545
+ voices: path.join(modelDir, "voices.bin"),
546
+ tokens,
547
+ dataDir,
548
+ lengthScale: 1.0,
549
+ ...(lexicon ? { lexicon } : {}),
550
+ },
551
+ },
552
+ numThreads,
553
+ provider: "cpu",
554
+ };
555
+ }
556
+ }
557
+ }
558
+
559
+ /**
560
+ * Per-model-class thread budget. TTS is a flow-matching / VITS / TDT-like
561
+ * autoregressive workload — same scaling characteristics as the STT
562
+ * transducer path. M-series Pro/Max chips scale to ~6 threads; non-Apple
563
+ * CPUs back off to 4 to leave headroom for the agent UI.
564
+ *
565
+ * Mirrors `getNumThreads(maxThreads)` in `sherpa-engine.ts` rather than
566
+ * importing it to keep TTS compileable in isolation if STT is later moved
567
+ * to a separate package.
568
+ */
569
+ function getTtsThreads(slot: TtsLocalModelInfo["sherpaSlot"]): number {
570
+ const cpus = os.cpus().length || 2;
571
+ if (cpus <= 2) return 1;
572
+ if (cpus <= 4) return 2;
573
+ // Per-slot tuning, mirroring the STT path's TRANSDUCER_MAX_THREADS=6
574
+ // vs the Whisper-class cap of 4. Decisions per sherpa-onnx published
575
+ // RTF curves and #2910 (CoreML regression for transformer graphs):
576
+ //
577
+ // - kitten (Kitten Nano TTS): small model, scales to 4 threads
578
+ // - vits (Piper): single-speaker VITS, scales to 4
579
+ // - kokoro (Kokoro v0.19/v1.0): larger transformer encoder, scales
580
+ // to ~6 P-cores on M-series
581
+ const max = slot === "kokoro" ? 6 : 4;
582
+ return Math.min(max, cpus - 2);
583
+ }
584
+
585
+ /** Clamp speaker id into the model's voice range. */
586
+ function clampSid(sid: number, model: TtsLocalModelInfo): number {
587
+ if (!Number.isFinite(sid)) return model.defaultSid;
588
+ const maxSid = Math.max(0, ...model.voices.map((v) => v.sid));
589
+ return Math.max(0, Math.min(maxSid, Math.floor(sid)));
590
+ }
591
+
592
+ function clampSpeed(speed: number): number {
593
+ if (!Number.isFinite(speed)) return 1.0;
594
+ return Math.max(0.5, Math.min(2.0, speed));
595
+ }
596
+
597
+ /**
598
+ * Locate the Piper VITS .onnx file inside the extracted model directory.
599
+ * Piper archives include exactly one `.onnx` file at the top level (no
600
+ * subdirectories), but the filename varies by voice — fd.readdirSync once
601
+ * and pick the first match.
602
+ */
603
+ function findPiperOnnx(modelDir: string): string {
604
+ const entries = fs.readdirSync(modelDir);
605
+ const onnx = entries.find((e) => e.endsWith(".onnx"));
606
+ if (!onnx) throw new Error(`No .onnx file found in Piper model directory: ${modelDir}`);
607
+ return path.join(modelDir, onnx);
608
+ }
609
+
610
+ /**
611
+ * Pick the first .onnx file present in `modelDir` matching one of the
612
+ * `candidates` (in priority order). Falls back to ANY .onnx file if no
613
+ * candidate is found — defends against future quantization variants
614
+ * (e.g. `model.q4.onnx`) without a code change.
615
+ *
616
+ * v7.1.2 fix: kitten + kokoro previously hardcoded `model.fp16.onnx`
617
+ * and `model.onnx` respectively. The actual sherpa-onnx model archives
618
+ * ship with different filenames per release (kokoro multilingual ships
619
+ * `model.int8.onnx`); the hardcoded path failed sherpa's
620
+ * "Failed to create OfflineTts. Check your config!" validation.
621
+ */
622
+ function findFirstOnnx(modelDir: string, candidates: string[]): string {
623
+ for (const c of candidates) {
624
+ const p = path.join(modelDir, c);
625
+ if (fs.existsSync(p)) return p;
626
+ }
627
+ const entries = fs.readdirSync(modelDir);
628
+ const onnx = entries.find((e) => e.endsWith(".onnx"));
629
+ if (!onnx) throw new Error(`No .onnx file found in model directory: ${modelDir}`);
630
+ return path.join(modelDir, onnx);
631
+ }
632
+
633
+ /**
634
+ * Find Kokoro lexicon files for multilingual support. Kokoro v1.0 ships
635
+ * `lexicon-us-en.txt`, `lexicon-zh.txt`, etc. for grapheme-to-phoneme
636
+ * conversion in non-English languages. Returns a comma-separated path
637
+ * string the engine accepts, or null for English-only Kokoro v0.19.
638
+ */
639
+ function findKokoroLexicons(modelDir: string): string | null {
640
+ const entries = fs.readdirSync(modelDir);
641
+ const lex = entries.filter((e) => e.startsWith("lexicon-") && e.endsWith(".txt"));
642
+ if (lex.length === 0) return null;
643
+ return lex.map((e) => path.join(modelDir, e)).join(",");
644
+ }
645
+
646
+ /**
647
+ * Tiny convenience for callers that have a model id but not the full model
648
+ * record. Looks up the catalog entry and forwards to synthesize().
649
+ */
650
+ export async function synthesizeById(opts: Omit<SynthesizeOpts, "model"> & { modelId: string }): Promise<TtsAudio> {
651
+ const model = getTtsModel(opts.modelId);
652
+ return synthesize({ ...opts, model });
653
+ }