react-native-executorch 0.11.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/android/CMakeLists.txt +13 -0
  2. package/android/build.gradle.kts +14 -3
  3. package/android/src/main/AndroidManifest.xml +6 -0
  4. package/android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchModule.kt +22 -2
  5. package/cpp/core/dtype.cpp +12 -1
  6. package/cpp/core/dtype.h +2 -1
  7. package/cpp/core/install.cpp +1 -0
  8. package/cpp/core/utils.cpp +72 -21
  9. package/cpp/core/utils.h +10 -0
  10. package/cpp/extensions/cv/utils.h +1 -0
  11. package/ios/PrivacyInfo.xcprivacy +36 -0
  12. package/lib/module/constants.js +11 -0
  13. package/lib/module/constants.js.map +1 -1
  14. package/lib/module/core/lifetime.js +6 -0
  15. package/lib/module/core/lifetime.js.map +1 -1
  16. package/lib/module/core/schema.js +7 -0
  17. package/lib/module/core/schema.js.map +1 -1
  18. package/lib/module/core/tensor.js.map +1 -1
  19. package/lib/module/extensions/cv/ops/box.js +3 -3
  20. package/lib/module/extensions/cv/ops/box.js.map +1 -1
  21. package/lib/module/extensions/cv/ops/image.js +10 -14
  22. package/lib/module/extensions/cv/ops/image.js.map +1 -1
  23. package/lib/module/extensions/cv/ops/quad.js +2 -2
  24. package/lib/module/extensions/cv/ops/quad.js.map +1 -1
  25. package/lib/module/extensions/cv/tasks/sdxsTextToImage.js +11 -4
  26. package/lib/module/extensions/cv/tasks/sdxsTextToImage.js.map +1 -1
  27. package/lib/module/extensions/cv/utils/imagePreprocessor.js +3 -0
  28. package/lib/module/extensions/cv/utils/imagePreprocessor.js.map +1 -1
  29. package/lib/module/extensions/cv/utils/paddleOcrUtils.js +2 -2
  30. package/lib/module/extensions/cv/utils/paddleOcrUtils.js.map +1 -1
  31. package/lib/module/extensions/llm/sampler.js +21 -7
  32. package/lib/module/extensions/llm/sampler.js.map +1 -1
  33. package/lib/module/extensions/llm/tasks/llmChatSession.js +4 -1
  34. package/lib/module/extensions/llm/tasks/llmChatSession.js.map +1 -1
  35. package/lib/module/extensions/speech/index.js +2 -0
  36. package/lib/module/extensions/speech/index.js.map +1 -1
  37. package/lib/module/extensions/speech/tasks/fsmnVoiceActivityDetection.js +5 -2
  38. package/lib/module/extensions/speech/tasks/fsmnVoiceActivityDetection.js.map +1 -1
  39. package/lib/module/extensions/speech/tasks/reverbTextToSpeech.js +461 -0
  40. package/lib/module/extensions/speech/tasks/reverbTextToSpeech.js.map +1 -0
  41. package/lib/module/extensions/speech/utils/reverbBackbone.js +275 -0
  42. package/lib/module/extensions/speech/utils/reverbBackbone.js.map +1 -0
  43. package/lib/module/extensions/speech/utils/reverbPartitioner.js +86 -0
  44. package/lib/module/extensions/speech/utils/reverbPartitioner.js.map +1 -0
  45. package/lib/module/extensions/speech/utils/reverbUtils.js +240 -0
  46. package/lib/module/extensions/speech/utils/reverbUtils.js.map +1 -0
  47. package/lib/module/extensions/speech/utils/textPartitioner.js +1 -1
  48. package/lib/module/fetcher/telemetry.js +1 -1
  49. package/lib/module/hooks/useTextToSpeech.js +22 -1
  50. package/lib/module/hooks/useTextToSpeech.js.map +1 -1
  51. package/lib/module/index.js +1 -0
  52. package/lib/module/index.js.map +1 -1
  53. package/lib/module/models.js +391 -43
  54. package/lib/module/models.js.map +1 -1
  55. package/lib/module/native/bridge.js +18 -0
  56. package/lib/module/native/bridge.js.map +1 -1
  57. package/lib/typescript/src/constants.d.ts +10 -0
  58. package/lib/typescript/src/constants.d.ts.map +1 -1
  59. package/lib/typescript/src/core/lifetime.d.ts.map +1 -1
  60. package/lib/typescript/src/core/schema.d.ts +7 -0
  61. package/lib/typescript/src/core/schema.d.ts.map +1 -1
  62. package/lib/typescript/src/core/tensor.d.ts +7 -6
  63. package/lib/typescript/src/core/tensor.d.ts.map +1 -1
  64. package/lib/typescript/src/extensions/cv/ops/image.d.ts.map +1 -1
  65. package/lib/typescript/src/extensions/cv/tasks/sdxsTextToImage.d.ts.map +1 -1
  66. package/lib/typescript/src/extensions/cv/utils/imagePreprocessor.d.ts.map +1 -1
  67. package/lib/typescript/src/extensions/llm/sampler.d.ts.map +1 -1
  68. package/lib/typescript/src/extensions/llm/tasks/llmChatSession.d.ts.map +1 -1
  69. package/lib/typescript/src/extensions/speech/index.d.ts +2 -0
  70. package/lib/typescript/src/extensions/speech/index.d.ts.map +1 -1
  71. package/lib/typescript/src/extensions/speech/tasks/fsmnVoiceActivityDetection.d.ts.map +1 -1
  72. package/lib/typescript/src/extensions/speech/tasks/reverbTextToSpeech.d.ts +175 -0
  73. package/lib/typescript/src/extensions/speech/tasks/reverbTextToSpeech.d.ts.map +1 -0
  74. package/lib/typescript/src/extensions/speech/utils/reverbBackbone.d.ts +67 -0
  75. package/lib/typescript/src/extensions/speech/utils/reverbBackbone.d.ts.map +1 -0
  76. package/lib/typescript/src/extensions/speech/utils/reverbPartitioner.d.ts +27 -0
  77. package/lib/typescript/src/extensions/speech/utils/reverbPartitioner.d.ts.map +1 -0
  78. package/lib/typescript/src/extensions/speech/utils/reverbUtils.d.ts +83 -0
  79. package/lib/typescript/src/extensions/speech/utils/reverbUtils.d.ts.map +1 -0
  80. package/lib/typescript/src/hooks/useTextToSpeech.d.ts +30 -0
  81. package/lib/typescript/src/hooks/useTextToSpeech.d.ts.map +1 -1
  82. package/lib/typescript/src/index.d.ts +1 -0
  83. package/lib/typescript/src/index.d.ts.map +1 -1
  84. package/lib/typescript/src/models.d.ts +56 -0
  85. package/lib/typescript/src/models.d.ts.map +1 -1
  86. package/package.json +4 -3
  87. package/react-native-executorch.podspec +5 -0
  88. package/scripts/download-libs.js +31 -6
  89. package/src/constants.ts +29 -0
  90. package/src/core/lifetime.ts +3 -0
  91. package/src/core/schema.ts +7 -0
  92. package/src/core/tensor.ts +9 -6
  93. package/src/extensions/cv/ops/box.ts +3 -3
  94. package/src/extensions/cv/ops/image.ts +11 -12
  95. package/src/extensions/cv/ops/quad.ts +2 -2
  96. package/src/extensions/cv/tasks/sdxsTextToImage.ts +10 -4
  97. package/src/extensions/cv/utils/imagePreprocessor.ts +3 -0
  98. package/src/extensions/cv/utils/paddleOcrUtils.ts +2 -2
  99. package/src/extensions/llm/sampler.ts +17 -3
  100. package/src/extensions/llm/tasks/llmChatSession.ts +5 -1
  101. package/src/extensions/speech/index.ts +2 -0
  102. package/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +7 -1
  103. package/src/extensions/speech/tasks/reverbTextToSpeech.ts +683 -0
  104. package/src/extensions/speech/utils/reverbBackbone.ts +342 -0
  105. package/src/extensions/speech/utils/reverbPartitioner.ts +95 -0
  106. package/src/extensions/speech/utils/reverbUtils.ts +251 -0
  107. package/src/extensions/speech/utils/textPartitioner.ts +1 -1
  108. package/src/fetcher/telemetry.ts +1 -1
  109. package/src/hooks/useTextToSpeech.ts +45 -3
  110. package/src/index.ts +1 -0
  111. package/src/models.ts +369 -8
  112. package/src/native/bridge.ts +21 -0
@@ -0,0 +1,683 @@
1
+ /**
2
+ * Reverb Text-to-Speech (TTS) synthesis task pipeline.
3
+ */
4
+
5
+ import type { WorkletRuntime } from 'react-native-worklets';
6
+
7
+ import RNBlobUtil from 'react-native-blob-util';
8
+
9
+ import { tensor } from '../../../core/tensor';
10
+ import { loadModel } from '../../../core/model';
11
+ import {
12
+ validateSpec,
13
+ method,
14
+ i64,
15
+ f32,
16
+ DynamicDim as Dyn,
17
+ constraint,
18
+ type Range,
19
+ } from '../../../core/schema';
20
+ import { wrapAsync } from '../../../core/runtime';
21
+ import { createResourceScope } from '../../../core/lifetime';
22
+ import { RnExecuTorchError } from '../../../core/error';
23
+ import { mulberry32 } from '../../math';
24
+ import { loadTokenizer } from '../../nlp/tokenizer';
25
+ import {
26
+ CODEBOOKS,
27
+ CODES,
28
+ MASK_CODE,
29
+ createReverbBackbone,
30
+ frameDim,
31
+ type ReverbPass,
32
+ } from '../utils/reverbBackbone';
33
+ import { partitionReverbText } from '../utils/reverbPartitioner';
34
+ import {
35
+ buildPrompt,
36
+ estimateTargetFrames,
37
+ parseVoice,
38
+ pauseAfter,
39
+ prepareText,
40
+ roundHalfEven,
41
+ speechLength,
42
+ trimSilence,
43
+ } from '../utils/reverbUtils';
44
+
45
+ /**
46
+ * Reverb audio sampling rate in Hz (24000).
47
+ * @category Speech / Constants
48
+ */
49
+ export const REVERB_SAMPLE_RATE = 24000;
50
+
51
+ /**
52
+ * Codec frames per second (25).
53
+ * @category Speech / Constants
54
+ */
55
+ export const REVERB_FRAME_RATE = 25;
56
+
57
+ /**
58
+ * Languages Reverb speaks.
59
+ * @category Speech / Constants
60
+ */
61
+ export const REVERB_LANGUAGES = ['de', 'en', 'es', 'fr', 'it', 'pl', 'pt'] as const;
62
+
63
+ const FRAME_SAMPLES = REVERB_SAMPLE_RATE / REVERB_FRAME_RATE; // 960
64
+
65
+ /**
66
+ * Sampler setup.
67
+ *
68
+ * The sampler is a modified version of a standard diffusion masked-language-model,
69
+ * steered with Gumbel noise. It consists of 2 phases:
70
+ * - **Main pass**: fills the first two codebooks of the RVQ codec, which corresponds to content & prosody
71
+ * - **Polish pass**: fills the upper codebooks (2-7), which encode timbre and acoustic details.
72
+ */
73
+ const T_SHIFT = 0.1; // few commits early, most in the last passes
74
+ const CODEBOOK_PENALTY = 5; // per codebook index: codebook 0 commits before 1
75
+ const POSITION_TEMPERATURE = 5; // scales confidence and penalty against the Gumbel noise
76
+ const DEFAULT_STEPS = 6; // main passes; the polish pass adds 2 more
77
+ const DEFAULT_REMASK_RATIO = 0.25; // held codes re-masked per new commit
78
+
79
+ const MIN_FRAMES = 10; // shortest take in codec frames, whatever the estimate
80
+ const DEFAULT_SPEED = 1.0; // the voice's own pace
81
+ const MIN_SPEED = 0.5; // slowest allowed speed factor
82
+ const MAX_SPEED = 2.0; // fastest allowed speed factor
83
+ const DEFAULT_MAX_CHUNK_DURATION = 20; // max seconds of speech per chunk
84
+ const DEFAULT_POSTPROCESS = { enabled: true, endCut: 0.2, silenceThreshold: -50, keep: 0.1 };
85
+
86
+ /**
87
+ * A language Reverb speaks.
88
+ * @category Speech / Types
89
+ */
90
+ export type ReverbLanguage = (typeof REVERB_LANGUAGES)[number];
91
+
92
+ /**
93
+ * A voice to speak in: a reference clip's codec codes and transcript.
94
+ * @category Speech / Types
95
+ */
96
+ export type ReverbVoice = {
97
+ /** Codec codes, `[8, frames]` row-major. */
98
+ readonly codes: BigInt64Array;
99
+ /** What the clip says, verbalized and normalized. */
100
+ readonly transcript: string;
101
+ /** The clip's language. */
102
+ readonly language: ReverbLanguage;
103
+ /** Frames of the clip spent speaking: without its silent edges, and with pauses over 0.5 s counted as 0.5 s. */
104
+ readonly speechFrames?: number;
105
+ };
106
+
107
+ /**
108
+ * Model configuration of the Reverb Text-to-Speech pipeline.
109
+ * @category Speech / Types
110
+ * @typeParam K Preset voice keys.
111
+ */
112
+ export type ReverbTtsModel<K extends PropertyKey = never> = {
113
+ /** Discriminates this config from the other Text-to-Speech pipelines. */
114
+ readonly name: 'reverb';
115
+ /** Local or remote paths to the `.pte` programs. */
116
+ readonly modelPaths: {
117
+ /** The backbone: `prefill`, `step` and `polish`, or `prefill` and `step` methods of fixed sizes. */
118
+ readonly backbone: string;
119
+ /** The codec decoder: codes to audio. */
120
+ readonly codecDecoder: string;
121
+ /** The codec encoder: audio to codes. Only `encodeVoice` loads it. */
122
+ readonly codecEncoder?: string;
123
+ };
124
+ /** Local or remote path to `tokenizer.json`. */
125
+ readonly tokenizerPath: string;
126
+ /** Preset voices: name to `{ codes, transcript, language }` JSON file. */
127
+ readonly voices?: Record<K, string>;
128
+ };
129
+
130
+ /**
131
+ * Sampler settings; the defaults are the measured ones.
132
+ * @category Speech / Types
133
+ */
134
+ export type ReverbSamplingOptions = {
135
+ /**
136
+ * Main passes. Defaults to 6.
137
+ * NOTE: total number of steps (including polish pass) is always N + 2
138
+ */
139
+ readonly steps?: number;
140
+ /** Held codes re-masked per new commit, in `[0, 1]`. Defaults to 0.25. */
141
+ readonly remaskRatio?: number;
142
+ /** Integer seed for reproducible takes. Random by default. */
143
+ readonly seed?: number;
144
+ };
145
+
146
+ /**
147
+ * Per-chunk post-processing; the defaults are Voice Studio's.
148
+ * @category Speech / Types
149
+ */
150
+ export type ReverbPostprocessOptions = {
151
+ /** `false` returns the raw decoder output. Defaults to `true`. */
152
+ readonly enabled?: boolean;
153
+ /** Seconds cut from the end. Defaults to 0.2. */
154
+ readonly endCut?: number;
155
+ /** Level in dBFS below which an edge window is silent. Defaults to -50. */
156
+ readonly silenceThreshold?: number;
157
+ /** Seconds of silence kept at each edge. Defaults to 0.1. */
158
+ readonly keep?: number;
159
+ };
160
+
161
+ /**
162
+ * Per-call options of Reverb synthesis.
163
+ * @category Speech / Types
164
+ * @typeParam K Preset voice keys.
165
+ */
166
+ export type ReverbTtsOptions<K extends PropertyKey = never> = {
167
+ /** A preset voice key, or a {@link ReverbVoice} from `encodeVoice`. */
168
+ readonly voice: K | ReverbVoice;
169
+ /** Language of the text. */
170
+ readonly language: ReverbLanguage;
171
+ /** Speech speed factor, 0.5 to 2.0. Defaults to 1.0. */
172
+ readonly speed?: number;
173
+ /**
174
+ * The way we estimate duration of the output sample.
175
+ * The duration is relative to the reference length, which we can be measured it two ways:
176
+ * `'speech'`: we count only speech frames and inner pauses not longer than 0.5s
177
+ * `'recording'`: we count the whole clip, pauses and prefix/suffix silence included.
178
+ */
179
+ readonly pace?: 'speech' | 'recording';
180
+ /** Max seconds of speech per chunk. Defaults to 20, or to the models' limit when that is shorter. */
181
+ readonly maxChunkDuration?: number;
182
+ /** Post-processing of each chunk's audio. */
183
+ readonly postprocess?: ReverbPostprocessOptions;
184
+ /** Sampler settings. */
185
+ readonly sampling?: ReverbSamplingOptions;
186
+ };
187
+
188
+ /**
189
+ * Audio chunk yielded by {@link ReverbTextToSpeech.synthesize}.
190
+ * @category Speech / Types
191
+ */
192
+ export type ReverbTtsChunk = {
193
+ /** Float32 PCM samples in `[-1, 1]`. */
194
+ readonly audio: Float32Array;
195
+ /** Sampling rate in Hz. */
196
+ readonly sampleRate: number;
197
+ /** Duration in seconds. */
198
+ readonly duration: number;
199
+ /** Zero-based index of this chunk. */
200
+ readonly chunkIndex: number;
201
+ /** Number of chunks the text was split into. */
202
+ readonly totalChunks: number;
203
+ };
204
+
205
+ /**
206
+ * Reverb text-to-speech task runner.
207
+ * @category Speech / Types
208
+ * @typeParam K Preset voice keys.
209
+ */
210
+ export type ReverbTextToSpeech<K extends PropertyKey = never> = {
211
+ /** Releases all native resources. */
212
+ readonly dispose: () => void;
213
+
214
+ /**
215
+ * Encodes a recording into a voice, loading the codec encoder on first use.
216
+ * @param audio 24 kHz mono PCM, 1 to 20 s.
217
+ * @param transcript What the recording says.
218
+ * @param language The recording's language.
219
+ * @returns The voice.
220
+ * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` on a bad argument,
221
+ * or `INVALID_STATE` without a codec encoder
222
+ */
223
+ readonly encodeVoice: (
224
+ audio: Float32Array,
225
+ transcript: string,
226
+ language: ReverbLanguage
227
+ ) => Promise<ReverbVoice>;
228
+
229
+ /**
230
+ * Streams synthesized audio, one chunk of text at a time.
231
+ * @param text The text to speak.
232
+ * @param options See {@link ReverbTtsOptions}.
233
+ * @returns An AsyncGenerator of {@link ReverbTtsChunk}s.
234
+ * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` on a bad argument,
235
+ * `INVALID_STATE` if a synthesis is running, `RESOURCE_BUSY` if the model is in use
236
+ */
237
+ readonly synthesize: (
238
+ text: string,
239
+ options: ReverbTtsOptions<K>
240
+ ) => AsyncGenerator<ReverbTtsChunk>;
241
+
242
+ /** Cancels an in-flight synthesis. */
243
+ readonly synthesizeStop: () => void;
244
+ };
245
+
246
+ const DECODER_SPEC = method(
247
+ 'forward',
248
+ [i64(CODEBOOKS, Dyn('T'))], // codes
249
+ [f32(Dyn('Samples'))], // audio
250
+ [constraint.linear(frameDim('output', 0, 0), frameDim('input', 0), FRAME_SAMPLES)]
251
+ );
252
+
253
+ const ENCODER_SPEC = method(
254
+ 'forward',
255
+ [f32(Dyn('F'), FRAME_SAMPLES)], // audio, one row per frame
256
+ [i64(CODEBOOKS, Dyn('F'))], // codes
257
+ [constraint.equality(frameDim('input', 0, 0), frameDim('output', 0))]
258
+ );
259
+
260
+ // Cumulative share of the slots committed after each main pass, from 0 to 1.
261
+ function commitSchedule(steps: number): number[] {
262
+ 'worklet';
263
+ const schedule: number[] = [];
264
+ for (let i = 0; i <= steps; i++) {
265
+ const t = i / steps;
266
+ schedule.push((T_SHIFT * t) / (1 - (1 - T_SHIFT) * t));
267
+ }
268
+ return schedule;
269
+ }
270
+
271
+ // One main pass over codebooks 0-1 of `tokens`, in place: commits the pass's share of the masked slots by
272
+ // Gumbel-perturbed confidence, then, except on the last pass, re-masks the least likely codes held before it.
273
+ function samplePass(
274
+ tokens: BigInt64Array,
275
+ frames: number,
276
+ pass: ReverbPass,
277
+ step: number,
278
+ schedule: readonly number[],
279
+ remaskRatio: number,
280
+ random: () => number
281
+ ): void {
282
+ 'worklet';
283
+ const slots = 2 * frames;
284
+ const last = step === schedule.length - 2;
285
+ const score = new Float64Array(slots);
286
+ const masked: number[] = [];
287
+ const held: number[] = [];
288
+ for (let i = 0; i < slots; i++) {
289
+ const gumbel = -Math.log(-Math.log(random() + 1e-10) + 1e-10);
290
+ if (tokens[i] !== MASK_CODE) {
291
+ held.push(i);
292
+ continue;
293
+ }
294
+ const penalty = CODEBOOK_PENALTY * Math.floor(i / frames);
295
+ score[i] = (pass.confidence[i]! - penalty) / POSITION_TEMPERATURE + gumbel;
296
+ masked.push(i);
297
+ }
298
+
299
+ const share = Math.ceil(slots * (schedule[step + 1]! - schedule[step]!));
300
+ const count = last ? masked.length : Math.min(masked.length, share);
301
+ masked.sort((a, b) => score[b]! - score[a]!);
302
+ for (const i of masked.slice(0, count)) tokens[i] = pass.prediction[i]!;
303
+
304
+ if (last || remaskRatio <= 0) return;
305
+ held.sort((a, b) => pass.held[a]! - pass.held[b]!);
306
+ for (const i of held.slice(0, roundHalfEven(remaskRatio * count))) tokens[i] = MASK_CODE;
307
+ }
308
+
309
+ /**
310
+ * Creates a Reverb Text-to-Speech pipeline: speech in the voice of a short
311
+ * recording, in 7 languages.
312
+ * @category Speech / Tasks
313
+ * @typeParam K Preset voice keys (inferred from `config.voices`).
314
+ * @param config Paths to the programs, tokenizer, and voices. See
315
+ * {@link ReverbTtsModel}.
316
+ * @param runtime Optional worklet runtime to run inference on.
317
+ * @returns The {@link ReverbTextToSpeech} runner.
318
+ * @throws {RnExecuTorchError} With code `LOAD_FAILED` if an asset fails to
319
+ * load, or `SCHEMA_MISMATCH` if a program does not match Reverb's.
320
+ */
321
+ export async function createReverbTextToSpeech<K extends PropertyKey = never>(
322
+ config: ReverbTtsModel<K>,
323
+ runtime?: WorkletRuntime
324
+ ): Promise<ReverbTextToSpeech<K>> {
325
+ const scope = createResourceScope();
326
+ const dispose = (): void => {
327
+ scope.dispose();
328
+ };
329
+
330
+ try {
331
+ const load = wrapAsync(loadModel, runtime);
332
+ const backbone = scope.track(
333
+ createReverbBackbone(scope.track(await load(config.modelPaths.backbone)))
334
+ );
335
+ const decoder = scope.track(await load(config.modelPaths.codecDecoder));
336
+ const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(config.tokenizerPath));
337
+
338
+ const { promptTokens, referenceFrames, targetFrames } = backbone.limits;
339
+ const openBackbone = backbone.open;
340
+ const [decodedFrames] = validateSpec(decoder.schema, { default: DECODER_SPEC }).dims.range('T');
341
+ const minFrames = Math.max(MIN_FRAMES, targetFrames.min, decodedFrames.min);
342
+ const maxFrames = Math.min(targetFrames.max, decodedFrames.max);
343
+
344
+ // A voice the programs take: [8, frames] codes in range, a transcript, and a supported language.
345
+ const validateVoice = (
346
+ voice: ReverbVoice,
347
+ code: 'INVALID_ARGUMENT' | 'LOAD_FAILED',
348
+ ctx: string
349
+ ): ReverbVoice => {
350
+ const frames = voice.codes.length / CODEBOOKS;
351
+ const valid =
352
+ Number.isInteger(frames) &&
353
+ frames >= referenceFrames.min &&
354
+ frames <= referenceFrames.max &&
355
+ voice.codes.every((c) => c >= 0n && c < MASK_CODE) &&
356
+ voice.transcript.trim().length > 0 &&
357
+ REVERB_LANGUAGES.includes(voice.language) &&
358
+ (voice.speechFrames === undefined ||
359
+ (Number.isInteger(voice.speechFrames) && voice.speechFrames > 0));
360
+ if (!valid) {
361
+ throw RnExecuTorchError(
362
+ code,
363
+ `${ctx}: A voice holds [${CODEBOOKS}, frames] codes below ${CODES}, with ` +
364
+ `${referenceFrames.min} to ${referenceFrames.max} frames, a transcript, a ` +
365
+ `supported language, and, if given, a positive integer of speech frames.`
366
+ );
367
+ }
368
+ return voice;
369
+ };
370
+
371
+ const voices = {} as Record<K, ReverbVoice>;
372
+ for (const [key, path] of Object.entries(config.voices ?? {}) as [K, string][]) {
373
+ const json = JSON.parse(await RNBlobUtil.fs.readFile(path, 'utf8'));
374
+ voices[key] = validateVoice(parseVoice(json), 'LOAD_FAILED', `Voice '${String(key)}'`);
375
+ }
376
+
377
+ const synthesizeChunkWorklet = (
378
+ ids: Int32Array,
379
+ voice: ReverbVoice,
380
+ frames: number,
381
+ chunkOpts: {
382
+ steps: number;
383
+ remaskRatio: number;
384
+ seed: number;
385
+ postprocess: Required<ReverbPostprocessOptions>;
386
+ pause: number;
387
+ }
388
+ ): Float32Array => {
389
+ 'worklet';
390
+
391
+ const { steps, remaskRatio, seed, postprocess, pause } = chunkOpts;
392
+ const slots = 2 * frames;
393
+ const tokens = new BigInt64Array(CODEBOOKS * frames).fill(MASK_CODE);
394
+ const pass = {
395
+ confidence: new Float32Array(slots),
396
+ prediction: new BigInt64Array(slots),
397
+ held: new Float32Array(slots),
398
+ };
399
+ const schedule = commitSchedule(steps);
400
+ const random = mulberry32(seed);
401
+
402
+ // The chunk's session and tensors, released together when it ends
403
+ const local = createResourceScope();
404
+
405
+ try {
406
+ // 1. Prefill the prompt and the reference
407
+ const session = local.track(openBackbone(ids, voice.codes, frames));
408
+
409
+ // 2. Main passes over codebooks 0-1
410
+ for (let step = 0; step < steps; step++) {
411
+ session.step(tokens, pass);
412
+ samplePass(tokens, frames, pass, step, schedule, remaskRatio, random);
413
+ }
414
+
415
+ // 3. Polish codebooks 2-7
416
+ session.polish(tokens);
417
+
418
+ // 4. Decode
419
+ const tCodes = local.track(tensor('int64', [CODEBOOKS, frames], tokens));
420
+ const tAudio = local.track(tensor('float32', [frames * FRAME_SAMPLES]));
421
+ decoder.execute('forward', [tCodes], [tAudio]);
422
+ let audio: Float32Array = tAudio.getData(new Float32Array(frames * FRAME_SAMPLES));
423
+
424
+ // 5. Cut the end, then trim the silent edges
425
+ if (postprocess.enabled) {
426
+ const { endCut, silenceThreshold, keep } = postprocess;
427
+ const end = Math.max(0, audio.length - Math.floor(endCut * REVERB_SAMPLE_RATE));
428
+ audio = trimSilence(audio.subarray(0, end), REVERB_SAMPLE_RATE, silenceThreshold, keep);
429
+ }
430
+
431
+ // 6. Append the pause the chunk's final punctuation calls for
432
+ const result = new Float32Array(audio.length + Math.round(pause * REVERB_SAMPLE_RATE));
433
+ result.set(audio);
434
+ return result;
435
+ } finally {
436
+ local.dispose();
437
+ }
438
+ };
439
+
440
+ const synthesizeChunk = wrapAsync(synthesizeChunkWorklet, runtime);
441
+
442
+ const checkLanguage = (language: unknown, ctx: string): void => {
443
+ if (!REVERB_LANGUAGES.includes(language as ReverbLanguage)) {
444
+ throw RnExecuTorchError(
445
+ 'INVALID_ARGUMENT',
446
+ `${ctx}: Unsupported language: ${String(language)}.`
447
+ );
448
+ }
449
+ };
450
+
451
+ const resolveVoice = (voice: K | ReverbVoice): ReverbVoice => {
452
+ const resolved = typeof voice === 'object' ? voice : voices[voice];
453
+ if (!resolved) {
454
+ throw RnExecuTorchError('INVALID_ARGUMENT', `synthesize: Unknown voice: ${String(voice)}.`);
455
+ }
456
+ return validateVoice(resolved, 'INVALID_ARGUMENT', 'synthesize');
457
+ };
458
+
459
+ let encoder:
460
+ | Promise<{ encode: (audio: Float32Array) => Promise<BigInt64Array>; frames: Range }>
461
+ | undefined;
462
+ const loadEncoder = async () => {
463
+ if (!config.modelPaths.codecEncoder) {
464
+ throw RnExecuTorchError('INVALID_STATE', 'encodeVoice: The config names no codecEncoder.');
465
+ }
466
+ const model = await load(config.modelPaths.codecEncoder);
467
+ try {
468
+ const [frames] = validateSpec(model.schema, { default: ENCODER_SPEC }).dims.range('F');
469
+ scope.track(model);
470
+ const encodeWorklet = (audio: Float32Array): BigInt64Array => {
471
+ 'worklet';
472
+ const count = audio.length / FRAME_SAMPLES;
473
+ const tAudio = tensor('float32', [count, FRAME_SAMPLES], audio);
474
+ const tCodes = tensor('int64', [CODEBOOKS, count]);
475
+ try {
476
+ model.execute('forward', [tAudio], [tCodes]);
477
+ return tCodes.getData(new BigInt64Array(CODEBOOKS * count));
478
+ } finally {
479
+ tAudio.dispose();
480
+ tCodes.dispose();
481
+ }
482
+ };
483
+ return { encode: wrapAsync(encodeWorklet, runtime), frames };
484
+ } catch (error) {
485
+ model.dispose();
486
+ throw error;
487
+ }
488
+ };
489
+
490
+ const encodeVoice = async (
491
+ audio: Float32Array,
492
+ transcript: string,
493
+ language: ReverbLanguage
494
+ ): Promise<ReverbVoice> => {
495
+ checkLanguage(language, 'encodeVoice');
496
+ if (!transcript.trim()) {
497
+ throw RnExecuTorchError('INVALID_ARGUMENT', 'encodeVoice: The transcript cannot be empty.');
498
+ }
499
+ encoder ??= loadEncoder().catch((error) => {
500
+ encoder = undefined;
501
+ throw error;
502
+ });
503
+ const { encode, frames } = await encoder;
504
+
505
+ const count = Math.floor(audio.length / FRAME_SAMPLES);
506
+ const min = Math.max(frames.min, referenceFrames.min);
507
+ const max = Math.min(frames.max, referenceFrames.max);
508
+ if (count < min || count > max) {
509
+ throw RnExecuTorchError(
510
+ 'INVALID_ARGUMENT',
511
+ `encodeVoice: The audio must last ${min / REVERB_FRAME_RATE} to ` +
512
+ `${max / REVERB_FRAME_RATE} s at ${REVERB_SAMPLE_RATE} Hz ` +
513
+ `(got ${audio.length / REVERB_SAMPLE_RATE} s).`
514
+ );
515
+ }
516
+ const clip = audio.slice(0, count * FRAME_SAMPLES);
517
+ const codes = await encode(clip);
518
+ // The clip stays whole; only its speaking time is measured, for the length estimate.
519
+ const speechFrames = roundHalfEven(
520
+ speechLength(clip, REVERB_SAMPLE_RATE) * REVERB_FRAME_RATE
521
+ );
522
+
523
+ return { codes, transcript: prepareText(transcript, language), language, speechFrames };
524
+ };
525
+
526
+ let isSynthesizing = false;
527
+ const synthesizeStop = (): void => {
528
+ isSynthesizing = false;
529
+ };
530
+
531
+ async function* synthesize(
532
+ text: string,
533
+ options: ReverbTtsOptions<K>
534
+ ): AsyncGenerator<ReverbTtsChunk> {
535
+ if (isSynthesizing) {
536
+ throw RnExecuTorchError('INVALID_STATE', 'synthesize: Synthesis is already in progress.');
537
+ }
538
+
539
+ if (!text || !text.trim()) {
540
+ throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: Input text cannot be empty.');
541
+ }
542
+
543
+ const { language } = options;
544
+ checkLanguage(language, 'synthesize');
545
+ const voice = resolveVoice(options.voice);
546
+ const references = voice.codes.length / CODEBOOKS;
547
+ const pace = options.pace ?? 'speech';
548
+ if (pace !== 'speech' && pace !== 'recording') {
549
+ throw RnExecuTorchError(
550
+ 'INVALID_ARGUMENT',
551
+ "synthesize: pace must be 'speech' or 'recording'."
552
+ );
553
+ }
554
+ // The reference length the estimate follows; the model still takes every reference frame.
555
+ const paceFrames =
556
+ pace === 'speech' ? Math.min(voice.speechFrames ?? references, references) : references;
557
+
558
+ const speed = options.speed ?? DEFAULT_SPEED;
559
+ if (!(speed >= MIN_SPEED && speed <= MAX_SPEED)) {
560
+ throw RnExecuTorchError(
561
+ 'INVALID_ARGUMENT',
562
+ `synthesize: speed must be between ${MIN_SPEED} and ${MAX_SPEED}.`
563
+ );
564
+ }
565
+
566
+ // A chunk at the limit must still fit the backbone's target window.
567
+ const maxDuration = maxFrames / REVERB_FRAME_RATE;
568
+ const maxChunkDuration =
569
+ options.maxChunkDuration ?? Math.min(DEFAULT_MAX_CHUNK_DURATION, maxDuration);
570
+ if (!(maxChunkDuration > 0 && maxChunkDuration <= maxDuration)) {
571
+ throw RnExecuTorchError(
572
+ 'INVALID_ARGUMENT',
573
+ `synthesize: maxChunkDuration must be above 0 and at most ${maxDuration} s.`
574
+ );
575
+ }
576
+
577
+ const steps = options.sampling?.steps ?? DEFAULT_STEPS;
578
+ const remaskRatio = options.sampling?.remaskRatio ?? DEFAULT_REMASK_RATIO;
579
+ const seed = options.sampling?.seed ?? Math.floor(Math.random() * 0x100000000);
580
+ if (!Number.isInteger(steps) || steps <= 0) {
581
+ throw RnExecuTorchError(
582
+ 'INVALID_ARGUMENT',
583
+ 'synthesize: steps must be a positive integer.'
584
+ );
585
+ }
586
+ if (!(remaskRatio >= 0 && remaskRatio <= 1)) {
587
+ throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: remaskRatio must be in [0, 1].');
588
+ }
589
+ if (!Number.isInteger(seed)) {
590
+ throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: seed must be an integer.');
591
+ }
592
+ const postprocess = {
593
+ enabled: options.postprocess?.enabled ?? DEFAULT_POSTPROCESS.enabled,
594
+ endCut: options.postprocess?.endCut ?? DEFAULT_POSTPROCESS.endCut,
595
+ silenceThreshold:
596
+ options.postprocess?.silenceThreshold ?? DEFAULT_POSTPROCESS.silenceThreshold,
597
+ keep: options.postprocess?.keep ?? DEFAULT_POSTPROCESS.keep,
598
+ };
599
+ if (postprocess.enabled) {
600
+ const { endCut, silenceThreshold, keep } = postprocess;
601
+ if (!(Number.isFinite(endCut) && endCut >= 0)) {
602
+ throw RnExecuTorchError(
603
+ 'INVALID_ARGUMENT',
604
+ 'synthesize: postprocess.endCut must be a finite number of seconds, 0 or more.'
605
+ );
606
+ }
607
+ if (!Number.isFinite(silenceThreshold)) {
608
+ throw RnExecuTorchError(
609
+ 'INVALID_ARGUMENT',
610
+ 'synthesize: postprocess.silenceThreshold must be a finite level in dBFS.'
611
+ );
612
+ }
613
+ if (!(Number.isFinite(keep) && keep >= 0)) {
614
+ throw RnExecuTorchError(
615
+ 'INVALID_ARGUMENT',
616
+ 'synthesize: postprocess.keep must be a finite number of seconds, 0 or more.'
617
+ );
618
+ }
619
+ }
620
+
621
+ const encodePrompt = (chunk: string) =>
622
+ tokenizer.encode(buildPrompt(language, voice.transcript, chunk));
623
+ if (encodePrompt('').length >= promptTokens.max) {
624
+ throw RnExecuTorchError(
625
+ 'INVALID_ARGUMENT',
626
+ `synthesize: The voice's transcript leaves no room for text in the ` +
627
+ `${promptTokens.max}-token prompt.`
628
+ );
629
+ }
630
+ const targetLength = (chunk: string) =>
631
+ estimateTargetFrames(chunk, voice.transcript, paceFrames) / speed;
632
+
633
+ isSynthesizing = true;
634
+ try {
635
+ const chunks = partitionReverbText(
636
+ prepareText(text, language),
637
+ (chunk) => targetLength(chunk) / REVERB_FRAME_RATE,
638
+ {
639
+ maxDuration: maxChunkDuration,
640
+ fits: (chunk) => encodePrompt(chunk).length <= promptTokens.max,
641
+ }
642
+ );
643
+
644
+ for (const [chunkIndex, chunk] of chunks.entries()) {
645
+ if (!isSynthesizing) break;
646
+
647
+ const ids = encodePrompt(chunk);
648
+ if (ids.length > promptTokens.max) {
649
+ throw RnExecuTorchError(
650
+ 'INVALID_ARGUMENT',
651
+ `synthesize: "${chunk}" does not fit the ${promptTokens.max}-token prompt.`
652
+ );
653
+ }
654
+ const frames = Math.min(
655
+ Math.max(roundHalfEven(targetLength(chunk)), minFrames),
656
+ maxFrames
657
+ );
658
+ const audio = await synthesizeChunk(ids, voice, frames, {
659
+ steps,
660
+ remaskRatio,
661
+ seed: seed + chunkIndex,
662
+ postprocess,
663
+ pause: pauseAfter(chunk),
664
+ });
665
+ yield {
666
+ audio,
667
+ sampleRate: REVERB_SAMPLE_RATE,
668
+ duration: audio.length / REVERB_SAMPLE_RATE,
669
+ chunkIndex,
670
+ totalChunks: chunks.length,
671
+ };
672
+ }
673
+ } finally {
674
+ isSynthesizing = false;
675
+ }
676
+ }
677
+
678
+ return { dispose, encodeVoice, synthesize, synthesizeStop };
679
+ } catch (error) {
680
+ dispose();
681
+ throw error;
682
+ }
683
+ }