react-native-executorch 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/CMakeLists.txt +13 -0
- package/android/build.gradle.kts +14 -3
- package/android/src/main/AndroidManifest.xml +6 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchModule.kt +22 -2
- package/cpp/core/dtype.cpp +12 -1
- package/cpp/core/dtype.h +2 -1
- package/cpp/core/install.cpp +1 -0
- package/cpp/core/utils.cpp +72 -21
- package/cpp/core/utils.h +10 -0
- package/cpp/extensions/cv/utils.h +1 -0
- package/ios/PrivacyInfo.xcprivacy +36 -0
- package/lib/module/constants.js +11 -0
- package/lib/module/constants.js.map +1 -1
- package/lib/module/core/lifetime.js +6 -0
- package/lib/module/core/lifetime.js.map +1 -1
- package/lib/module/core/schema.js +7 -0
- package/lib/module/core/schema.js.map +1 -1
- package/lib/module/core/tensor.js.map +1 -1
- package/lib/module/extensions/cv/ops/box.js +3 -3
- package/lib/module/extensions/cv/ops/box.js.map +1 -1
- package/lib/module/extensions/cv/ops/image.js +10 -14
- package/lib/module/extensions/cv/ops/image.js.map +1 -1
- package/lib/module/extensions/cv/ops/quad.js +2 -2
- package/lib/module/extensions/cv/ops/quad.js.map +1 -1
- package/lib/module/extensions/cv/tasks/sdxsTextToImage.js +11 -4
- package/lib/module/extensions/cv/tasks/sdxsTextToImage.js.map +1 -1
- package/lib/module/extensions/cv/utils/imagePreprocessor.js +3 -0
- package/lib/module/extensions/cv/utils/imagePreprocessor.js.map +1 -1
- package/lib/module/extensions/cv/utils/paddleOcrUtils.js +2 -2
- package/lib/module/extensions/cv/utils/paddleOcrUtils.js.map +1 -1
- package/lib/module/extensions/llm/sampler.js +21 -7
- package/lib/module/extensions/llm/sampler.js.map +1 -1
- package/lib/module/extensions/llm/tasks/llmChatSession.js +4 -1
- package/lib/module/extensions/llm/tasks/llmChatSession.js.map +1 -1
- package/lib/module/extensions/speech/index.js +2 -0
- package/lib/module/extensions/speech/index.js.map +1 -1
- package/lib/module/extensions/speech/tasks/fsmnVoiceActivityDetection.js +5 -2
- package/lib/module/extensions/speech/tasks/fsmnVoiceActivityDetection.js.map +1 -1
- package/lib/module/extensions/speech/tasks/reverbTextToSpeech.js +461 -0
- package/lib/module/extensions/speech/tasks/reverbTextToSpeech.js.map +1 -0
- package/lib/module/extensions/speech/utils/reverbBackbone.js +275 -0
- package/lib/module/extensions/speech/utils/reverbBackbone.js.map +1 -0
- package/lib/module/extensions/speech/utils/reverbPartitioner.js +86 -0
- package/lib/module/extensions/speech/utils/reverbPartitioner.js.map +1 -0
- package/lib/module/extensions/speech/utils/reverbUtils.js +240 -0
- package/lib/module/extensions/speech/utils/reverbUtils.js.map +1 -0
- package/lib/module/extensions/speech/utils/textPartitioner.js +1 -1
- package/lib/module/fetcher/telemetry.js +1 -1
- package/lib/module/hooks/useTextToSpeech.js +22 -1
- package/lib/module/hooks/useTextToSpeech.js.map +1 -1
- package/lib/module/index.js +1 -0
- package/lib/module/index.js.map +1 -1
- package/lib/module/models.js +391 -43
- package/lib/module/models.js.map +1 -1
- package/lib/module/native/bridge.js +18 -0
- package/lib/module/native/bridge.js.map +1 -1
- package/lib/typescript/src/constants.d.ts +10 -0
- package/lib/typescript/src/constants.d.ts.map +1 -1
- package/lib/typescript/src/core/lifetime.d.ts.map +1 -1
- package/lib/typescript/src/core/schema.d.ts +7 -0
- package/lib/typescript/src/core/schema.d.ts.map +1 -1
- package/lib/typescript/src/core/tensor.d.ts +7 -6
- package/lib/typescript/src/core/tensor.d.ts.map +1 -1
- package/lib/typescript/src/extensions/cv/ops/image.d.ts.map +1 -1
- package/lib/typescript/src/extensions/cv/tasks/sdxsTextToImage.d.ts.map +1 -1
- package/lib/typescript/src/extensions/cv/utils/imagePreprocessor.d.ts.map +1 -1
- package/lib/typescript/src/extensions/llm/sampler.d.ts.map +1 -1
- package/lib/typescript/src/extensions/llm/tasks/llmChatSession.d.ts.map +1 -1
- package/lib/typescript/src/extensions/speech/index.d.ts +2 -0
- package/lib/typescript/src/extensions/speech/index.d.ts.map +1 -1
- package/lib/typescript/src/extensions/speech/tasks/fsmnVoiceActivityDetection.d.ts.map +1 -1
- package/lib/typescript/src/extensions/speech/tasks/reverbTextToSpeech.d.ts +175 -0
- package/lib/typescript/src/extensions/speech/tasks/reverbTextToSpeech.d.ts.map +1 -0
- package/lib/typescript/src/extensions/speech/utils/reverbBackbone.d.ts +67 -0
- package/lib/typescript/src/extensions/speech/utils/reverbBackbone.d.ts.map +1 -0
- package/lib/typescript/src/extensions/speech/utils/reverbPartitioner.d.ts +27 -0
- package/lib/typescript/src/extensions/speech/utils/reverbPartitioner.d.ts.map +1 -0
- package/lib/typescript/src/extensions/speech/utils/reverbUtils.d.ts +83 -0
- package/lib/typescript/src/extensions/speech/utils/reverbUtils.d.ts.map +1 -0
- package/lib/typescript/src/hooks/useTextToSpeech.d.ts +30 -0
- package/lib/typescript/src/hooks/useTextToSpeech.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +1 -0
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/models.d.ts +56 -0
- package/lib/typescript/src/models.d.ts.map +1 -1
- package/package.json +4 -3
- package/react-native-executorch.podspec +5 -0
- package/scripts/download-libs.js +31 -6
- package/src/constants.ts +29 -0
- package/src/core/lifetime.ts +3 -0
- package/src/core/schema.ts +7 -0
- package/src/core/tensor.ts +9 -6
- package/src/extensions/cv/ops/box.ts +3 -3
- package/src/extensions/cv/ops/image.ts +11 -12
- package/src/extensions/cv/ops/quad.ts +2 -2
- package/src/extensions/cv/tasks/sdxsTextToImage.ts +10 -4
- package/src/extensions/cv/utils/imagePreprocessor.ts +3 -0
- package/src/extensions/cv/utils/paddleOcrUtils.ts +2 -2
- package/src/extensions/llm/sampler.ts +17 -3
- package/src/extensions/llm/tasks/llmChatSession.ts +5 -1
- package/src/extensions/speech/index.ts +2 -0
- package/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +7 -1
- package/src/extensions/speech/tasks/reverbTextToSpeech.ts +683 -0
- package/src/extensions/speech/utils/reverbBackbone.ts +342 -0
- package/src/extensions/speech/utils/reverbPartitioner.ts +95 -0
- package/src/extensions/speech/utils/reverbUtils.ts +251 -0
- package/src/extensions/speech/utils/textPartitioner.ts +1 -1
- package/src/fetcher/telemetry.ts +1 -1
- package/src/hooks/useTextToSpeech.ts +45 -3
- package/src/index.ts +1 -0
- package/src/models.ts +369 -8
- package/src/native/bridge.ts +21 -0
|
@@ -0,0 +1,683 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reverb Text-to-Speech (TTS) synthesis task pipeline.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import type { WorkletRuntime } from 'react-native-worklets';
|
|
6
|
+
|
|
7
|
+
import RNBlobUtil from 'react-native-blob-util';
|
|
8
|
+
|
|
9
|
+
import { tensor } from '../../../core/tensor';
|
|
10
|
+
import { loadModel } from '../../../core/model';
|
|
11
|
+
import {
|
|
12
|
+
validateSpec,
|
|
13
|
+
method,
|
|
14
|
+
i64,
|
|
15
|
+
f32,
|
|
16
|
+
DynamicDim as Dyn,
|
|
17
|
+
constraint,
|
|
18
|
+
type Range,
|
|
19
|
+
} from '../../../core/schema';
|
|
20
|
+
import { wrapAsync } from '../../../core/runtime';
|
|
21
|
+
import { createResourceScope } from '../../../core/lifetime';
|
|
22
|
+
import { RnExecuTorchError } from '../../../core/error';
|
|
23
|
+
import { mulberry32 } from '../../math';
|
|
24
|
+
import { loadTokenizer } from '../../nlp/tokenizer';
|
|
25
|
+
import {
|
|
26
|
+
CODEBOOKS,
|
|
27
|
+
CODES,
|
|
28
|
+
MASK_CODE,
|
|
29
|
+
createReverbBackbone,
|
|
30
|
+
frameDim,
|
|
31
|
+
type ReverbPass,
|
|
32
|
+
} from '../utils/reverbBackbone';
|
|
33
|
+
import { partitionReverbText } from '../utils/reverbPartitioner';
|
|
34
|
+
import {
|
|
35
|
+
buildPrompt,
|
|
36
|
+
estimateTargetFrames,
|
|
37
|
+
parseVoice,
|
|
38
|
+
pauseAfter,
|
|
39
|
+
prepareText,
|
|
40
|
+
roundHalfEven,
|
|
41
|
+
speechLength,
|
|
42
|
+
trimSilence,
|
|
43
|
+
} from '../utils/reverbUtils';
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Reverb audio sampling rate in Hz (24000).
|
|
47
|
+
* @category Speech / Constants
|
|
48
|
+
*/
|
|
49
|
+
export const REVERB_SAMPLE_RATE = 24000;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Codec frames per second (25).
|
|
53
|
+
* @category Speech / Constants
|
|
54
|
+
*/
|
|
55
|
+
export const REVERB_FRAME_RATE = 25;
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Languages Reverb speaks.
|
|
59
|
+
* @category Speech / Constants
|
|
60
|
+
*/
|
|
61
|
+
export const REVERB_LANGUAGES = ['de', 'en', 'es', 'fr', 'it', 'pl', 'pt'] as const;
|
|
62
|
+
|
|
63
|
+
const FRAME_SAMPLES = REVERB_SAMPLE_RATE / REVERB_FRAME_RATE; // 960
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Sampler setup.
|
|
67
|
+
*
|
|
68
|
+
* The sampler is a modified version of a standard diffusion masked-language-model,
|
|
69
|
+
* steered with Gumbel noise. It consists of 2 phases:
|
|
70
|
+
* - **Main pass**: fills the first two codebooks of the RVQ codec, which corresponds to content & prosody
|
|
71
|
+
* - **Polish pass**: fills the upper codebooks (2-7), which encode timbre and acoustic details.
|
|
72
|
+
*/
|
|
73
|
+
const T_SHIFT = 0.1; // few commits early, most in the last passes
|
|
74
|
+
const CODEBOOK_PENALTY = 5; // per codebook index: codebook 0 commits before 1
|
|
75
|
+
const POSITION_TEMPERATURE = 5; // scales confidence and penalty against the Gumbel noise
|
|
76
|
+
const DEFAULT_STEPS = 6; // main passes; the polish pass adds 2 more
|
|
77
|
+
const DEFAULT_REMASK_RATIO = 0.25; // held codes re-masked per new commit
|
|
78
|
+
|
|
79
|
+
const MIN_FRAMES = 10; // shortest take in codec frames, whatever the estimate
|
|
80
|
+
const DEFAULT_SPEED = 1.0; // the voice's own pace
|
|
81
|
+
const MIN_SPEED = 0.5; // slowest allowed speed factor
|
|
82
|
+
const MAX_SPEED = 2.0; // fastest allowed speed factor
|
|
83
|
+
const DEFAULT_MAX_CHUNK_DURATION = 20; // max seconds of speech per chunk
|
|
84
|
+
const DEFAULT_POSTPROCESS = { enabled: true, endCut: 0.2, silenceThreshold: -50, keep: 0.1 };
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* A language Reverb speaks.
|
|
88
|
+
* @category Speech / Types
|
|
89
|
+
*/
|
|
90
|
+
export type ReverbLanguage = (typeof REVERB_LANGUAGES)[number];
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* A voice to speak in: a reference clip's codec codes and transcript.
|
|
94
|
+
* @category Speech / Types
|
|
95
|
+
*/
|
|
96
|
+
export type ReverbVoice = {
|
|
97
|
+
/** Codec codes, `[8, frames]` row-major. */
|
|
98
|
+
readonly codes: BigInt64Array;
|
|
99
|
+
/** What the clip says, verbalized and normalized. */
|
|
100
|
+
readonly transcript: string;
|
|
101
|
+
/** The clip's language. */
|
|
102
|
+
readonly language: ReverbLanguage;
|
|
103
|
+
/** Frames of the clip spent speaking: without its silent edges, and with pauses over 0.5 s counted as 0.5 s. */
|
|
104
|
+
readonly speechFrames?: number;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Model configuration of the Reverb Text-to-Speech pipeline.
|
|
109
|
+
* @category Speech / Types
|
|
110
|
+
* @typeParam K Preset voice keys.
|
|
111
|
+
*/
|
|
112
|
+
export type ReverbTtsModel<K extends PropertyKey = never> = {
|
|
113
|
+
/** Discriminates this config from the other Text-to-Speech pipelines. */
|
|
114
|
+
readonly name: 'reverb';
|
|
115
|
+
/** Local or remote paths to the `.pte` programs. */
|
|
116
|
+
readonly modelPaths: {
|
|
117
|
+
/** The backbone: `prefill`, `step` and `polish`, or `prefill` and `step` methods of fixed sizes. */
|
|
118
|
+
readonly backbone: string;
|
|
119
|
+
/** The codec decoder: codes to audio. */
|
|
120
|
+
readonly codecDecoder: string;
|
|
121
|
+
/** The codec encoder: audio to codes. Only `encodeVoice` loads it. */
|
|
122
|
+
readonly codecEncoder?: string;
|
|
123
|
+
};
|
|
124
|
+
/** Local or remote path to `tokenizer.json`. */
|
|
125
|
+
readonly tokenizerPath: string;
|
|
126
|
+
/** Preset voices: name to `{ codes, transcript, language }` JSON file. */
|
|
127
|
+
readonly voices?: Record<K, string>;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Sampler settings; the defaults are the measured ones.
|
|
132
|
+
* @category Speech / Types
|
|
133
|
+
*/
|
|
134
|
+
export type ReverbSamplingOptions = {
|
|
135
|
+
/**
|
|
136
|
+
* Main passes. Defaults to 6.
|
|
137
|
+
* NOTE: total number of steps (including polish pass) is always N + 2
|
|
138
|
+
*/
|
|
139
|
+
readonly steps?: number;
|
|
140
|
+
/** Held codes re-masked per new commit, in `[0, 1]`. Defaults to 0.25. */
|
|
141
|
+
readonly remaskRatio?: number;
|
|
142
|
+
/** Integer seed for reproducible takes. Random by default. */
|
|
143
|
+
readonly seed?: number;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Per-chunk post-processing; the defaults are Voice Studio's.
|
|
148
|
+
* @category Speech / Types
|
|
149
|
+
*/
|
|
150
|
+
export type ReverbPostprocessOptions = {
|
|
151
|
+
/** `false` returns the raw decoder output. Defaults to `true`. */
|
|
152
|
+
readonly enabled?: boolean;
|
|
153
|
+
/** Seconds cut from the end. Defaults to 0.2. */
|
|
154
|
+
readonly endCut?: number;
|
|
155
|
+
/** Level in dBFS below which an edge window is silent. Defaults to -50. */
|
|
156
|
+
readonly silenceThreshold?: number;
|
|
157
|
+
/** Seconds of silence kept at each edge. Defaults to 0.1. */
|
|
158
|
+
readonly keep?: number;
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Per-call options of Reverb synthesis.
|
|
163
|
+
* @category Speech / Types
|
|
164
|
+
* @typeParam K Preset voice keys.
|
|
165
|
+
*/
|
|
166
|
+
export type ReverbTtsOptions<K extends PropertyKey = never> = {
|
|
167
|
+
/** A preset voice key, or a {@link ReverbVoice} from `encodeVoice`. */
|
|
168
|
+
readonly voice: K | ReverbVoice;
|
|
169
|
+
/** Language of the text. */
|
|
170
|
+
readonly language: ReverbLanguage;
|
|
171
|
+
/** Speech speed factor, 0.5 to 2.0. Defaults to 1.0. */
|
|
172
|
+
readonly speed?: number;
|
|
173
|
+
/**
|
|
174
|
+
* The way we estimate duration of the output sample.
|
|
175
|
+
* The duration is relative to the reference length, which we can be measured it two ways:
|
|
176
|
+
* `'speech'`: we count only speech frames and inner pauses not longer than 0.5s
|
|
177
|
+
* `'recording'`: we count the whole clip, pauses and prefix/suffix silence included.
|
|
178
|
+
*/
|
|
179
|
+
readonly pace?: 'speech' | 'recording';
|
|
180
|
+
/** Max seconds of speech per chunk. Defaults to 20, or to the models' limit when that is shorter. */
|
|
181
|
+
readonly maxChunkDuration?: number;
|
|
182
|
+
/** Post-processing of each chunk's audio. */
|
|
183
|
+
readonly postprocess?: ReverbPostprocessOptions;
|
|
184
|
+
/** Sampler settings. */
|
|
185
|
+
readonly sampling?: ReverbSamplingOptions;
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Audio chunk yielded by {@link ReverbTextToSpeech.synthesize}.
|
|
190
|
+
* @category Speech / Types
|
|
191
|
+
*/
|
|
192
|
+
export type ReverbTtsChunk = {
|
|
193
|
+
/** Float32 PCM samples in `[-1, 1]`. */
|
|
194
|
+
readonly audio: Float32Array;
|
|
195
|
+
/** Sampling rate in Hz. */
|
|
196
|
+
readonly sampleRate: number;
|
|
197
|
+
/** Duration in seconds. */
|
|
198
|
+
readonly duration: number;
|
|
199
|
+
/** Zero-based index of this chunk. */
|
|
200
|
+
readonly chunkIndex: number;
|
|
201
|
+
/** Number of chunks the text was split into. */
|
|
202
|
+
readonly totalChunks: number;
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Reverb text-to-speech task runner.
|
|
207
|
+
* @category Speech / Types
|
|
208
|
+
* @typeParam K Preset voice keys.
|
|
209
|
+
*/
|
|
210
|
+
export type ReverbTextToSpeech<K extends PropertyKey = never> = {
|
|
211
|
+
/** Releases all native resources. */
|
|
212
|
+
readonly dispose: () => void;
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Encodes a recording into a voice, loading the codec encoder on first use.
|
|
216
|
+
* @param audio 24 kHz mono PCM, 1 to 20 s.
|
|
217
|
+
* @param transcript What the recording says.
|
|
218
|
+
* @param language The recording's language.
|
|
219
|
+
* @returns The voice.
|
|
220
|
+
* @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` on a bad argument,
|
|
221
|
+
* or `INVALID_STATE` without a codec encoder
|
|
222
|
+
*/
|
|
223
|
+
readonly encodeVoice: (
|
|
224
|
+
audio: Float32Array,
|
|
225
|
+
transcript: string,
|
|
226
|
+
language: ReverbLanguage
|
|
227
|
+
) => Promise<ReverbVoice>;
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Streams synthesized audio, one chunk of text at a time.
|
|
231
|
+
* @param text The text to speak.
|
|
232
|
+
* @param options See {@link ReverbTtsOptions}.
|
|
233
|
+
* @returns An AsyncGenerator of {@link ReverbTtsChunk}s.
|
|
234
|
+
* @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` on a bad argument,
|
|
235
|
+
* `INVALID_STATE` if a synthesis is running, `RESOURCE_BUSY` if the model is in use
|
|
236
|
+
*/
|
|
237
|
+
readonly synthesize: (
|
|
238
|
+
text: string,
|
|
239
|
+
options: ReverbTtsOptions<K>
|
|
240
|
+
) => AsyncGenerator<ReverbTtsChunk>;
|
|
241
|
+
|
|
242
|
+
/** Cancels an in-flight synthesis. */
|
|
243
|
+
readonly synthesizeStop: () => void;
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
const DECODER_SPEC = method(
|
|
247
|
+
'forward',
|
|
248
|
+
[i64(CODEBOOKS, Dyn('T'))], // codes
|
|
249
|
+
[f32(Dyn('Samples'))], // audio
|
|
250
|
+
[constraint.linear(frameDim('output', 0, 0), frameDim('input', 0), FRAME_SAMPLES)]
|
|
251
|
+
);
|
|
252
|
+
|
|
253
|
+
const ENCODER_SPEC = method(
|
|
254
|
+
'forward',
|
|
255
|
+
[f32(Dyn('F'), FRAME_SAMPLES)], // audio, one row per frame
|
|
256
|
+
[i64(CODEBOOKS, Dyn('F'))], // codes
|
|
257
|
+
[constraint.equality(frameDim('input', 0, 0), frameDim('output', 0))]
|
|
258
|
+
);
|
|
259
|
+
|
|
260
|
+
// Cumulative share of the slots committed after each main pass, from 0 to 1.
|
|
261
|
+
function commitSchedule(steps: number): number[] {
|
|
262
|
+
'worklet';
|
|
263
|
+
const schedule: number[] = [];
|
|
264
|
+
for (let i = 0; i <= steps; i++) {
|
|
265
|
+
const t = i / steps;
|
|
266
|
+
schedule.push((T_SHIFT * t) / (1 - (1 - T_SHIFT) * t));
|
|
267
|
+
}
|
|
268
|
+
return schedule;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// One main pass over codebooks 0-1 of `tokens`, in place: commits the pass's share of the masked slots by
|
|
272
|
+
// Gumbel-perturbed confidence, then, except on the last pass, re-masks the least likely codes held before it.
|
|
273
|
+
function samplePass(
|
|
274
|
+
tokens: BigInt64Array,
|
|
275
|
+
frames: number,
|
|
276
|
+
pass: ReverbPass,
|
|
277
|
+
step: number,
|
|
278
|
+
schedule: readonly number[],
|
|
279
|
+
remaskRatio: number,
|
|
280
|
+
random: () => number
|
|
281
|
+
): void {
|
|
282
|
+
'worklet';
|
|
283
|
+
const slots = 2 * frames;
|
|
284
|
+
const last = step === schedule.length - 2;
|
|
285
|
+
const score = new Float64Array(slots);
|
|
286
|
+
const masked: number[] = [];
|
|
287
|
+
const held: number[] = [];
|
|
288
|
+
for (let i = 0; i < slots; i++) {
|
|
289
|
+
const gumbel = -Math.log(-Math.log(random() + 1e-10) + 1e-10);
|
|
290
|
+
if (tokens[i] !== MASK_CODE) {
|
|
291
|
+
held.push(i);
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
const penalty = CODEBOOK_PENALTY * Math.floor(i / frames);
|
|
295
|
+
score[i] = (pass.confidence[i]! - penalty) / POSITION_TEMPERATURE + gumbel;
|
|
296
|
+
masked.push(i);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
const share = Math.ceil(slots * (schedule[step + 1]! - schedule[step]!));
|
|
300
|
+
const count = last ? masked.length : Math.min(masked.length, share);
|
|
301
|
+
masked.sort((a, b) => score[b]! - score[a]!);
|
|
302
|
+
for (const i of masked.slice(0, count)) tokens[i] = pass.prediction[i]!;
|
|
303
|
+
|
|
304
|
+
if (last || remaskRatio <= 0) return;
|
|
305
|
+
held.sort((a, b) => pass.held[a]! - pass.held[b]!);
|
|
306
|
+
for (const i of held.slice(0, roundHalfEven(remaskRatio * count))) tokens[i] = MASK_CODE;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Creates a Reverb Text-to-Speech pipeline: speech in the voice of a short
|
|
311
|
+
* recording, in 7 languages.
|
|
312
|
+
* @category Speech / Tasks
|
|
313
|
+
* @typeParam K Preset voice keys (inferred from `config.voices`).
|
|
314
|
+
* @param config Paths to the programs, tokenizer, and voices. See
|
|
315
|
+
* {@link ReverbTtsModel}.
|
|
316
|
+
* @param runtime Optional worklet runtime to run inference on.
|
|
317
|
+
* @returns The {@link ReverbTextToSpeech} runner.
|
|
318
|
+
* @throws {RnExecuTorchError} With code `LOAD_FAILED` if an asset fails to
|
|
319
|
+
* load, or `SCHEMA_MISMATCH` if a program does not match Reverb's.
|
|
320
|
+
*/
|
|
321
|
+
export async function createReverbTextToSpeech<K extends PropertyKey = never>(
|
|
322
|
+
config: ReverbTtsModel<K>,
|
|
323
|
+
runtime?: WorkletRuntime
|
|
324
|
+
): Promise<ReverbTextToSpeech<K>> {
|
|
325
|
+
const scope = createResourceScope();
|
|
326
|
+
const dispose = (): void => {
|
|
327
|
+
scope.dispose();
|
|
328
|
+
};
|
|
329
|
+
|
|
330
|
+
try {
|
|
331
|
+
const load = wrapAsync(loadModel, runtime);
|
|
332
|
+
const backbone = scope.track(
|
|
333
|
+
createReverbBackbone(scope.track(await load(config.modelPaths.backbone)))
|
|
334
|
+
);
|
|
335
|
+
const decoder = scope.track(await load(config.modelPaths.codecDecoder));
|
|
336
|
+
const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(config.tokenizerPath));
|
|
337
|
+
|
|
338
|
+
const { promptTokens, referenceFrames, targetFrames } = backbone.limits;
|
|
339
|
+
const openBackbone = backbone.open;
|
|
340
|
+
const [decodedFrames] = validateSpec(decoder.schema, { default: DECODER_SPEC }).dims.range('T');
|
|
341
|
+
const minFrames = Math.max(MIN_FRAMES, targetFrames.min, decodedFrames.min);
|
|
342
|
+
const maxFrames = Math.min(targetFrames.max, decodedFrames.max);
|
|
343
|
+
|
|
344
|
+
// A voice the programs take: [8, frames] codes in range, a transcript, and a supported language.
|
|
345
|
+
const validateVoice = (
|
|
346
|
+
voice: ReverbVoice,
|
|
347
|
+
code: 'INVALID_ARGUMENT' | 'LOAD_FAILED',
|
|
348
|
+
ctx: string
|
|
349
|
+
): ReverbVoice => {
|
|
350
|
+
const frames = voice.codes.length / CODEBOOKS;
|
|
351
|
+
const valid =
|
|
352
|
+
Number.isInteger(frames) &&
|
|
353
|
+
frames >= referenceFrames.min &&
|
|
354
|
+
frames <= referenceFrames.max &&
|
|
355
|
+
voice.codes.every((c) => c >= 0n && c < MASK_CODE) &&
|
|
356
|
+
voice.transcript.trim().length > 0 &&
|
|
357
|
+
REVERB_LANGUAGES.includes(voice.language) &&
|
|
358
|
+
(voice.speechFrames === undefined ||
|
|
359
|
+
(Number.isInteger(voice.speechFrames) && voice.speechFrames > 0));
|
|
360
|
+
if (!valid) {
|
|
361
|
+
throw RnExecuTorchError(
|
|
362
|
+
code,
|
|
363
|
+
`${ctx}: A voice holds [${CODEBOOKS}, frames] codes below ${CODES}, with ` +
|
|
364
|
+
`${referenceFrames.min} to ${referenceFrames.max} frames, a transcript, a ` +
|
|
365
|
+
`supported language, and, if given, a positive integer of speech frames.`
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
return voice;
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
const voices = {} as Record<K, ReverbVoice>;
|
|
372
|
+
for (const [key, path] of Object.entries(config.voices ?? {}) as [K, string][]) {
|
|
373
|
+
const json = JSON.parse(await RNBlobUtil.fs.readFile(path, 'utf8'));
|
|
374
|
+
voices[key] = validateVoice(parseVoice(json), 'LOAD_FAILED', `Voice '${String(key)}'`);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const synthesizeChunkWorklet = (
|
|
378
|
+
ids: Int32Array,
|
|
379
|
+
voice: ReverbVoice,
|
|
380
|
+
frames: number,
|
|
381
|
+
chunkOpts: {
|
|
382
|
+
steps: number;
|
|
383
|
+
remaskRatio: number;
|
|
384
|
+
seed: number;
|
|
385
|
+
postprocess: Required<ReverbPostprocessOptions>;
|
|
386
|
+
pause: number;
|
|
387
|
+
}
|
|
388
|
+
): Float32Array => {
|
|
389
|
+
'worklet';
|
|
390
|
+
|
|
391
|
+
const { steps, remaskRatio, seed, postprocess, pause } = chunkOpts;
|
|
392
|
+
const slots = 2 * frames;
|
|
393
|
+
const tokens = new BigInt64Array(CODEBOOKS * frames).fill(MASK_CODE);
|
|
394
|
+
const pass = {
|
|
395
|
+
confidence: new Float32Array(slots),
|
|
396
|
+
prediction: new BigInt64Array(slots),
|
|
397
|
+
held: new Float32Array(slots),
|
|
398
|
+
};
|
|
399
|
+
const schedule = commitSchedule(steps);
|
|
400
|
+
const random = mulberry32(seed);
|
|
401
|
+
|
|
402
|
+
// The chunk's session and tensors, released together when it ends
|
|
403
|
+
const local = createResourceScope();
|
|
404
|
+
|
|
405
|
+
try {
|
|
406
|
+
// 1. Prefill the prompt and the reference
|
|
407
|
+
const session = local.track(openBackbone(ids, voice.codes, frames));
|
|
408
|
+
|
|
409
|
+
// 2. Main passes over codebooks 0-1
|
|
410
|
+
for (let step = 0; step < steps; step++) {
|
|
411
|
+
session.step(tokens, pass);
|
|
412
|
+
samplePass(tokens, frames, pass, step, schedule, remaskRatio, random);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// 3. Polish codebooks 2-7
|
|
416
|
+
session.polish(tokens);
|
|
417
|
+
|
|
418
|
+
// 4. Decode
|
|
419
|
+
const tCodes = local.track(tensor('int64', [CODEBOOKS, frames], tokens));
|
|
420
|
+
const tAudio = local.track(tensor('float32', [frames * FRAME_SAMPLES]));
|
|
421
|
+
decoder.execute('forward', [tCodes], [tAudio]);
|
|
422
|
+
let audio: Float32Array = tAudio.getData(new Float32Array(frames * FRAME_SAMPLES));
|
|
423
|
+
|
|
424
|
+
// 5. Cut the end, then trim the silent edges
|
|
425
|
+
if (postprocess.enabled) {
|
|
426
|
+
const { endCut, silenceThreshold, keep } = postprocess;
|
|
427
|
+
const end = Math.max(0, audio.length - Math.floor(endCut * REVERB_SAMPLE_RATE));
|
|
428
|
+
audio = trimSilence(audio.subarray(0, end), REVERB_SAMPLE_RATE, silenceThreshold, keep);
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// 6. Append the pause the chunk's final punctuation calls for
|
|
432
|
+
const result = new Float32Array(audio.length + Math.round(pause * REVERB_SAMPLE_RATE));
|
|
433
|
+
result.set(audio);
|
|
434
|
+
return result;
|
|
435
|
+
} finally {
|
|
436
|
+
local.dispose();
|
|
437
|
+
}
|
|
438
|
+
};
|
|
439
|
+
|
|
440
|
+
const synthesizeChunk = wrapAsync(synthesizeChunkWorklet, runtime);
|
|
441
|
+
|
|
442
|
+
const checkLanguage = (language: unknown, ctx: string): void => {
|
|
443
|
+
if (!REVERB_LANGUAGES.includes(language as ReverbLanguage)) {
|
|
444
|
+
throw RnExecuTorchError(
|
|
445
|
+
'INVALID_ARGUMENT',
|
|
446
|
+
`${ctx}: Unsupported language: ${String(language)}.`
|
|
447
|
+
);
|
|
448
|
+
}
|
|
449
|
+
};
|
|
450
|
+
|
|
451
|
+
const resolveVoice = (voice: K | ReverbVoice): ReverbVoice => {
|
|
452
|
+
const resolved = typeof voice === 'object' ? voice : voices[voice];
|
|
453
|
+
if (!resolved) {
|
|
454
|
+
throw RnExecuTorchError('INVALID_ARGUMENT', `synthesize: Unknown voice: ${String(voice)}.`);
|
|
455
|
+
}
|
|
456
|
+
return validateVoice(resolved, 'INVALID_ARGUMENT', 'synthesize');
|
|
457
|
+
};
|
|
458
|
+
|
|
459
|
+
let encoder:
|
|
460
|
+
| Promise<{ encode: (audio: Float32Array) => Promise<BigInt64Array>; frames: Range }>
|
|
461
|
+
| undefined;
|
|
462
|
+
const loadEncoder = async () => {
|
|
463
|
+
if (!config.modelPaths.codecEncoder) {
|
|
464
|
+
throw RnExecuTorchError('INVALID_STATE', 'encodeVoice: The config names no codecEncoder.');
|
|
465
|
+
}
|
|
466
|
+
const model = await load(config.modelPaths.codecEncoder);
|
|
467
|
+
try {
|
|
468
|
+
const [frames] = validateSpec(model.schema, { default: ENCODER_SPEC }).dims.range('F');
|
|
469
|
+
scope.track(model);
|
|
470
|
+
const encodeWorklet = (audio: Float32Array): BigInt64Array => {
|
|
471
|
+
'worklet';
|
|
472
|
+
const count = audio.length / FRAME_SAMPLES;
|
|
473
|
+
const tAudio = tensor('float32', [count, FRAME_SAMPLES], audio);
|
|
474
|
+
const tCodes = tensor('int64', [CODEBOOKS, count]);
|
|
475
|
+
try {
|
|
476
|
+
model.execute('forward', [tAudio], [tCodes]);
|
|
477
|
+
return tCodes.getData(new BigInt64Array(CODEBOOKS * count));
|
|
478
|
+
} finally {
|
|
479
|
+
tAudio.dispose();
|
|
480
|
+
tCodes.dispose();
|
|
481
|
+
}
|
|
482
|
+
};
|
|
483
|
+
return { encode: wrapAsync(encodeWorklet, runtime), frames };
|
|
484
|
+
} catch (error) {
|
|
485
|
+
model.dispose();
|
|
486
|
+
throw error;
|
|
487
|
+
}
|
|
488
|
+
};
|
|
489
|
+
|
|
490
|
+
const encodeVoice = async (
|
|
491
|
+
audio: Float32Array,
|
|
492
|
+
transcript: string,
|
|
493
|
+
language: ReverbLanguage
|
|
494
|
+
): Promise<ReverbVoice> => {
|
|
495
|
+
checkLanguage(language, 'encodeVoice');
|
|
496
|
+
if (!transcript.trim()) {
|
|
497
|
+
throw RnExecuTorchError('INVALID_ARGUMENT', 'encodeVoice: The transcript cannot be empty.');
|
|
498
|
+
}
|
|
499
|
+
encoder ??= loadEncoder().catch((error) => {
|
|
500
|
+
encoder = undefined;
|
|
501
|
+
throw error;
|
|
502
|
+
});
|
|
503
|
+
const { encode, frames } = await encoder;
|
|
504
|
+
|
|
505
|
+
const count = Math.floor(audio.length / FRAME_SAMPLES);
|
|
506
|
+
const min = Math.max(frames.min, referenceFrames.min);
|
|
507
|
+
const max = Math.min(frames.max, referenceFrames.max);
|
|
508
|
+
if (count < min || count > max) {
|
|
509
|
+
throw RnExecuTorchError(
|
|
510
|
+
'INVALID_ARGUMENT',
|
|
511
|
+
`encodeVoice: The audio must last ${min / REVERB_FRAME_RATE} to ` +
|
|
512
|
+
`${max / REVERB_FRAME_RATE} s at ${REVERB_SAMPLE_RATE} Hz ` +
|
|
513
|
+
`(got ${audio.length / REVERB_SAMPLE_RATE} s).`
|
|
514
|
+
);
|
|
515
|
+
}
|
|
516
|
+
const clip = audio.slice(0, count * FRAME_SAMPLES);
|
|
517
|
+
const codes = await encode(clip);
|
|
518
|
+
// The clip stays whole; only its speaking time is measured, for the length estimate.
|
|
519
|
+
const speechFrames = roundHalfEven(
|
|
520
|
+
speechLength(clip, REVERB_SAMPLE_RATE) * REVERB_FRAME_RATE
|
|
521
|
+
);
|
|
522
|
+
|
|
523
|
+
return { codes, transcript: prepareText(transcript, language), language, speechFrames };
|
|
524
|
+
};
|
|
525
|
+
|
|
526
|
+
let isSynthesizing = false;
|
|
527
|
+
const synthesizeStop = (): void => {
|
|
528
|
+
isSynthesizing = false;
|
|
529
|
+
};
|
|
530
|
+
|
|
531
|
+
async function* synthesize(
|
|
532
|
+
text: string,
|
|
533
|
+
options: ReverbTtsOptions<K>
|
|
534
|
+
): AsyncGenerator<ReverbTtsChunk> {
|
|
535
|
+
if (isSynthesizing) {
|
|
536
|
+
throw RnExecuTorchError('INVALID_STATE', 'synthesize: Synthesis is already in progress.');
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
if (!text || !text.trim()) {
|
|
540
|
+
throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: Input text cannot be empty.');
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
const { language } = options;
|
|
544
|
+
checkLanguage(language, 'synthesize');
|
|
545
|
+
const voice = resolveVoice(options.voice);
|
|
546
|
+
const references = voice.codes.length / CODEBOOKS;
|
|
547
|
+
const pace = options.pace ?? 'speech';
|
|
548
|
+
if (pace !== 'speech' && pace !== 'recording') {
|
|
549
|
+
throw RnExecuTorchError(
|
|
550
|
+
'INVALID_ARGUMENT',
|
|
551
|
+
"synthesize: pace must be 'speech' or 'recording'."
|
|
552
|
+
);
|
|
553
|
+
}
|
|
554
|
+
// The reference length the estimate follows; the model still takes every reference frame.
|
|
555
|
+
const paceFrames =
|
|
556
|
+
pace === 'speech' ? Math.min(voice.speechFrames ?? references, references) : references;
|
|
557
|
+
|
|
558
|
+
const speed = options.speed ?? DEFAULT_SPEED;
|
|
559
|
+
if (!(speed >= MIN_SPEED && speed <= MAX_SPEED)) {
|
|
560
|
+
throw RnExecuTorchError(
|
|
561
|
+
'INVALID_ARGUMENT',
|
|
562
|
+
`synthesize: speed must be between ${MIN_SPEED} and ${MAX_SPEED}.`
|
|
563
|
+
);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// A chunk at the limit must still fit the backbone's target window.
|
|
567
|
+
const maxDuration = maxFrames / REVERB_FRAME_RATE;
|
|
568
|
+
const maxChunkDuration =
|
|
569
|
+
options.maxChunkDuration ?? Math.min(DEFAULT_MAX_CHUNK_DURATION, maxDuration);
|
|
570
|
+
if (!(maxChunkDuration > 0 && maxChunkDuration <= maxDuration)) {
|
|
571
|
+
throw RnExecuTorchError(
|
|
572
|
+
'INVALID_ARGUMENT',
|
|
573
|
+
`synthesize: maxChunkDuration must be above 0 and at most ${maxDuration} s.`
|
|
574
|
+
);
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
const steps = options.sampling?.steps ?? DEFAULT_STEPS;
|
|
578
|
+
const remaskRatio = options.sampling?.remaskRatio ?? DEFAULT_REMASK_RATIO;
|
|
579
|
+
const seed = options.sampling?.seed ?? Math.floor(Math.random() * 0x100000000);
|
|
580
|
+
if (!Number.isInteger(steps) || steps <= 0) {
|
|
581
|
+
throw RnExecuTorchError(
|
|
582
|
+
'INVALID_ARGUMENT',
|
|
583
|
+
'synthesize: steps must be a positive integer.'
|
|
584
|
+
);
|
|
585
|
+
}
|
|
586
|
+
if (!(remaskRatio >= 0 && remaskRatio <= 1)) {
|
|
587
|
+
throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: remaskRatio must be in [0, 1].');
|
|
588
|
+
}
|
|
589
|
+
if (!Number.isInteger(seed)) {
|
|
590
|
+
throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: seed must be an integer.');
|
|
591
|
+
}
|
|
592
|
+
const postprocess = {
|
|
593
|
+
enabled: options.postprocess?.enabled ?? DEFAULT_POSTPROCESS.enabled,
|
|
594
|
+
endCut: options.postprocess?.endCut ?? DEFAULT_POSTPROCESS.endCut,
|
|
595
|
+
silenceThreshold:
|
|
596
|
+
options.postprocess?.silenceThreshold ?? DEFAULT_POSTPROCESS.silenceThreshold,
|
|
597
|
+
keep: options.postprocess?.keep ?? DEFAULT_POSTPROCESS.keep,
|
|
598
|
+
};
|
|
599
|
+
if (postprocess.enabled) {
|
|
600
|
+
const { endCut, silenceThreshold, keep } = postprocess;
|
|
601
|
+
if (!(Number.isFinite(endCut) && endCut >= 0)) {
|
|
602
|
+
throw RnExecuTorchError(
|
|
603
|
+
'INVALID_ARGUMENT',
|
|
604
|
+
'synthesize: postprocess.endCut must be a finite number of seconds, 0 or more.'
|
|
605
|
+
);
|
|
606
|
+
}
|
|
607
|
+
if (!Number.isFinite(silenceThreshold)) {
|
|
608
|
+
throw RnExecuTorchError(
|
|
609
|
+
'INVALID_ARGUMENT',
|
|
610
|
+
'synthesize: postprocess.silenceThreshold must be a finite level in dBFS.'
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
if (!(Number.isFinite(keep) && keep >= 0)) {
|
|
614
|
+
throw RnExecuTorchError(
|
|
615
|
+
'INVALID_ARGUMENT',
|
|
616
|
+
'synthesize: postprocess.keep must be a finite number of seconds, 0 or more.'
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
const encodePrompt = (chunk: string) =>
|
|
622
|
+
tokenizer.encode(buildPrompt(language, voice.transcript, chunk));
|
|
623
|
+
if (encodePrompt('').length >= promptTokens.max) {
|
|
624
|
+
throw RnExecuTorchError(
|
|
625
|
+
'INVALID_ARGUMENT',
|
|
626
|
+
`synthesize: The voice's transcript leaves no room for text in the ` +
|
|
627
|
+
`${promptTokens.max}-token prompt.`
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
const targetLength = (chunk: string) =>
|
|
631
|
+
estimateTargetFrames(chunk, voice.transcript, paceFrames) / speed;
|
|
632
|
+
|
|
633
|
+
isSynthesizing = true;
|
|
634
|
+
try {
|
|
635
|
+
const chunks = partitionReverbText(
|
|
636
|
+
prepareText(text, language),
|
|
637
|
+
(chunk) => targetLength(chunk) / REVERB_FRAME_RATE,
|
|
638
|
+
{
|
|
639
|
+
maxDuration: maxChunkDuration,
|
|
640
|
+
fits: (chunk) => encodePrompt(chunk).length <= promptTokens.max,
|
|
641
|
+
}
|
|
642
|
+
);
|
|
643
|
+
|
|
644
|
+
for (const [chunkIndex, chunk] of chunks.entries()) {
|
|
645
|
+
if (!isSynthesizing) break;
|
|
646
|
+
|
|
647
|
+
const ids = encodePrompt(chunk);
|
|
648
|
+
if (ids.length > promptTokens.max) {
|
|
649
|
+
throw RnExecuTorchError(
|
|
650
|
+
'INVALID_ARGUMENT',
|
|
651
|
+
`synthesize: "${chunk}" does not fit the ${promptTokens.max}-token prompt.`
|
|
652
|
+
);
|
|
653
|
+
}
|
|
654
|
+
const frames = Math.min(
|
|
655
|
+
Math.max(roundHalfEven(targetLength(chunk)), minFrames),
|
|
656
|
+
maxFrames
|
|
657
|
+
);
|
|
658
|
+
const audio = await synthesizeChunk(ids, voice, frames, {
|
|
659
|
+
steps,
|
|
660
|
+
remaskRatio,
|
|
661
|
+
seed: seed + chunkIndex,
|
|
662
|
+
postprocess,
|
|
663
|
+
pause: pauseAfter(chunk),
|
|
664
|
+
});
|
|
665
|
+
yield {
|
|
666
|
+
audio,
|
|
667
|
+
sampleRate: REVERB_SAMPLE_RATE,
|
|
668
|
+
duration: audio.length / REVERB_SAMPLE_RATE,
|
|
669
|
+
chunkIndex,
|
|
670
|
+
totalChunks: chunks.length,
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
} finally {
|
|
674
|
+
isSynthesizing = false;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
return { dispose, encodeVoice, synthesize, synthesizeStop };
|
|
679
|
+
} catch (error) {
|
|
680
|
+
dispose();
|
|
681
|
+
throw error;
|
|
682
|
+
}
|
|
683
|
+
}
|