react-native-executorch 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/CMakeLists.txt +13 -0
- package/android/build.gradle.kts +14 -3
- package/android/src/main/AndroidManifest.xml +6 -0
- package/android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchModule.kt +22 -2
- package/cpp/core/dtype.cpp +12 -1
- package/cpp/core/dtype.h +2 -1
- package/cpp/core/install.cpp +1 -0
- package/cpp/core/utils.cpp +72 -21
- package/cpp/core/utils.h +10 -0
- package/cpp/extensions/cv/utils.h +1 -0
- package/ios/PrivacyInfo.xcprivacy +36 -0
- package/lib/module/constants.js +11 -0
- package/lib/module/constants.js.map +1 -1
- package/lib/module/core/lifetime.js +6 -0
- package/lib/module/core/lifetime.js.map +1 -1
- package/lib/module/core/schema.js +7 -0
- package/lib/module/core/schema.js.map +1 -1
- package/lib/module/core/tensor.js.map +1 -1
- package/lib/module/extensions/cv/ops/box.js +3 -3
- package/lib/module/extensions/cv/ops/box.js.map +1 -1
- package/lib/module/extensions/cv/ops/image.js +10 -14
- package/lib/module/extensions/cv/ops/image.js.map +1 -1
- package/lib/module/extensions/cv/ops/quad.js +2 -2
- package/lib/module/extensions/cv/ops/quad.js.map +1 -1
- package/lib/module/extensions/cv/tasks/sdxsTextToImage.js +11 -4
- package/lib/module/extensions/cv/tasks/sdxsTextToImage.js.map +1 -1
- package/lib/module/extensions/cv/utils/imagePreprocessor.js +3 -0
- package/lib/module/extensions/cv/utils/imagePreprocessor.js.map +1 -1
- package/lib/module/extensions/cv/utils/paddleOcrUtils.js +2 -2
- package/lib/module/extensions/cv/utils/paddleOcrUtils.js.map +1 -1
- package/lib/module/extensions/llm/sampler.js +21 -7
- package/lib/module/extensions/llm/sampler.js.map +1 -1
- package/lib/module/extensions/llm/tasks/llmChatSession.js +4 -1
- package/lib/module/extensions/llm/tasks/llmChatSession.js.map +1 -1
- package/lib/module/extensions/speech/index.js +2 -0
- package/lib/module/extensions/speech/index.js.map +1 -1
- package/lib/module/extensions/speech/tasks/fsmnVoiceActivityDetection.js +5 -2
- package/lib/module/extensions/speech/tasks/fsmnVoiceActivityDetection.js.map +1 -1
- package/lib/module/extensions/speech/tasks/reverbTextToSpeech.js +461 -0
- package/lib/module/extensions/speech/tasks/reverbTextToSpeech.js.map +1 -0
- package/lib/module/extensions/speech/utils/reverbBackbone.js +275 -0
- package/lib/module/extensions/speech/utils/reverbBackbone.js.map +1 -0
- package/lib/module/extensions/speech/utils/reverbPartitioner.js +86 -0
- package/lib/module/extensions/speech/utils/reverbPartitioner.js.map +1 -0
- package/lib/module/extensions/speech/utils/reverbUtils.js +240 -0
- package/lib/module/extensions/speech/utils/reverbUtils.js.map +1 -0
- package/lib/module/extensions/speech/utils/textPartitioner.js +1 -1
- package/lib/module/fetcher/telemetry.js +1 -1
- package/lib/module/hooks/useTextToSpeech.js +22 -1
- package/lib/module/hooks/useTextToSpeech.js.map +1 -1
- package/lib/module/index.js +1 -0
- package/lib/module/index.js.map +1 -1
- package/lib/module/models.js +391 -43
- package/lib/module/models.js.map +1 -1
- package/lib/module/native/bridge.js +18 -0
- package/lib/module/native/bridge.js.map +1 -1
- package/lib/typescript/src/constants.d.ts +10 -0
- package/lib/typescript/src/constants.d.ts.map +1 -1
- package/lib/typescript/src/core/lifetime.d.ts.map +1 -1
- package/lib/typescript/src/core/schema.d.ts +7 -0
- package/lib/typescript/src/core/schema.d.ts.map +1 -1
- package/lib/typescript/src/core/tensor.d.ts +7 -6
- package/lib/typescript/src/core/tensor.d.ts.map +1 -1
- package/lib/typescript/src/extensions/cv/ops/image.d.ts.map +1 -1
- package/lib/typescript/src/extensions/cv/tasks/sdxsTextToImage.d.ts.map +1 -1
- package/lib/typescript/src/extensions/cv/utils/imagePreprocessor.d.ts.map +1 -1
- package/lib/typescript/src/extensions/llm/sampler.d.ts.map +1 -1
- package/lib/typescript/src/extensions/llm/tasks/llmChatSession.d.ts.map +1 -1
- package/lib/typescript/src/extensions/speech/index.d.ts +2 -0
- package/lib/typescript/src/extensions/speech/index.d.ts.map +1 -1
- package/lib/typescript/src/extensions/speech/tasks/fsmnVoiceActivityDetection.d.ts.map +1 -1
- package/lib/typescript/src/extensions/speech/tasks/reverbTextToSpeech.d.ts +175 -0
- package/lib/typescript/src/extensions/speech/tasks/reverbTextToSpeech.d.ts.map +1 -0
- package/lib/typescript/src/extensions/speech/utils/reverbBackbone.d.ts +67 -0
- package/lib/typescript/src/extensions/speech/utils/reverbBackbone.d.ts.map +1 -0
- package/lib/typescript/src/extensions/speech/utils/reverbPartitioner.d.ts +27 -0
- package/lib/typescript/src/extensions/speech/utils/reverbPartitioner.d.ts.map +1 -0
- package/lib/typescript/src/extensions/speech/utils/reverbUtils.d.ts +83 -0
- package/lib/typescript/src/extensions/speech/utils/reverbUtils.d.ts.map +1 -0
- package/lib/typescript/src/hooks/useTextToSpeech.d.ts +30 -0
- package/lib/typescript/src/hooks/useTextToSpeech.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +1 -0
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/models.d.ts +56 -0
- package/lib/typescript/src/models.d.ts.map +1 -1
- package/package.json +4 -3
- package/react-native-executorch.podspec +5 -0
- package/scripts/download-libs.js +31 -6
- package/src/constants.ts +29 -0
- package/src/core/lifetime.ts +3 -0
- package/src/core/schema.ts +7 -0
- package/src/core/tensor.ts +9 -6
- package/src/extensions/cv/ops/box.ts +3 -3
- package/src/extensions/cv/ops/image.ts +11 -12
- package/src/extensions/cv/ops/quad.ts +2 -2
- package/src/extensions/cv/tasks/sdxsTextToImage.ts +10 -4
- package/src/extensions/cv/utils/imagePreprocessor.ts +3 -0
- package/src/extensions/cv/utils/paddleOcrUtils.ts +2 -2
- package/src/extensions/llm/sampler.ts +17 -3
- package/src/extensions/llm/tasks/llmChatSession.ts +5 -1
- package/src/extensions/speech/index.ts +2 -0
- package/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +7 -1
- package/src/extensions/speech/tasks/reverbTextToSpeech.ts +683 -0
- package/src/extensions/speech/utils/reverbBackbone.ts +342 -0
- package/src/extensions/speech/utils/reverbPartitioner.ts +95 -0
- package/src/extensions/speech/utils/reverbUtils.ts +251 -0
- package/src/extensions/speech/utils/textPartitioner.ts +1 -1
- package/src/fetcher/telemetry.ts +1 -1
- package/src/hooks/useTextToSpeech.ts +45 -3
- package/src/index.ts +1 -0
- package/src/models.ts +369 -8
- package/src/native/bridge.ts +21 -0
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Reverb backbone behind one interface.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline works a chunk through a session: `open` prefills the prompt and
|
|
5
|
+
* the voice, `step` runs one main pass over codebooks 0-1, and `polish` fills
|
|
6
|
+
* codebooks 2-7. Which model calls that takes depends on how the program was
|
|
7
|
+
* exported, so the program's schema picks the backbone that makes them.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { Model } from '../../../core/model';
|
|
11
|
+
import { tensor, type Tensor } from '../../../core/tensor';
|
|
12
|
+
import {
|
|
13
|
+
validateSpec,
|
|
14
|
+
method,
|
|
15
|
+
i32,
|
|
16
|
+
i64,
|
|
17
|
+
f16,
|
|
18
|
+
f32,
|
|
19
|
+
DynamicDim as Dyn,
|
|
20
|
+
constraint,
|
|
21
|
+
type DimRef,
|
|
22
|
+
type Range,
|
|
23
|
+
type SpecMatch,
|
|
24
|
+
} from '../../../core/schema';
|
|
25
|
+
|
|
26
|
+
// Shared constants
|
|
27
|
+
export const CODEBOOKS = 8;
|
|
28
|
+
export const LATE_CODEBOOKS = CODEBOOKS - 4; // codebooks 4-7, which polish guides with a prior
|
|
29
|
+
export const CODES = 1024; // per codebook; the next id marks a masked slot
|
|
30
|
+
export const MASK_CODE = BigInt(CODES);
|
|
31
|
+
// K/V cache: [keys and values, layers, KV heads, prefix, head dim]
|
|
32
|
+
const CACHE_LAYERS = 24;
|
|
33
|
+
const CACHE_HEADS = 6;
|
|
34
|
+
const CACHE_HEAD_DIM = 128;
|
|
35
|
+
|
|
36
|
+
/** What one main pass says about codebooks 0-1, `[2, frames]` row-major each. */
|
|
37
|
+
export type ReverbPass = {
|
|
38
|
+
/** Log-probability of the best code per slot. */
|
|
39
|
+
readonly confidence: Float32Array;
|
|
40
|
+
/** The best code per slot. */
|
|
41
|
+
readonly prediction: BigInt64Array;
|
|
42
|
+
/** Log-probability of the code a slot holds; meaningless on masked slots. */
|
|
43
|
+
readonly held: Float32Array;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/** One chunk on the backbone, after its prompt and voice were prefilled. */
|
|
47
|
+
export type ReverbSession = {
|
|
48
|
+
/** Runs one main pass on `tokens` (`[8, frames]`) and writes it to `pass`. */
|
|
49
|
+
readonly step: (tokens: BigInt64Array, pass: ReverbPass) => void;
|
|
50
|
+
/** Writes codebooks 2-7 into `tokens`, whose codebooks 0-1 are complete. */
|
|
51
|
+
readonly polish: (tokens: BigInt64Array) => void;
|
|
52
|
+
/** Releases the chunk's tensors. */
|
|
53
|
+
readonly dispose: () => void;
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/** A loaded backbone program and the sizes it takes. */
|
|
57
|
+
export type ReverbBackbone = {
|
|
58
|
+
readonly limits: {
|
|
59
|
+
readonly promptTokens: Range;
|
|
60
|
+
readonly referenceFrames: Range;
|
|
61
|
+
readonly targetFrames: Range;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* Prefills one chunk. A worklet, to call on the runtime the model runs on.
|
|
65
|
+
* @param ids The prompt's token ids.
|
|
66
|
+
* @param codes The voice's codec codes, `[8, reference frames]` row-major.
|
|
67
|
+
* @param frames The target length.
|
|
68
|
+
* @returns The chunk's session; dispose it when the chunk is done.
|
|
69
|
+
*/
|
|
70
|
+
readonly open: (ids: Int32Array, codes: BigInt64Array, frames: number) => ReverbSession;
|
|
71
|
+
/** Releases the tensors the backbone keeps between chunks. Not the program. */
|
|
72
|
+
readonly dispose: () => void;
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Names one dimension of a method's tensor, for runtime constraints.
|
|
77
|
+
* @param paramSide Whether the tensor is an input or an output.
|
|
78
|
+
* @param tensorIdx The tensor's index among that side's tensors.
|
|
79
|
+
* @param dimIdx The dimension; the frame axis (1) by default.
|
|
80
|
+
* @returns The dimension reference.
|
|
81
|
+
*/
|
|
82
|
+
export const frameDim = (paramSide: 'input' | 'output', tensorIdx: number, dimIdx = 1): DimRef => ({
|
|
83
|
+
paramSide,
|
|
84
|
+
tensorIdx,
|
|
85
|
+
dimIdx,
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
const cache = () => f16(2, CACHE_LAYERS, CACHE_HEADS, Dyn('P'), CACHE_HEAD_DIM);
|
|
89
|
+
|
|
90
|
+
// The MLX and Vulkan programs: every length is dynamic and polish is a method.
|
|
91
|
+
// prettier-ignore
|
|
92
|
+
const DYNAMIC_SPEC = {
|
|
93
|
+
...method(
|
|
94
|
+
'prefill',
|
|
95
|
+
[i64(Dyn('L')), i64(CODEBOOKS, Dyn('R'))], // prompt tokens, reference codes
|
|
96
|
+
[cache()]
|
|
97
|
+
),
|
|
98
|
+
...method(
|
|
99
|
+
'step',
|
|
100
|
+
[i64(CODEBOOKS, Dyn('T')), cache()], // tokens
|
|
101
|
+
[f32(2, Dyn('T')), i64(2, Dyn('T')), f32(2, Dyn('T'))], // confidence, prediction, held
|
|
102
|
+
[constraint.equality(frameDim('input', 0), frameDim('output', 0), frameDim('output', 1), frameDim('output', 2))]
|
|
103
|
+
),
|
|
104
|
+
...method(
|
|
105
|
+
'polish',
|
|
106
|
+
[i64(CODEBOOKS, Dyn('T')), i64(1), cache()], // tokens, reference frames
|
|
107
|
+
[i64(CODEBOOKS - 2, Dyn('T'))], // codebooks 2-7
|
|
108
|
+
[constraint.equality(frameDim('input', 0), frameDim('output', 0))]
|
|
109
|
+
),
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
function dynamicBackbone(model: Model, match: SpecMatch): ReverbBackbone {
|
|
113
|
+
const [promptTokens, referenceFrames, targetFrames] = match.dims.range('L', 'R', 'T');
|
|
114
|
+
|
|
115
|
+
const open = (ids: Int32Array, codes: BigInt64Array, frames: number): ReverbSession => {
|
|
116
|
+
'worklet';
|
|
117
|
+
const references = codes.length / CODEBOOKS;
|
|
118
|
+
const slots = 2 * frames;
|
|
119
|
+
|
|
120
|
+
const tensors: Tensor[] = [];
|
|
121
|
+
const alloc = (...args: Parameters<typeof tensor>): Tensor => {
|
|
122
|
+
const t = tensor(...args);
|
|
123
|
+
tensors.push(t);
|
|
124
|
+
return t;
|
|
125
|
+
};
|
|
126
|
+
const dispose = (): void => {
|
|
127
|
+
tensors.splice(0).forEach((t) => t.dispose());
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
try {
|
|
131
|
+
// The prompt and the reference go into a K/V cache every pass reads.
|
|
132
|
+
const tCache = alloc('float16', [
|
|
133
|
+
2,
|
|
134
|
+
CACHE_LAYERS,
|
|
135
|
+
CACHE_HEADS,
|
|
136
|
+
ids.length + references,
|
|
137
|
+
CACHE_HEAD_DIM,
|
|
138
|
+
]);
|
|
139
|
+
const tIds = alloc('int64', [ids.length], BigInt64Array.from(ids, BigInt));
|
|
140
|
+
const tReference = alloc('int64', [CODEBOOKS, references], codes);
|
|
141
|
+
model.execute('prefill', [tIds, tReference], [tCache]);
|
|
142
|
+
|
|
143
|
+
const tTokens = alloc('int64', [CODEBOOKS, frames]);
|
|
144
|
+
const tConfidence = alloc('float32', [2, frames]);
|
|
145
|
+
const tPrediction = alloc('int64', [2, frames]);
|
|
146
|
+
const tHeld = alloc('float32', [2, frames]);
|
|
147
|
+
const tReferenceFrames = alloc('int64', [1], BigInt64Array.of(BigInt(references)));
|
|
148
|
+
const tFine = alloc('int64', [CODEBOOKS - 2, frames]);
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
step: (tokens, pass) => {
|
|
152
|
+
tTokens.setData(tokens);
|
|
153
|
+
model.execute('step', [tTokens, tCache], [tConfidence, tPrediction, tHeld]);
|
|
154
|
+
tConfidence.getData(pass.confidence);
|
|
155
|
+
tPrediction.getData(pass.prediction);
|
|
156
|
+
tHeld.getData(pass.held);
|
|
157
|
+
},
|
|
158
|
+
polish: (tokens) => {
|
|
159
|
+
tTokens.setData(tokens);
|
|
160
|
+
model.execute('polish', [tTokens, tReferenceFrames, tCache], [tFine]);
|
|
161
|
+
tokens.set(tFine.getData(new BigInt64Array((CODEBOOKS - 2) * frames)), slots);
|
|
162
|
+
},
|
|
163
|
+
dispose,
|
|
164
|
+
};
|
|
165
|
+
} catch (error) {
|
|
166
|
+
dispose();
|
|
167
|
+
throw error;
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
return { limits: { promptTokens, referenceFrames, targetFrames }, open, dispose: () => {} };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// The QNN programs: static shapes and int32 ids. The prompt and the reference
|
|
175
|
+
// are padded into one prefix (text slots first, then reference slots, half
|
|
176
|
+
// each) and the target to the frames of a `step` method, with the real lengths
|
|
177
|
+
// passed alongside.
|
|
178
|
+
// Polish pass has no separate method: it is two `step` calls that say how many reference frames to lean on.
|
|
179
|
+
const cachePieces = () =>
|
|
180
|
+
Array.from({ length: 2 * CACHE_LAYERS }, () => f16(CACHE_HEADS, 'P', CACHE_HEAD_DIM));
|
|
181
|
+
const bucketedSpec = (step: string) => ({
|
|
182
|
+
...method(
|
|
183
|
+
'prefill',
|
|
184
|
+
[i32('P'), i32(CODEBOOKS, 'P'), i32(2)], // prompt tokens, reference codes, their lengths
|
|
185
|
+
cachePieces()
|
|
186
|
+
),
|
|
187
|
+
...method(
|
|
188
|
+
step,
|
|
189
|
+
// tokens, cache, lengths (prompt, reference, target, reference frames to lean on), prior
|
|
190
|
+
[i32(CODEBOOKS, 'T'), ...cachePieces(), i32(4), f16('T', LATE_CODEBOOKS, CODES)],
|
|
191
|
+
// confidence, prediction, held; codebooks 2-7 and the logits of 4-7, for polish
|
|
192
|
+
[
|
|
193
|
+
f32(2, 'T'),
|
|
194
|
+
i32(2, 'T'),
|
|
195
|
+
f32(2, 'T'),
|
|
196
|
+
i32(CODEBOOKS - 2, 'T'),
|
|
197
|
+
f16('T', LATE_CODEBOOKS, CODES),
|
|
198
|
+
]
|
|
199
|
+
),
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
function bucketedBackbone(model: Model, match: SpecMatch): ReverbBackbone {
|
|
203
|
+
const prefix = match.dim('P', 'constant');
|
|
204
|
+
const half = prefix / 2; // text in slots [0, half), reference in [half, prefix)
|
|
205
|
+
// Every `step*` method is the same pass at its own target length.
|
|
206
|
+
const sizes = Object.keys(model.schema)
|
|
207
|
+
.filter((name) => name.startsWith('step'))
|
|
208
|
+
.map((name) => ({
|
|
209
|
+
name,
|
|
210
|
+
frames: validateSpec(model.schema, { bucket: bucketedSpec(name) }).dim('T', 'constant'),
|
|
211
|
+
}))
|
|
212
|
+
.sort((a, b) => a.frames - b.frames);
|
|
213
|
+
|
|
214
|
+
// Shapes never change, so every tensor is made once and serves every chunk.
|
|
215
|
+
const tText = tensor('int32', [prefix]);
|
|
216
|
+
const tReference = tensor('int32', [CODEBOOKS, prefix]);
|
|
217
|
+
const tPrefixLengths = tensor('int32', [2]);
|
|
218
|
+
const tLengths = tensor('int32', [4]);
|
|
219
|
+
const tCache = Array.from({ length: 2 * CACHE_LAYERS }, () =>
|
|
220
|
+
tensor('float16', [CACHE_HEADS, prefix, CACHE_HEAD_DIM])
|
|
221
|
+
);
|
|
222
|
+
const buckets = sizes.map(({ name, frames }) => ({
|
|
223
|
+
name,
|
|
224
|
+
frames,
|
|
225
|
+
tTokens: tensor('int32', [CODEBOOKS, frames]),
|
|
226
|
+
tConfidence: tensor('float32', [2, frames]),
|
|
227
|
+
tPrediction: tensor('int32', [2, frames]),
|
|
228
|
+
tHeld: tensor('float32', [2, frames]),
|
|
229
|
+
tFine: tensor('int32', [CODEBOOKS - 2, frames]),
|
|
230
|
+
// Two, to hand one pass's logits to the next as its prior.
|
|
231
|
+
tLogits: [
|
|
232
|
+
tensor('float16', [frames, LATE_CODEBOOKS, CODES]),
|
|
233
|
+
tensor('float16', [frames, LATE_CODEBOOKS, CODES]),
|
|
234
|
+
] as const,
|
|
235
|
+
}));
|
|
236
|
+
const dispose = (): void => {
|
|
237
|
+
[tText, tReference, tPrefixLengths, tLengths, ...tCache].forEach((t) => t.dispose());
|
|
238
|
+
for (const b of buckets) {
|
|
239
|
+
[b.tTokens, b.tConfidence, b.tPrediction, b.tHeld, b.tFine, ...b.tLogits].forEach((t) =>
|
|
240
|
+
t.dispose()
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
const open = (ids: Int32Array, codes: BigInt64Array, frames: number): ReverbSession => {
|
|
246
|
+
'worklet';
|
|
247
|
+
const references = codes.length / CODEBOOKS;
|
|
248
|
+
const bucket = buckets.find((b) => b.frames >= frames)!;
|
|
249
|
+
const padded = bucket.frames;
|
|
250
|
+
|
|
251
|
+
const text = new Int32Array(prefix);
|
|
252
|
+
text.set(ids);
|
|
253
|
+
const reference = new Int32Array(CODEBOOKS * prefix);
|
|
254
|
+
for (let c = 0; c < CODEBOOKS; c++) {
|
|
255
|
+
for (let r = 0; r < references; r++) {
|
|
256
|
+
reference[c * prefix + half + r] = Number(codes[c * references + r]);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
tText.setData(text);
|
|
260
|
+
tReference.setData(reference);
|
|
261
|
+
tPrefixLengths.setData(Int32Array.of(ids.length, references));
|
|
262
|
+
model.execute('prefill', [tText, tReference, tPrefixLengths], tCache);
|
|
263
|
+
|
|
264
|
+
const padTokens = new Int32Array(CODEBOOKS * padded).fill(CODES);
|
|
265
|
+
const confidence = new Float32Array(2 * padded);
|
|
266
|
+
const prediction = new Int32Array(2 * padded);
|
|
267
|
+
const held = new Float32Array(2 * padded);
|
|
268
|
+
const fine = new Int32Array((CODEBOOKS - 2) * padded);
|
|
269
|
+
|
|
270
|
+
// One pass on `tokens`, with `prior` read and `logits` written.
|
|
271
|
+
const run = (tokens: BigInt64Array, lean: number, prior: Tensor, logits: Tensor): void => {
|
|
272
|
+
for (let c = 0; c < CODEBOOKS; c++) {
|
|
273
|
+
for (let f = 0; f < frames; f++) padTokens[c * padded + f] = Number(tokens[c * frames + f]);
|
|
274
|
+
}
|
|
275
|
+
bucket.tTokens.setData(padTokens);
|
|
276
|
+
tLengths.setData(Int32Array.of(ids.length, references, frames, lean));
|
|
277
|
+
model.execute(
|
|
278
|
+
bucket.name,
|
|
279
|
+
[bucket.tTokens, ...tCache, tLengths, prior],
|
|
280
|
+
[bucket.tConfidence, bucket.tPrediction, bucket.tHeld, bucket.tFine, logits]
|
|
281
|
+
);
|
|
282
|
+
};
|
|
283
|
+
// Rows `from` to `to` of the last pass's codebooks 2-7 into `tokens`.
|
|
284
|
+
const takeFine = (tokens: BigInt64Array, from: number, to: number): void => {
|
|
285
|
+
bucket.tFine.getData(fine);
|
|
286
|
+
for (let c = from; c < to; c++) {
|
|
287
|
+
for (let f = 0; f < frames; f++)
|
|
288
|
+
tokens[(c + 2) * frames + f] = BigInt(fine[c * padded + f]!);
|
|
289
|
+
}
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
return {
|
|
293
|
+
step: (tokens, pass) => {
|
|
294
|
+
run(tokens, 0, bucket.tLogits[0], bucket.tLogits[1]);
|
|
295
|
+
bucket.tConfidence.getData(confidence);
|
|
296
|
+
bucket.tPrediction.getData(prediction);
|
|
297
|
+
bucket.tHeld.getData(held);
|
|
298
|
+
for (let c = 0; c < 2; c++) {
|
|
299
|
+
for (let f = 0; f < frames; f++) {
|
|
300
|
+
pass.confidence[c * frames + f] = confidence[c * padded + f]!;
|
|
301
|
+
pass.prediction[c * frames + f] = BigInt(prediction[c * padded + f]!);
|
|
302
|
+
pass.held[c * frames + f] = held[c * padded + f]!;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
polish: (tokens) => {
|
|
307
|
+
// Codebooks 2-3 come from the first pass, whose logits guide the second's 4-7.
|
|
308
|
+
run(tokens, references, bucket.tLogits[1], bucket.tLogits[0]);
|
|
309
|
+
takeFine(tokens, 0, 2);
|
|
310
|
+
run(tokens, references, bucket.tLogits[0], bucket.tLogits[1]);
|
|
311
|
+
takeFine(tokens, 2, CODEBOOKS - 2);
|
|
312
|
+
},
|
|
313
|
+
dispose: () => {},
|
|
314
|
+
};
|
|
315
|
+
};
|
|
316
|
+
|
|
317
|
+
const size = (max: number): Range => ({ min: 1, max, step: 1 });
|
|
318
|
+
return {
|
|
319
|
+
limits: {
|
|
320
|
+
promptTokens: size(half),
|
|
321
|
+
referenceFrames: size(half),
|
|
322
|
+
targetFrames: size(buckets[buckets.length - 1]!.frames),
|
|
323
|
+
},
|
|
324
|
+
open,
|
|
325
|
+
dispose,
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Wraps a loaded backbone program in the backbone its schema calls for.
|
|
331
|
+
* @param model The backbone program.
|
|
332
|
+
* @returns The backbone.
|
|
333
|
+
* @throws {RnExecuTorchError} With code `SCHEMA_MISMATCH` if the program is
|
|
334
|
+
* not a Reverb backbone.
|
|
335
|
+
*/
|
|
336
|
+
export function createReverbBackbone(model: Model): ReverbBackbone {
|
|
337
|
+
const match = validateSpec(model.schema, {
|
|
338
|
+
dynamic: DYNAMIC_SPEC,
|
|
339
|
+
bucketed: bucketedSpec('step'),
|
|
340
|
+
});
|
|
341
|
+
return { dynamic: dynamicBackbone, bucketed: bucketedBackbone }[match.variant](model, match);
|
|
342
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Splits long text into chunks within a duration limit.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { RnExecuTorchError } from '../../../core/error';
|
|
6
|
+
|
|
7
|
+
const DEFAULT_MAX_DURATION = 20;
|
|
8
|
+
const CLOSERS = /["'”’»)\]}]+$/u; // closing quotes and brackets after a word's punctuation
|
|
9
|
+
|
|
10
|
+
// Punctuation patterns
|
|
11
|
+
const EOS_PATTERN = /[.?!;…|।॥¿¡。.!?;]/;
|
|
12
|
+
const PAUSE_PATTERN = /[,:\u2014\u00AB\u00BB、,:〜~]/;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Options for {@link partitionReverbText}.
|
|
16
|
+
* @category Speech / Types
|
|
17
|
+
*/
|
|
18
|
+
export type ReverbPartitionOptions = {
|
|
19
|
+
/** Max seconds per chunk. Defaults to 20. */
|
|
20
|
+
readonly maxDuration?: number;
|
|
21
|
+
/** A further condition every chunk must meet. */
|
|
22
|
+
readonly fits?: (chunk: string) => boolean;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
// Splits words into runs, each ending at a word whose final mark (past closers) matches `boundary`.
|
|
26
|
+
function splitAfter(words: readonly string[], boundary: RegExp): string[][] {
|
|
27
|
+
const runs: string[][] = [[]];
|
|
28
|
+
for (const word of words) {
|
|
29
|
+
runs[runs.length - 1]!.push(word);
|
|
30
|
+
if (boundary.test(word.replace(CLOSERS, '').slice(-1))) runs.push([]);
|
|
31
|
+
}
|
|
32
|
+
return runs.filter((run) => run.length > 0);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Splits text into chunks of at most `maxDuration` seconds each. Greedy,
|
|
37
|
+
* coarsest boundary first: whole sentences, then other punctuation, then words.
|
|
38
|
+
* A single word that doesn't fit is a chunk of its own.
|
|
39
|
+
* @category Speech / Functions
|
|
40
|
+
* @param text The text.
|
|
41
|
+
* @param duration Estimated seconds of speech of a chunk.
|
|
42
|
+
* @param options See {@link ReverbPartitionOptions}.
|
|
43
|
+
* @returns The chunks, which joined with spaces give back the text.
|
|
44
|
+
* @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `maxDuration` is
|
|
45
|
+
* not positive.
|
|
46
|
+
*/
|
|
47
|
+
export function partitionReverbText(
|
|
48
|
+
text: string,
|
|
49
|
+
duration: (chunk: string) => number,
|
|
50
|
+
options?: ReverbPartitionOptions
|
|
51
|
+
): string[] {
|
|
52
|
+
const maxDuration = options?.maxDuration ?? DEFAULT_MAX_DURATION;
|
|
53
|
+
if (!(maxDuration > 0)) {
|
|
54
|
+
throw RnExecuTorchError(
|
|
55
|
+
'INVALID_ARGUMENT',
|
|
56
|
+
'partitionReverbText: maxDuration must be positive.'
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
const fits = (words: readonly string[]): boolean => {
|
|
60
|
+
const chunk = words.join(' ');
|
|
61
|
+
return duration(chunk) <= maxDuration && (options?.fits?.(chunk) ?? true);
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
// A unit too long on its own is replaced by its pieces at the next finer boundary.
|
|
65
|
+
const boundaries = [EOS_PATTERN, PAUSE_PATTERN];
|
|
66
|
+
const queue = splitAfter(text.split(/\s+/).filter(Boolean), EOS_PATTERN).map((words) => ({
|
|
67
|
+
words,
|
|
68
|
+
level: 0,
|
|
69
|
+
}));
|
|
70
|
+
const chunks: string[] = [];
|
|
71
|
+
let current: string[] = [];
|
|
72
|
+
|
|
73
|
+
while (queue.length > 0) {
|
|
74
|
+
const unit = queue.shift()!;
|
|
75
|
+
if (fits([...current, ...unit.words])) {
|
|
76
|
+
current = [...current, ...unit.words];
|
|
77
|
+
} else if (current.length > 0) {
|
|
78
|
+
chunks.push(current.join(' '));
|
|
79
|
+
current = [];
|
|
80
|
+
queue.unshift(unit);
|
|
81
|
+
} else if (unit.words.length === 1) {
|
|
82
|
+
chunks.push(unit.words[0]!);
|
|
83
|
+
} else {
|
|
84
|
+
let level = unit.level + 1;
|
|
85
|
+
let pieces = level < boundaries.length ? splitAfter(unit.words, boundaries[level]!) : [];
|
|
86
|
+
if (pieces.length < 2) {
|
|
87
|
+
level = boundaries.length;
|
|
88
|
+
pieces = unit.words.map((word) => [word]);
|
|
89
|
+
}
|
|
90
|
+
queue.unshift(...pieces.map((words) => ({ words, level })));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (current.length > 0) chunks.push(current.join(' '));
|
|
94
|
+
return chunks;
|
|
95
|
+
}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text preparation, length estimate, and post-processing utilities for Reverb.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { Verbalizer, type LanguageHandle } from 'verbalis/core';
|
|
6
|
+
import { language as de } from 'verbalis/de';
|
|
7
|
+
import { language as en } from 'verbalis/en';
|
|
8
|
+
import { language as es } from 'verbalis/es';
|
|
9
|
+
import { language as fr } from 'verbalis/fr';
|
|
10
|
+
import { language as it } from 'verbalis/it';
|
|
11
|
+
import { language as pl } from 'verbalis/pl';
|
|
12
|
+
import { language as pt } from 'verbalis/pt';
|
|
13
|
+
|
|
14
|
+
import { RnExecuTorchError } from '../../../core/error';
|
|
15
|
+
import type { ReverbLanguage, ReverbVoice } from '../tasks/reverbTextToSpeech';
|
|
16
|
+
|
|
17
|
+
const BLIP_WINDOWS = 12; // an edge sound of at most 120 ms is a codec burst
|
|
18
|
+
const SILENCE_DB = -50; // a 10 ms window of a leveled clip below this is silent
|
|
19
|
+
const SOUND_GAP_WINDOWS = 20; // loud windows over 200 ms apart are separate sounds
|
|
20
|
+
|
|
21
|
+
const VERBALIS_LANGUAGES: Record<ReverbLanguage, () => LanguageHandle> = {
|
|
22
|
+
de,
|
|
23
|
+
en,
|
|
24
|
+
es,
|
|
25
|
+
fr,
|
|
26
|
+
it,
|
|
27
|
+
pl,
|
|
28
|
+
pt,
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Parses a voice file's JSON: `{ codes, transcript, language }`, the codes as
|
|
33
|
+
* `8 × frames` integers, row-major, with an optional `speech_frames`.
|
|
34
|
+
* @param json The parsed JSON.
|
|
35
|
+
* @returns The voice.
|
|
36
|
+
* @throws {RnExecuTorchError} With code `LOAD_FAILED` on a missing or
|
|
37
|
+
* mistyped field.
|
|
38
|
+
*/
|
|
39
|
+
export function parseVoice(json: any): ReverbVoice {
|
|
40
|
+
const { codes, transcript, language, speech_frames: speechFrames } = json ?? {};
|
|
41
|
+
const valid =
|
|
42
|
+
Array.isArray(codes) &&
|
|
43
|
+
codes.every((code) => Number.isInteger(code)) &&
|
|
44
|
+
typeof transcript === 'string' &&
|
|
45
|
+
typeof language === 'string' &&
|
|
46
|
+
(speechFrames === undefined || (Number.isInteger(speechFrames) && speechFrames > 0));
|
|
47
|
+
if (!valid) {
|
|
48
|
+
throw RnExecuTorchError('LOAD_FAILED', 'parseVoice: Invalid voice JSON format.');
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
codes: BigInt64Array.from(codes as number[], BigInt),
|
|
52
|
+
transcript,
|
|
53
|
+
language: language as ReverbLanguage,
|
|
54
|
+
...(speechFrames === undefined ? {} : { speechFrames }),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Seconds of a voice clip that set the speaker's pace, as Echo's
|
|
60
|
+
* `speech_length` measures them: the clip without its silent edges,
|
|
61
|
+
* with every pause inside the speech counted as at most `maxPause` seconds.
|
|
62
|
+
*
|
|
63
|
+
* Silence is judged in 10 ms windows after leveling the clip to an RMS of 0.1.
|
|
64
|
+
* @param audio The clip.
|
|
65
|
+
* @param sampleRate Its sampling rate in Hz.
|
|
66
|
+
* @param maxPause Seconds an inner pause counts at most.
|
|
67
|
+
* @param lead Seconds of leading silence counted at most.
|
|
68
|
+
* @param trail Seconds of trailing silence counted at most.
|
|
69
|
+
* @returns The speaking time in seconds.
|
|
70
|
+
*/
|
|
71
|
+
export function speechLength(
|
|
72
|
+
audio: Float32Array,
|
|
73
|
+
sampleRate: number,
|
|
74
|
+
maxPause = 0.5,
|
|
75
|
+
lead = 0.1,
|
|
76
|
+
trail = 0.2
|
|
77
|
+
): number {
|
|
78
|
+
const window = Math.floor(sampleRate / 100);
|
|
79
|
+
const windows = Math.floor(audio.length / window);
|
|
80
|
+
if (windows === 0) return audio.length / sampleRate;
|
|
81
|
+
|
|
82
|
+
const energies = new Float64Array(windows);
|
|
83
|
+
let total = 0;
|
|
84
|
+
for (let w = 0; w < windows; w++) {
|
|
85
|
+
let energy = 0;
|
|
86
|
+
for (let i = w * window; i < (w + 1) * window; i++) {
|
|
87
|
+
const sample = audio[i]!;
|
|
88
|
+
energy += sample * sample; // `** 2` makes this loop 2.5x slower in Hermes
|
|
89
|
+
}
|
|
90
|
+
energies[w] = energy;
|
|
91
|
+
total += energy;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Leveled to RMS 0.1: a window is loud when its RMS, times this gain, reaches SILENCE_DB.
|
|
95
|
+
const gain = 0.1 / Math.max(Math.sqrt(total / (windows * window)), 1e-8);
|
|
96
|
+
const cap = roundHalfEven(maxPause * 100);
|
|
97
|
+
let first = -1;
|
|
98
|
+
let last = -1;
|
|
99
|
+
let kept = 0;
|
|
100
|
+
for (let w = 0; w < windows; w++) {
|
|
101
|
+
if (20 * Math.log10(gain * Math.sqrt(energies[w]! / window) + 1e-10) < SILENCE_DB) continue;
|
|
102
|
+
kept += first < 0 ? 1 : 1 + Math.min(w - last - 1, cap);
|
|
103
|
+
if (first < 0) first = w;
|
|
104
|
+
last = w;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (first < 0) return audio.length / sampleRate;
|
|
108
|
+
|
|
109
|
+
const edges =
|
|
110
|
+
Math.min(first, roundHalfEven(lead * 100)) +
|
|
111
|
+
Math.min(windows - 1 - last, roundHalfEven(trail * 100));
|
|
112
|
+
|
|
113
|
+
return (kept + edges) / 100;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Writes text the way it is read aloud with Verbalis: numbers, symbols, and
|
|
118
|
+
* abbreviations as words, and quotes, dashes, ellipses, and spaces in the forms
|
|
119
|
+
* Reverb was trained on. JS thread only.
|
|
120
|
+
* @param text The text.
|
|
121
|
+
* @param language The text's language.
|
|
122
|
+
* @returns The prepared text.
|
|
123
|
+
*/
|
|
124
|
+
export function prepareText(text: string, language: ReverbLanguage): string {
|
|
125
|
+
return new Verbalizer(VERBALIS_LANGUAGES[language]()).process(text);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The backbone's prompt: the text's language, then the voice's transcript and
|
|
130
|
+
* the text.
|
|
131
|
+
* @param language The text's language.
|
|
132
|
+
* @param transcript The voice's transcript.
|
|
133
|
+
* @param text The text to speak.
|
|
134
|
+
* @returns The prompt.
|
|
135
|
+
*/
|
|
136
|
+
export function buildPrompt(language: ReverbLanguage, transcript: string, text: string): string {
|
|
137
|
+
return (
|
|
138
|
+
`<|denoise|><|lang_start|>${language}<|lang_end|><|instruct_start|>None<|instruct_end|>` +
|
|
139
|
+
`<|text_start|>${transcript} ${text}<|text_end|>`
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Spoken weight by Unicode class: spaces, marks, punctuation and symbols, digits, the rest.
|
|
144
|
+
function spokenWeight(text: string): number {
|
|
145
|
+
'worklet';
|
|
146
|
+
let weight = 0;
|
|
147
|
+
for (const char of text) {
|
|
148
|
+
if (/\p{Z}/u.test(char)) weight += 0.2;
|
|
149
|
+
else if (/\p{M}/u.test(char)) weight += 0;
|
|
150
|
+
else if (/[\p{P}\p{S}]/u.test(char)) weight += 0.5;
|
|
151
|
+
else if (/\p{N}/u.test(char)) weight += 3.5;
|
|
152
|
+
else weight += 1;
|
|
153
|
+
}
|
|
154
|
+
return weight;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Estimates the codec frames (25 per second) it takes to speak `text` at a
|
|
159
|
+
* reference clip's pace, as Echo's `estimate_frames` does.
|
|
160
|
+
* @category Speech / Functions
|
|
161
|
+
* @param text The text to speak.
|
|
162
|
+
* @param referenceText The reference clip's transcript.
|
|
163
|
+
* @param referenceFrames The reference clip's length in frames.
|
|
164
|
+
* @returns The estimate, at least 1.
|
|
165
|
+
*/
|
|
166
|
+
export function estimateTargetFrames(
|
|
167
|
+
text: string,
|
|
168
|
+
referenceText: string,
|
|
169
|
+
referenceFrames: number
|
|
170
|
+
): number {
|
|
171
|
+
'worklet';
|
|
172
|
+
let frames = (spokenWeight(text) / Math.max(spokenWeight(referenceText), 1)) * referenceFrames;
|
|
173
|
+
if (frames < 50) frames = 50 * Math.pow(frames / 50, 1 / 3); // short texts get relatively more time
|
|
174
|
+
return Math.max(1, Math.floor(frames));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Rounds half to even, like Python's `round`.
|
|
179
|
+
* @param value The value.
|
|
180
|
+
* @returns The rounded value.
|
|
181
|
+
*/
|
|
182
|
+
export function roundHalfEven(value: number): number {
|
|
183
|
+
'worklet';
|
|
184
|
+
const rounded = Math.round(value);
|
|
185
|
+
return rounded - value === 0.5 && rounded % 2 !== 0 ? rounded - 1 : rounded;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Trims silent edges like Voice Studio: in 10 ms windows, dropping an isolated
|
|
190
|
+
* edge sound of at most 120 ms.
|
|
191
|
+
* @param audio The audio.
|
|
192
|
+
* @param sampleRate Its sampling rate in Hz.
|
|
193
|
+
* @param threshold Level in dBFS below which a window is silent.
|
|
194
|
+
* @param keep Seconds of silence kept at each edge.
|
|
195
|
+
* @returns A view of `audio` without the silent edges, or `audio` if all silent.
|
|
196
|
+
*/
|
|
197
|
+
export function trimSilence(
|
|
198
|
+
audio: Float32Array,
|
|
199
|
+
sampleRate: number,
|
|
200
|
+
threshold: number,
|
|
201
|
+
keep: number
|
|
202
|
+
): Float32Array {
|
|
203
|
+
'worklet';
|
|
204
|
+
const window = Math.floor(sampleRate / 100);
|
|
205
|
+
const windows = Math.floor(audio.length / window);
|
|
206
|
+
const sounds: { first: number; last: number; loud: number }[] = [];
|
|
207
|
+
for (let w = 0; w < windows; w++) {
|
|
208
|
+
let energy = 0;
|
|
209
|
+
for (let i = w * window; i < (w + 1) * window; i++) {
|
|
210
|
+
const sample = audio[i]!;
|
|
211
|
+
energy += sample * sample; // `** 2` makes this loop 2.5x slower in Hermes
|
|
212
|
+
}
|
|
213
|
+
if (20 * Math.log10(Math.sqrt(energy / window) + 1e-10) < threshold) continue;
|
|
214
|
+
const sound = sounds[sounds.length - 1];
|
|
215
|
+
if (sound && w - sound.last <= SOUND_GAP_WINDOWS) {
|
|
216
|
+
sound.last = w;
|
|
217
|
+
sound.loud++;
|
|
218
|
+
} else {
|
|
219
|
+
sounds.push({ first: w, last: w, loud: 1 });
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
if (sounds.length === 0) return audio;
|
|
223
|
+
|
|
224
|
+
if (sounds.length > 1 && sounds[0]!.loud <= BLIP_WINDOWS) sounds.shift();
|
|
225
|
+
if (sounds.length > 1 && sounds[sounds.length - 1]!.loud <= BLIP_WINDOWS) sounds.pop();
|
|
226
|
+
const margin = roundHalfEven(keep * 100);
|
|
227
|
+
const start = Math.max(sounds[0]!.first - margin, 0);
|
|
228
|
+
const end = Math.min(sounds[sounds.length - 1]!.last + 1 + margin, windows);
|
|
229
|
+
return audio.subarray(start * window, end * window);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Pause after a chunk by its final punctuation, in milliseconds: Kokoro's
|
|
234
|
+
* values, `...` taking the ellipsis's.
|
|
235
|
+
*/
|
|
236
|
+
// prettier-ignore
|
|
237
|
+
export const REVERB_PAUSE_MS: Record<string, number> = {
|
|
238
|
+
'.': 375, '?': 500, '!': 250, ';': 400, '...': 1000, ',': 130, ':': 250, '-': 200,
|
|
239
|
+
};
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* The pause after a chunk, by its final punctuation past closing quotes.
|
|
243
|
+
* @param chunk The chunk.
|
|
244
|
+
* @returns The pause in seconds.
|
|
245
|
+
*/
|
|
246
|
+
export function pauseAfter(chunk: string): number {
|
|
247
|
+
'worklet';
|
|
248
|
+
const end = chunk.replace(/[\s"'”’»)\]}]+$/u, '');
|
|
249
|
+
const mark = end.endsWith('...') ? '...' : end.slice(-1);
|
|
250
|
+
return (REVERB_PAUSE_MS[mark] ?? 0) / 1000;
|
|
251
|
+
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import { RnExecuTorchError } from '../../../core/error';
|
|
7
7
|
type Tag = 'eos' | 'pause' | 'whitespace';
|
|
8
8
|
|
|
9
|
-
// Punctuation
|
|
9
|
+
// Punctuation patterns
|
|
10
10
|
const EOS_PATTERN = /[.?!;…|।॥¿¡。.!?;]/;
|
|
11
11
|
const PAUSE_PATTERN = /[,:\u2014\u00AB\u00BB、,:〜~]/;
|
|
12
12
|
const WHITESPACE_PATTERN = /\s/;
|