react-native-executorch 0.5.1-rc.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/README.md +132 -0
  2. package/common/rnexecutorch/models/speech_to_text/SpeechToText.cpp +4 -10
  3. package/common/rnexecutorch/models/speech_to_text/SpeechToText.h +1 -1
  4. package/common/rnexecutorch/models/speech_to_text/SpeechToTextStrategy.h +3 -2
  5. package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.cpp +16 -4
  6. package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.h +2 -2
  7. package/ios/RnExecutorch.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -0
  8. package/ios/RnExecutorch.xcodeproj/project.xcworkspace/xcuserdata/jakubchmura.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
  9. package/ios/RnExecutorch.xcodeproj/xcuserdata/jakubchmura.xcuserdatad/xcschemes/xcschememanagement.plist +14 -0
  10. package/lib/module/constants/modelUrls.js +61 -36
  11. package/lib/module/constants/modelUrls.js.map +1 -1
  12. package/lib/module/constants/ocr/models.js +1 -1
  13. package/lib/module/hooks/natural_language_processing/useSpeechToText.js +71 -34
  14. package/lib/module/hooks/natural_language_processing/useSpeechToText.js.map +1 -1
  15. package/lib/module/index.js +2 -3
  16. package/lib/module/index.js.map +1 -1
  17. package/lib/module/modules/natural_language_processing/SpeechToTextModule.js +72 -31
  18. package/lib/module/modules/natural_language_processing/SpeechToTextModule.js.map +1 -1
  19. package/lib/module/types/stt.js +1 -85
  20. package/lib/module/types/stt.js.map +1 -1
  21. package/lib/module/utils/SpeechToTextModule/ASR.js +191 -0
  22. package/lib/module/utils/SpeechToTextModule/ASR.js.map +1 -0
  23. package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js +73 -0
  24. package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js.map +1 -0
  25. package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js +56 -0
  26. package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js.map +1 -0
  27. package/lib/tsconfig.tsbuildinfo +1 -0
  28. package/lib/typescript/constants/modelUrls.d.ts +24 -7
  29. package/lib/typescript/constants/modelUrls.d.ts.map +1 -1
  30. package/lib/typescript/constants/ocr/models.d.ts +126 -126
  31. package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts +15 -24
  32. package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts.map +1 -1
  33. package/lib/typescript/index.d.ts +2 -3
  34. package/lib/typescript/index.d.ts.map +1 -1
  35. package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts +19 -22
  36. package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts.map +1 -1
  37. package/lib/typescript/types/stt.d.ts +17 -91
  38. package/lib/typescript/types/stt.d.ts.map +1 -1
  39. package/lib/typescript/utils/SpeechToTextModule/ASR.d.ts +27 -0
  40. package/lib/typescript/utils/SpeechToTextModule/ASR.d.ts.map +1 -0
  41. package/lib/typescript/utils/SpeechToTextModule/OnlineProcessor.d.ts +23 -0
  42. package/lib/typescript/utils/SpeechToTextModule/OnlineProcessor.d.ts.map +1 -0
  43. package/lib/typescript/utils/SpeechToTextModule/hypothesisBuffer.d.ts +13 -0
  44. package/lib/typescript/utils/SpeechToTextModule/hypothesisBuffer.d.ts.map +1 -0
  45. package/package.json +5 -3
  46. package/src/constants/modelUrls.ts +70 -37
  47. package/src/constants/ocr/models.ts +1 -1
  48. package/src/hooks/natural_language_processing/useSpeechToText.ts +87 -92
  49. package/src/index.ts +6 -8
  50. package/src/modules/natural_language_processing/SpeechToTextModule.ts +81 -69
  51. package/src/types/stt.ts +97 -92
  52. package/src/utils/SpeechToTextModule/ASR.ts +303 -0
  53. package/src/utils/SpeechToTextModule/OnlineProcessor.ts +87 -0
  54. package/src/utils/SpeechToTextModule/hypothesisBuffer.ts +79 -0
  55. package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/xcuserdata/jakubchmura.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
  56. package/common/rnexecutorch/models/speech_to_text/MoonshineStrategy.cpp +0 -31
  57. package/common/rnexecutorch/models/speech_to_text/MoonshineStrategy.h +0 -21
  58. package/lib/common/Logger.d.ts +0 -8
  59. package/lib/common/Logger.js +0 -19
  60. package/lib/constants/modelUrls.d.ts +0 -89
  61. package/lib/constants/modelUrls.js +0 -116
  62. package/lib/constants/sttDefaults.js +0 -66
  63. package/lib/controllers/LLMController.js +0 -210
  64. package/lib/controllers/OCRController.js +0 -65
  65. package/lib/controllers/SpeechToTextController.d.ts +0 -52
  66. package/lib/controllers/SpeechToTextController.js +0 -343
  67. package/lib/hooks/natural_language_processing/useSpeechToText.js +0 -44
  68. package/lib/index.d.ts +0 -50
  69. package/lib/index.js +0 -59
  70. package/lib/module/constants/sttDefaults.js +0 -74
  71. package/lib/module/constants/sttDefaults.js.map +0 -1
  72. package/lib/module/controllers/SpeechToTextController.js +0 -320
  73. package/lib/module/controllers/SpeechToTextController.js.map +0 -1
  74. package/lib/modules/natural_language_processing/SpeechToTextModule.d.ts +0 -14
  75. package/lib/modules/natural_language_processing/SpeechToTextModule.js +0 -30
  76. package/lib/modules/natural_language_processing/TokenizerModule.js +0 -29
  77. package/lib/native/RnExecutorchModules.d.ts +0 -3
  78. package/lib/native/RnExecutorchModules.js +0 -16
  79. package/lib/typescript/constants/sttDefaults.d.ts +0 -29
  80. package/lib/typescript/constants/sttDefaults.d.ts.map +0 -1
  81. package/lib/typescript/controllers/SpeechToTextController.d.ts +0 -57
  82. package/lib/typescript/controllers/SpeechToTextController.d.ts.map +0 -1
  83. package/lib/utils/ResourceFetcherUtils.js +0 -119
  84. package/lib/utils/llm.js +0 -72
  85. package/src/constants/sttDefaults.ts +0 -82
  86. package/src/controllers/SpeechToTextController.ts +0 -471
  87. package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/xcuserdata/norbertklockiewicz.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
  88. /package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/xcuserdata/{norbertklockiewicz.xcuserdatad → jakubchmura.xcuserdatad}/xcschemes/xcschememanagement.plist +0 -0
@@ -0,0 +1,303 @@
1
+ // NOTE: This will be implemented in C++
2
+
3
+ import { TokenizerModule } from '../../modules/natural_language_processing/TokenizerModule';
4
+ import {
5
+ DecodingOptions,
6
+ Segment,
7
+ SpeechToTextModelConfig,
8
+ WordObject,
9
+ WordTuple,
10
+ } from '../../types/stt';
11
+ import { ResourceFetcher } from '../ResourceFetcher';
12
+
13
+ export class ASR {
14
+ private nativeModule: any;
15
+ private tokenizerModule: TokenizerModule = new TokenizerModule();
16
+
17
+ private timePrecision: number = 0.02; // Whisper timestamp precision
18
+ private maxDecodeLength: number = 128;
19
+ private chunkSize: number = 30; // 30 seconds
20
+ private minChunkSamples: number = 1 * 16000; // 1 second
21
+ private samplingRate: number = 16000;
22
+
23
+ private startOfTranscriptToken!: number;
24
+ private endOfTextToken!: number;
25
+ private timestampBeginToken!: number;
26
+
27
+ public async load(
28
+ model: SpeechToTextModelConfig,
29
+ onDownloadProgressCallback: (progress: number) => void
30
+ ) {
31
+ const tokenizerLoadPromise = this.tokenizerModule.load(model);
32
+ const encoderDecoderPromise = ResourceFetcher.fetch(
33
+ onDownloadProgressCallback,
34
+ model.encoderSource,
35
+ model.decoderSource
36
+ );
37
+ const [_, encoderDecoderResults] = await Promise.all([
38
+ tokenizerLoadPromise,
39
+ encoderDecoderPromise,
40
+ ]);
41
+ const encoderSource = encoderDecoderResults?.[0];
42
+ const decoderSource = encoderDecoderResults?.[1];
43
+ if (!encoderSource || !decoderSource) {
44
+ throw new Error('Download interrupted.');
45
+ }
46
+ this.nativeModule = await global.loadSpeechToText(
47
+ encoderSource,
48
+ decoderSource,
49
+ 'whisper'
50
+ );
51
+
52
+ this.startOfTranscriptToken = await this.tokenizerModule.tokenToId(
53
+ '<|startoftranscript|>'
54
+ );
55
+ this.endOfTextToken = await this.tokenizerModule.tokenToId('<|endoftext|>');
56
+ this.timestampBeginToken = await this.tokenizerModule.tokenToId('<|0.00|>');
57
+ }
58
+
59
+ private async getInitialSequence(
60
+ options: DecodingOptions
61
+ ): Promise<number[]> {
62
+ const initialSequence: number[] = [this.startOfTranscriptToken];
63
+ if (options.language) {
64
+ const languageToken = await this.tokenizerModule.tokenToId(
65
+ `<|${options.language}|>`
66
+ );
67
+ const taskToken = await this.tokenizerModule.tokenToId('<|transcribe|>');
68
+ initialSequence.push(languageToken);
69
+ initialSequence.push(taskToken);
70
+ }
71
+ initialSequence.push(this.timestampBeginToken);
72
+ return initialSequence;
73
+ }
74
+
75
+ private async generate(
76
+ audio: number[],
77
+ temperature: number,
78
+ options: DecodingOptions
79
+ ): Promise<{
80
+ sequencesIds: number[];
81
+ scores: number[];
82
+ }> {
83
+ await this.encode(new Float32Array(audio));
84
+ const initialSequence = await this.getInitialSequence(options);
85
+ const sequencesIds = [...initialSequence];
86
+ const scores: number[] = [];
87
+
88
+ while (sequencesIds.length <= this.maxDecodeLength) {
89
+ const logits = this.softmaxWithTemperature(
90
+ Array.from(await this.decode(sequencesIds)),
91
+ temperature === 0 ? 1 : temperature
92
+ );
93
+ const nextTokenId =
94
+ temperature === 0
95
+ ? logits.indexOf(Math.max(...logits))
96
+ : this.sampleFromDistribution(logits);
97
+ const nextTokenProb = logits[nextTokenId]!;
98
+ sequencesIds.push(nextTokenId);
99
+ scores.push(nextTokenProb);
100
+ if (nextTokenId === this.endOfTextToken) {
101
+ break;
102
+ }
103
+ }
104
+
105
+ return {
106
+ sequencesIds: sequencesIds.slice(initialSequence.length),
107
+ scores: scores.slice(initialSequence.length),
108
+ };
109
+ }
110
+
111
+ private softmaxWithTemperature(logits: number[], temperature = 1.0) {
112
+ const max = Math.max(...logits);
113
+ const exps = logits.map((logit) => Math.exp((logit - max) / temperature));
114
+ const sum = exps.reduce((a, b) => a + b, 0);
115
+ return exps.map((exp) => exp / sum);
116
+ }
117
+
118
+ private sampleFromDistribution(probs: number[]): number {
119
+ const r = Math.random();
120
+ let cumulative = 0;
121
+ for (let i = 0; i < probs.length; i++) {
122
+ cumulative += probs[i]!;
123
+ if (r < cumulative) {
124
+ return i;
125
+ }
126
+ }
127
+ return probs.length - 1;
128
+ }
129
+
130
+ private async generateWithFallback(
131
+ audio: number[],
132
+ options: DecodingOptions
133
+ ) {
134
+ const temperatures = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0];
135
+ let generatedTokens: number[] = [];
136
+
137
+ for (const temperature of temperatures) {
138
+ const result = await this.generate(audio, temperature, options);
139
+ const tokens = result.sequencesIds;
140
+ const scores = result.scores;
141
+
142
+ const seqLen = tokens.length;
143
+ const cumLogProb = scores.reduce(
144
+ (acc, score) => acc + Math.log(score),
145
+ 0
146
+ );
147
+ const avgLogProb = cumLogProb / seqLen;
148
+
149
+ if (avgLogProb >= -1.0) {
150
+ generatedTokens = tokens;
151
+ break;
152
+ }
153
+ }
154
+
155
+ return this.calculateWordLevelTimestamps(generatedTokens, audio);
156
+ }
157
+
158
+ private async calculateWordLevelTimestamps(
159
+ generatedTokens: number[],
160
+ audio: number[]
161
+ ): Promise<Segment[]> {
162
+ const segments: Segment[] = [];
163
+
164
+ let tokens: number[] = [];
165
+ let prevTimestamp = this.timestampBeginToken;
166
+ for (let i = 0; i < generatedTokens.length; i++) {
167
+ if (generatedTokens[i]! < this.timestampBeginToken) {
168
+ tokens.push(generatedTokens[i]!);
169
+ }
170
+
171
+ if (
172
+ i > 0 &&
173
+ generatedTokens[i - 1]! >= this.timestampBeginToken &&
174
+ generatedTokens[i]! >= this.timestampBeginToken
175
+ ) {
176
+ const start = prevTimestamp;
177
+ const end = generatedTokens[i - 1]!;
178
+ const wordObjects = await this.estimateWordTimestampsLinear(
179
+ tokens,
180
+ start,
181
+ end
182
+ );
183
+ segments.push({
184
+ words: wordObjects,
185
+ });
186
+ tokens = [];
187
+ prevTimestamp = generatedTokens[i]!;
188
+ }
189
+ }
190
+
191
+ const start = prevTimestamp;
192
+ const end = generatedTokens.at(-2)!;
193
+ const wordObjects = await this.estimateWordTimestampsLinear(
194
+ tokens,
195
+ start,
196
+ end
197
+ );
198
+ segments.push({
199
+ words: wordObjects,
200
+ });
201
+
202
+ const scalingFactor =
203
+ audio.length /
204
+ this.samplingRate /
205
+ ((end - this.timestampBeginToken) * this.timePrecision);
206
+ if (scalingFactor < 1) {
207
+ for (const segment of segments) {
208
+ for (const word of segment.words) {
209
+ word.start *= scalingFactor;
210
+ word.end *= scalingFactor;
211
+ }
212
+ }
213
+ }
214
+
215
+ return segments;
216
+ }
217
+
218
+ private async estimateWordTimestampsLinear(
219
+ tokens: number[],
220
+ start: number,
221
+ end: number
222
+ ): Promise<WordObject[]> {
223
+ const duration = (end - start) * this.timePrecision;
224
+ const segmentText = (
225
+ (await this.tokenizerModule.decode(tokens)) as string
226
+ ).trim();
227
+
228
+ const words = segmentText.split(' ').map((w) => ` ${w}`);
229
+ const numOfCharacters = words.reduce(
230
+ (acc: number, word: string) => acc + word.length,
231
+ 0
232
+ );
233
+
234
+ const timePerCharacter = duration / numOfCharacters;
235
+
236
+ const wordObjects: WordObject[] = [];
237
+ const startTimeOffset =
238
+ (start - this.timestampBeginToken) * this.timePrecision;
239
+
240
+ let prevCharNum = 0;
241
+ for (let j = 0; j < words.length; j++) {
242
+ const word = words[j]!;
243
+ const start = startTimeOffset + prevCharNum * timePerCharacter;
244
+ const end = start + timePerCharacter * word.length;
245
+ wordObjects.push({ word, start, end });
246
+ prevCharNum += word.length;
247
+ }
248
+
249
+ return wordObjects;
250
+ }
251
+
252
+ public async transcribe(
253
+ audio: number[],
254
+ options: DecodingOptions
255
+ ): Promise<Segment[]> {
256
+ let seek = 0;
257
+ const allSegments: Segment[] = [];
258
+
259
+ while (seek * this.samplingRate < audio.length) {
260
+ const chunk = audio.slice(
261
+ seek * this.samplingRate,
262
+ (seek + this.chunkSize) * this.samplingRate
263
+ );
264
+ if (chunk.length < this.minChunkSamples) {
265
+ return allSegments;
266
+ }
267
+ const segments = await this.generateWithFallback(chunk, options);
268
+ for (const segment of segments) {
269
+ for (const word of segment.words) {
270
+ word.start += seek;
271
+ word.end += seek;
272
+ }
273
+ }
274
+ allSegments.push(...segments);
275
+ const lastTimeStamp = segments.at(-1)!.words.at(-1)!.end;
276
+ seek = lastTimeStamp;
277
+ }
278
+
279
+ return allSegments;
280
+ }
281
+
282
+ public tsWords(segments: Segment[]): WordTuple[] {
283
+ const o: WordTuple[] = [];
284
+ for (const segment of segments) {
285
+ for (const word of segment.words) {
286
+ o.push([word.start, word.end, word.word]);
287
+ }
288
+ }
289
+ return o;
290
+ }
291
+
292
+ public segmentsEndTs(res: Segment[]) {
293
+ return res.map((segment) => segment.words.at(-1)!.end);
294
+ }
295
+
296
+ public async encode(waveform: Float32Array): Promise<void> {
297
+ await this.nativeModule.encode(waveform);
298
+ }
299
+
300
+ public async decode(tokens: number[]): Promise<Float32Array> {
301
+ return new Float32Array(await this.nativeModule.decode(tokens));
302
+ }
303
+ }
@@ -0,0 +1,87 @@
1
+ // NOTE: This will be implemented in C++
2
+
3
+ import { WordTuple, DecodingOptions, Segment } from '../../types/stt';
4
+ import { ASR } from './ASR';
5
+ import { HypothesisBuffer } from './hypothesisBuffer';
6
+
7
+ export class OnlineASRProcessor {
8
+ private asr: ASR;
9
+
10
+ private samplingRate: number = 16000;
11
+ public audioBuffer: number[] = [];
12
+ private transcriptBuffer: HypothesisBuffer = new HypothesisBuffer();
13
+ private bufferTimeOffset: number = 0;
14
+ private committed: WordTuple[] = [];
15
+
16
+ constructor(asr: ASR) {
17
+ this.asr = asr;
18
+ }
19
+
20
+ public insertAudioChunk(audio: number[]) {
21
+ this.audioBuffer.push(...audio);
22
+ }
23
+
24
+ public async processIter(options: DecodingOptions) {
25
+ const res = await this.asr.transcribe(this.audioBuffer, options);
26
+ const tsw = this.asr.tsWords(res);
27
+ this.transcriptBuffer.insert(tsw, this.bufferTimeOffset);
28
+ const o = this.transcriptBuffer.flush();
29
+ this.committed.push(...o);
30
+
31
+ const s = 15;
32
+ if (this.audioBuffer.length / this.samplingRate > s) {
33
+ this.chunkCompletedSegment(res);
34
+ }
35
+
36
+ const committed = this.toFlush(o)[2];
37
+ const nonCommitted = this.transcriptBuffer
38
+ .complete()
39
+ .map((x) => x[2])
40
+ .join('');
41
+ return { committed, nonCommitted };
42
+ }
43
+
44
+ private chunkCompletedSegment(res: Segment[]) {
45
+ if (this.committed.length === 0) {
46
+ return;
47
+ }
48
+
49
+ const ends = this.asr.segmentsEndTs(res);
50
+ const t = this.committed.at(-1)![1];
51
+
52
+ if (ends.length > 1) {
53
+ let e = ends.at(-2)! + this.bufferTimeOffset;
54
+ while (ends.length > 2 && e > t) {
55
+ ends.pop();
56
+ e = ends.at(-2)! + this.bufferTimeOffset;
57
+ }
58
+
59
+ if (e <= t) {
60
+ this.chunkAt(e);
61
+ }
62
+ }
63
+ }
64
+
65
+ private chunkAt(time: number) {
66
+ this.transcriptBuffer.popCommitted(time);
67
+ const cutSeconds = time - this.bufferTimeOffset;
68
+ this.audioBuffer = this.audioBuffer.slice(
69
+ Math.floor(cutSeconds * this.samplingRate)
70
+ );
71
+ this.bufferTimeOffset = time;
72
+ }
73
+
74
+ public async finish() {
75
+ const o = this.transcriptBuffer.complete();
76
+ const f = this.toFlush(o);
77
+ this.bufferTimeOffset += this.audioBuffer.length / this.samplingRate;
78
+ return { committed: f[2] };
79
+ }
80
+
81
+ private toFlush(words: WordTuple[]): [number | null, number | null, string] {
82
+ const t = words.map((s) => s[2]).join(' ');
83
+ const b = words.length === 0 ? null : words[0]![0];
84
+ const e = words.length === 0 ? null : words.at(-1)![1];
85
+ return [b, e, t];
86
+ }
87
+ }
@@ -0,0 +1,79 @@
1
+ // NOTE: This will be implemented in C++
2
+
3
+ import { WordTuple } from '../../types/stt';
4
+
5
+ export class HypothesisBuffer {
6
+ private committedInBuffer: WordTuple[] = [];
7
+ private buffer: WordTuple[] = [];
8
+ private new: WordTuple[] = [];
9
+
10
+ private lastCommittedTime: number = 0;
11
+ public lastCommittedWord: string | null = null;
12
+
13
+ public insert(newWords: WordTuple[], offset: number) {
14
+ const newWordsOffset: WordTuple[] = newWords.map(([a, b, t]) => [
15
+ a + offset,
16
+ b + offset,
17
+ t,
18
+ ]);
19
+ this.new = newWordsOffset.filter(
20
+ ([a, _b, _t]) => a > this.lastCommittedTime - 0.5
21
+ );
22
+
23
+ if (this.new.length > 0) {
24
+ const [a, _b, _t] = this.new[0]!;
25
+ if (
26
+ Math.abs(a - this.lastCommittedTime) < 1 &&
27
+ this.committedInBuffer.length > 0
28
+ ) {
29
+ const cn = this.committedInBuffer.length;
30
+ const nn = this.new.length;
31
+
32
+ for (let i = 1; i <= Math.min(cn, nn, 5); i++) {
33
+ const c = this.committedInBuffer
34
+ .slice(-i)
35
+ .map((w) => w[2])
36
+ .join(' ');
37
+ const tail = this.new
38
+ .slice(0, i)
39
+ .map((w) => w[2])
40
+ .join(' ');
41
+ if (c === tail) {
42
+ for (let j = 0; j < i; j++) {
43
+ this.new.shift();
44
+ }
45
+ break;
46
+ }
47
+ }
48
+ }
49
+ }
50
+ }
51
+
52
+ public flush(): WordTuple[] {
53
+ const commit: WordTuple[] = [];
54
+ while (this.new.length > 0 && this.buffer.length > 0) {
55
+ if (this.new[0]![2] !== this.buffer[0]![2]) {
56
+ break;
57
+ }
58
+ commit.push(this.new[0]!);
59
+ this.lastCommittedWord = this.new[0]![2];
60
+ this.lastCommittedTime = this.new[0]![1];
61
+ this.buffer.shift();
62
+ this.new.shift();
63
+ }
64
+ this.buffer = this.new;
65
+ this.new = [];
66
+ this.committedInBuffer.push(...commit);
67
+ return commit;
68
+ }
69
+
70
+ public popCommitted(time: number) {
71
+ this.committedInBuffer = this.committedInBuffer.filter(
72
+ ([_a, b, _t]) => b > time
73
+ );
74
+ }
75
+
76
+ public complete(): WordTuple[] {
77
+ return this.buffer;
78
+ }
79
+ }
@@ -1,31 +0,0 @@
1
- #include "MoonshineStrategy.h"
2
- #include "executorch/runtime/core/exec_aten/exec_aten.h"
3
- #include <executorch/extension/tensor/tensor_ptr_maker.h>
4
- #include <executorch/runtime/core/portable_type/scalar_type.h>
5
-
6
- namespace rnexecutorch {
7
-
8
- using namespace ::executorch::extension;
9
- using namespace ::executorch::aten;
10
-
11
- TensorPtr MoonshineStrategy::prepareAudioInput(std::span<float> waveform) {
12
- std::vector<int32_t> inputShape = {1, static_cast<int32_t>(waveform.size())};
13
- return make_tensor_ptr(std::move(inputShape), waveform.data(),
14
- ScalarType::Float);
15
- }
16
-
17
- TensorPtr
18
- MoonshineStrategy::prepareTokenInput(const std::vector<int64_t> &prevTokens) {
19
- std::vector<int32_t> tensorSizes = {1,
20
- static_cast<int32_t>(prevTokens.size())};
21
- // prevTokens gets copied!!
22
- return make_tensor_ptr(std::move(tensorSizes), prevTokens);
23
- }
24
-
25
- int64_t MoonshineStrategy::extractOutputToken(const void *outputPtr,
26
- size_t innerDim) const {
27
- const auto *data = static_cast<const int64_t *>(outputPtr);
28
- return data[innerDim - 1];
29
- }
30
-
31
- } // namespace rnexecutorch
@@ -1,21 +0,0 @@
1
- #pragma once
2
-
3
- #include "SpeechToTextStrategy.h"
4
- #include <span>
5
- #include <vector>
6
-
7
- namespace rnexecutorch {
8
-
9
- class MoonshineStrategy final : public SpeechToTextStrategy {
10
- public:
11
- TensorPtr prepareAudioInput(std::span<float> waveform) override;
12
-
13
- TensorPtr prepareTokenInput(const std::vector<int64_t> &prevTokens) override;
14
-
15
- std::string getDecoderMethod() const override { return "forward_cached"; }
16
-
17
- int64_t extractOutputToken(const void *outputPtr,
18
- size_t innerDim) const override;
19
- };
20
-
21
- } // namespace rnexecutorch
@@ -1,8 +0,0 @@
1
- export declare class Logger {
2
- private static readonly PREFIX;
3
- static log(...data: any[]): void;
4
- static debug(...data: any[]): void;
5
- static info(...data: any[]): void;
6
- static warn(...data: any[]): void;
7
- static error(...data: any[]): void;
8
- }
@@ -1,19 +0,0 @@
1
- /* eslint-disable no-console */
2
- export class Logger {
3
- static PREFIX = '[React Native ExecuTorch]';
4
- static log(...data) {
5
- console.log(Logger.PREFIX, ...data);
6
- }
7
- static debug(...data) {
8
- console.debug(Logger.PREFIX, ...data);
9
- }
10
- static info(...data) {
11
- console.info(Logger.PREFIX, ...data);
12
- }
13
- static warn(...data) {
14
- console.warn(Logger.PREFIX, ...data);
15
- }
16
- static error(...data) {
17
- console.error(Logger.PREFIX, ...data);
18
- }
19
- }
@@ -1,89 +0,0 @@
1
- export declare const LLAMA3_2_3B = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-3B/original/llama3_2_3B_bf16.pte";
2
- export declare const LLAMA3_2_3B_QLORA = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-3B/QLoRA/llama3_2-3B_qat_lora.pte";
3
- export declare const LLAMA3_2_3B_SPINQUANT = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-3B/spinquant/llama3_2_3B_spinquant.pte";
4
- export declare const LLAMA3_2_1B = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-1B/original/llama3_2_bf16.pte";
5
- export declare const LLAMA3_2_1B_QLORA = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-1B/QLoRA/llama3_2_qat_lora.pte";
6
- export declare const LLAMA3_2_1B_SPINQUANT = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-1B/spinquant/llama3_2_spinquant.pte";
7
- export declare const LLAMA3_2_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/tokenizer.json";
8
- export declare const LLAMA3_2_TOKENIZER_CONFIG = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/tokenizer_config.json";
9
- export declare const QWEN3_0_6B = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.4.0/qwen-3-0.6B/original/qwen3_0_6b_bf16.pte";
10
- export declare const QWEN3_0_6B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.4.0/qwen-3-0.6B/quantized/qwen3_0_6b_8da4w.pte";
11
- export declare const QWEN3_1_7B = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.4.0/qwen-3-1.7B/original/qwen3_1_7b_bf16.pte";
12
- export declare const QWEN3_1_7B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.4.0/qwen-3-1.7B/quantized/qwen3_1_7b_8da4w.pte";
13
- export declare const QWEN3_4B = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.4.0/qwen-3-4B/original/qwen3_4b_bf16.pte";
14
- export declare const QWEN3_4B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.4.0/qwen-3-4B/quantized/qwen3_4b_8da4w.pte";
15
- export declare const QWEN3_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.4.0/tokenizer.json";
16
- export declare const QWEN3_TOKENIZER_CONFIG = "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.4.0/tokenizer_config.json";
17
- export declare const HAMMER2_1_0_5B = "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.4.0/hammer-2.1-0.5B/original/hammer2_1_0_5B_bf16.pte";
18
- export declare const HAMMER2_1_0_5B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.4.0/hammer-2.1-0.5B/quantized/hammer2_1_0_5B_8da4w.pte";
19
- export declare const HAMMER2_1_1_5B = "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.4.0/hammer-2.1-1.5B/original/hammer2_1_1_5B_bf16.pte";
20
- export declare const HAMMER2_1_1_5B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.4.0/hammer-2.1-1.5B/quantized/hammer2_1_1_5B_8da4w.pte";
21
- export declare const HAMMER2_1_3B = "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.4.0/hammer-2.1-3B/original/hammer2_1_3B_bf16.pte";
22
- export declare const HAMMER2_1_3B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.4.0/hammer-2.1-3B/quantized/hammer2_1_3B_8da4w.pte";
23
- export declare const HAMMER2_1_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.4.0/tokenizer.json";
24
- export declare const HAMMER2_1_TOKENIZER_CONFIG = "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.4.0/tokenizer_config.json";
25
- export declare const SMOLLM2_1_135M = "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.4.0/smolLm-2-135M/original/smolLm2_135M_bf16.pte";
26
- export declare const SMOLLM2_1_135M_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.4.0/smolLm-2-135M/quantized/smolLm2_135M_8da4w.pte";
27
- export declare const SMOLLM2_1_360M = "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.4.0/smolLm-2-360M/original/smolLm2_360M_bf16.pte";
28
- export declare const SMOLLM2_1_360M_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.4.0/smolLm-2-360M/quantized/smolLm2_360M_8da4w.pte";
29
- export declare const SMOLLM2_1_1_7B = "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.4.0/smolLm-2-1.7B/original/smolLm2_1_7B_bf16.pte";
30
- export declare const SMOLLM2_1_1_7B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.4.0/smolLm-2-1.7B/quantized/smolLm2_1_7B_8da4w.pte";
31
- export declare const SMOLLM2_1_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.4.0/tokenizer.json";
32
- export declare const SMOLLM2_1_TOKENIZER_CONFIG = "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.4.0/tokenizer_config.json";
33
- export declare const QWEN2_5_0_5B = "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.4.0/qwen-2.5-0.5B/original/qwen2_5_0_5b_bf16.pte";
34
- export declare const QWEN2_5_0_5B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.4.0/qwen-2.5-0.5B/quantized/qwen2_5_0_5b_8da4w.pte";
35
- export declare const QWEN2_5_1_5B = "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.4.0/qwen-2.5-1.5B/original/qwen2_5_1_5b_bf16.pte";
36
- export declare const QWEN2_5_1_5B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.4.0/qwen-2.5-1.5B/quantized/qwen2_5_1_5b_8da4w.pte";
37
- export declare const QWEN2_5_3B = "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.4.0/qwen-2.5-3B/original/qwen2_5_3b_bf16.pte";
38
- export declare const QWEN2_5_3B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.4.0/qwen-2.5-3B/quantized/qwen2_5_3b_8da4w.pte";
39
- export declare const QWEN2_5_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.4.0/tokenizer.json";
40
- export declare const QWEN2_5_TOKENIZER_CONFIG = "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.4.0/tokenizer_config.json";
41
- export declare const PHI_4_MINI_4B = "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.4.0/original/phi-4-mini_bf16.pte";
42
- export declare const PHI_4_MINI_4B_QUANTIZED = "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.4.0/quantized/phi-4-mini_8da4w.pte";
43
- export declare const PHI_4_MINI_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.4.0/tokenizer.json";
44
- export declare const PHI_4_MINI_TOKENIZER_CONFIG = "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.4.0/tokenizer_config.json";
45
- export declare const EFFICIENTNET_V2_S: string;
46
- export declare const SSDLITE_320_MOBILENET_V3_LARGE = "https://huggingface.co/software-mansion/react-native-executorch-ssdlite320-mobilenet-v3-large/resolve/v0.4.0/ssdlite320-mobilenetv3-large.pte";
47
- export declare const STYLE_TRANSFER_CANDY: string;
48
- export declare const STYLE_TRANSFER_MOSAIC: string;
49
- export declare const STYLE_TRANSFER_RAIN_PRINCESS: string;
50
- export declare const STYLE_TRANSFER_UDNIE: string;
51
- export declare const MOONSHINE_TINY_DECODER = "https://huggingface.co/software-mansion/react-native-executorch-moonshine-tiny/resolve/v0.4.0/xnnpack/moonshine_tiny_xnnpack_decoder.pte";
52
- export declare const MOONSHINE_TINY_ENCODER = "https://huggingface.co/software-mansion/react-native-executorch-moonshine-tiny/resolve/v0.4.0/xnnpack/moonshine_tiny_xnnpack_encoder.pte";
53
- export declare const MOONSHINE_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-moonshine-tiny/resolve/v0.4.0/moonshine_tiny_tokenizer.json";
54
- export declare const WHISPER_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.4.0/whisper_tokenizer.json";
55
- export declare const WHISPER_TINY_DECODER = "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.4.0/xnnpack/whisper_tiny_en_xnnpack_decoder.pte";
56
- export declare const WHISPER_TINY_ENCODER = "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.4.0/xnnpack/whisper_tiny_en_xnnpack_encoder.pte";
57
- export declare const WHISPER_TINY_MULTILINGUAL_ENCODER = "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.4.0/xnnpack/xnnpack_whisper_encoder.pte";
58
- export declare const WHISPER_TINY_MULTILINGUAL_DECODER = "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.4.0/xnnpack/xnnpack_whisper_decoder.pte";
59
- export declare const WHISPER_TINY_MULTILINGUAL_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.4.0/tokenizer.json";
60
- export declare const DETECTOR_CRAFT_1280 = "https://huggingface.co/software-mansion/react-native-executorch-detector-craft/resolve/v0.4.0/xnnpack/xnnpack_craft_1280.pte";
61
- export declare const DETECTOR_CRAFT_800 = "https://huggingface.co/software-mansion/react-native-executorch-detector-craft/resolve/v0.4.0/xnnpack/xnnpack_craft_800.pte";
62
- export declare const DETECTOR_CRAFT_320 = "https://huggingface.co/software-mansion/react-native-executorch-detector-craft/resolve/v0.4.0/xnnpack/xnnpack_craft_320.pte";
63
- export declare const DEEPLAB_V3_RESNET50 = "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.4.0/xnnpack/deeplabV3_xnnpack_fp32.pte";
64
- export declare const CLIP_VIT_BASE_PATCH32_IMAGE_MODEL = "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.5.0/clip-vit-base-patch32-vision_xnnpack.pte";
65
- export declare const ALL_MINILM_L6_V2 = "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.5.0/all-MiniLM-L6-v2_xnnpack.pte";
66
- export declare const ALL_MINILM_L6_V2_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.5.0/tokenizer.json";
67
- export declare const ALL_MPNET_BASE_V2 = "https://huggingface.co/software-mansion/react-native-executorch-all-mpnet-base-v2/resolve/v0.5.0/all-mpnet-base-v2_xnnpack.pte";
68
- export declare const ALL_MPNET_BASE_V2_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-all-mpnet-base-v2/resolve/v0.5.0/tokenizer.json";
69
- export declare const MULTI_QA_MINILM_L6_COS_V1 = "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.5.0/multi-qa-MiniLM-L6-cos-v1_xnnpack.pte";
70
- export declare const MULTI_QA_MINILM_L6_COS_V1_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.5.0/tokenizer.json";
71
- export declare const MULTI_QA_MPNET_BASE_DOT_V1 = "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-mpnet-base-dot-v1/resolve/v0.5.0/multi-qa-mpnet-base-dot-v1_xnnpack.pte";
72
- export declare const MULTI_QA_MPNET_BASE_DOT_V1_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-mpnet-base-dot-v1/resolve/v0.5.0/tokenizer.json";
73
- export declare const CLIP_VIT_BASE_PATCH32_TEXT_MODEL = "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.5.0/clip-vit-base-patch32-text_xnnpack.pte";
74
- export declare const CLIP_VIT_BASE_PATCH32_TEXT_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.5.0/tokenizer.json";
75
- export declare const CLIP_VIT_BASE_PATCH32_TEXT: {
76
- modelSource: string;
77
- tokenizerSource: string;
78
- };
79
- export declare const CLIP_VIT_BASE_PATCH32_IMAGE: {
80
- modelSource: string;
81
- };
82
- export declare const LLAMA3_2_3B_URL = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-3B/original/llama3_2_3B_bf16.pte";
83
- export declare const LLAMA3_2_3B_QLORA_URL = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-3B/QLoRA/llama3_2-3B_qat_lora.pte";
84
- export declare const LLAMA3_2_3B_SPINQUANT_URL = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-3B/spinquant/llama3_2_3B_spinquant.pte";
85
- export declare const LLAMA3_2_1B_URL = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-1B/original/llama3_2_bf16.pte";
86
- export declare const LLAMA3_2_1B_QLORA_URL = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-1B/QLoRA/llama3_2_qat_lora.pte";
87
- export declare const LLAMA3_2_1B_SPINQUANT_URL = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/llama-3.2-1B/spinquant/llama3_2_spinquant.pte";
88
- export declare const LLAMA3_2_1B_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/tokenizer.json";
89
- export declare const LLAMA3_2_3B_TOKENIZER = "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.4.0/tokenizer.json";