react-native-nitro-onnx 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/NOTICE +241 -0
  2. package/NitroOnnxSpeech.podspec +51 -0
  3. package/Package.swift +52 -0
  4. package/README.md +398 -0
  5. package/android/CMakeLists.txt +117 -0
  6. package/android/build.gradle +65 -0
  7. package/android/src/main/AndroidManifest.xml +6 -0
  8. package/android/src/main/assets/silero_vad.onnx +0 -0
  9. package/android/src/main/cpp/cpp-adapter.cpp +24 -0
  10. package/android/src/main/java/com/margelo/nitro/onnx/speech/OnnxSpeechPackage.kt +67 -0
  11. package/assets/silero_vad.onnx +0 -0
  12. package/cpp/AndroidPthreadCompat.cpp +16 -0
  13. package/cpp/AsrEngine.cpp +277 -0
  14. package/cpp/AsrEngine.hpp +110 -0
  15. package/cpp/AudioFileReader.cpp +177 -0
  16. package/cpp/AudioFileReader.hpp +26 -0
  17. package/cpp/AudioUtils.cpp +28 -0
  18. package/cpp/AudioUtils.hpp +38 -0
  19. package/cpp/ModelSingleton.hpp +51 -0
  20. package/cpp/NitroOnnxSpeech.cpp +68 -0
  21. package/cpp/NitroOnnxSpeech.hpp +42 -0
  22. package/cpp/OfflineAsr.cpp +81 -0
  23. package/cpp/OfflineAsr.hpp +33 -0
  24. package/cpp/ResourceDir.cpp +29 -0
  25. package/cpp/ResourceDir.hpp +18 -0
  26. package/cpp/SpeakerEngine.cpp +170 -0
  27. package/cpp/SpeakerEngine.hpp +70 -0
  28. package/cpp/SpeakerManager.cpp +93 -0
  29. package/cpp/SpeakerManager.hpp +43 -0
  30. package/cpp/StreamingAsr.cpp +123 -0
  31. package/cpp/StreamingAsr.hpp +55 -0
  32. package/cpp/ThreadPool.cpp +41 -0
  33. package/cpp/ThreadPool.hpp +62 -0
  34. package/cpp/Tts.cpp +132 -0
  35. package/cpp/Tts.hpp +38 -0
  36. package/cpp/TtsEngine.cpp +222 -0
  37. package/cpp/TtsEngine.hpp +77 -0
  38. package/cpp/Vad.cpp +126 -0
  39. package/cpp/Vad.hpp +62 -0
  40. package/cpp/VadEngine.cpp +217 -0
  41. package/cpp/VadEngine.hpp +126 -0
  42. package/ios/OnnxSpeechInitializer.mm +37 -0
  43. package/ios/PrivacyInfo.xcprivacy +14 -0
  44. package/lib/index.d.ts +11 -0
  45. package/lib/index.d.ts.map +1 -0
  46. package/lib/index.js +19 -0
  47. package/lib/index.js.map +1 -0
  48. package/lib/specs/OnnxSpeech.nitro.d.ts +264 -0
  49. package/lib/specs/OnnxSpeech.nitro.d.ts.map +1 -0
  50. package/lib/specs/OnnxSpeech.nitro.js +6 -0
  51. package/lib/specs/OnnxSpeech.nitro.js.map +1 -0
  52. package/nitro.json +19 -0
  53. package/nitrogen/generated/.gitattributes +1 -0
  54. package/nitrogen/generated/android/NitroOnnxSpeech+autolinking.cmake +86 -0
  55. package/nitrogen/generated/android/NitroOnnxSpeech+autolinking.gradle +27 -0
  56. package/nitrogen/generated/android/NitroOnnxSpeechOnLoad.cpp +49 -0
  57. package/nitrogen/generated/android/NitroOnnxSpeechOnLoad.hpp +34 -0
  58. package/nitrogen/generated/android/kotlin/com/margelo/nitro/onnx/speech/NitroOnnxSpeechOnLoad.kt +35 -0
  59. package/nitrogen/generated/ios/NitroOnnxSpeech+autolinking.rb +62 -0
  60. package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Bridge.cpp +17 -0
  61. package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Bridge.hpp +27 -0
  62. package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Umbrella.hpp +38 -0
  63. package/nitrogen/generated/ios/NitroOnnxSpeechAutolinking.mm +35 -0
  64. package/nitrogen/generated/ios/NitroOnnxSpeechAutolinking.swift +16 -0
  65. package/nitrogen/generated/shared/c++/AsrModelConfig.hpp +142 -0
  66. package/nitrogen/generated/shared/c++/AsrModelType.hpp +112 -0
  67. package/nitrogen/generated/shared/c++/AsrResult.hpp +105 -0
  68. package/nitrogen/generated/shared/c++/HybridOfflineAsrSpec.cpp +25 -0
  69. package/nitrogen/generated/shared/c++/HybridOfflineAsrSpec.hpp +73 -0
  70. package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.cpp +27 -0
  71. package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.hpp +82 -0
  72. package/nitrogen/generated/shared/c++/HybridSpeakerManagerSpec.cpp +28 -0
  73. package/nitrogen/generated/shared/c++/HybridSpeakerManagerSpec.hpp +77 -0
  74. package/nitrogen/generated/shared/c++/HybridStreamingAsrSpec.cpp +32 -0
  75. package/nitrogen/generated/shared/c++/HybridStreamingAsrSpec.hpp +81 -0
  76. package/nitrogen/generated/shared/c++/HybridTtsSpec.cpp +26 -0
  77. package/nitrogen/generated/shared/c++/HybridTtsSpec.hpp +74 -0
  78. package/nitrogen/generated/shared/c++/HybridVadSpec.cpp +31 -0
  79. package/nitrogen/generated/shared/c++/HybridVadSpec.hpp +81 -0
  80. package/nitrogen/generated/shared/c++/RegisteredSpeaker.hpp +91 -0
  81. package/nitrogen/generated/shared/c++/SpeakerEmbeddingConfig.hpp +91 -0
  82. package/nitrogen/generated/shared/c++/TtsModelConfig.hpp +178 -0
  83. package/nitrogen/generated/shared/c++/TtsModelType.hpp +88 -0
  84. package/nitrogen/generated/shared/c++/TtsResult.hpp +91 -0
  85. package/nitrogen/generated/shared/c++/VadConfig.hpp +100 -0
  86. package/nitrogen/generated/shared/c++/VadSegment.hpp +91 -0
  87. package/package.json +62 -0
  88. package/scripts/prepare-sherpa-onnx.js +236 -0
  89. package/scripts/test-cpp.js +24 -0
  90. package/src/index.ts +45 -0
  91. package/src/specs/OnnxSpeech.nitro.ts +326 -0
package/README.md ADDED
@@ -0,0 +1,398 @@
1
+ # react-native-nitro-onnx
2
+
3
+ A React Native [Nitro Module](https://nitro.margelo.com) that wraps [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) for on-device speech processing:
4
+
5
+ - **ASR** (Automatic Speech Recognition) - offline and streaming
6
+ - **TTS** (Text-to-Speech)
7
+ - **VAD** (Voice Activity Detection) with sliding pre-buffer
8
+ - **Voice cloning** via speaker embeddings and reference-audio TTS
9
+
10
+ All audio I/O uses zero-copy `ArrayBuffer` with **16 kHz mono f32 PCM**.
11
+
12
+ > **Note:** This repository is a structural scaffold. Every model family has a typed config slot and a singleton-backed engine. Only one model per category is fully wired in the reference implementation; the remaining model types map to the correct sherpa-onnx C API fields and are ready for incremental completion.
13
+
14
+ ## Table of Contents
15
+
16
+ - [Architecture](#architecture)
17
+ - [Supported Models](#supported-models)
18
+ - [ASR](#asr)
19
+ - [TTS](#tts)
20
+ - [VAD](#vad)
21
+ - [Model Downloads](#model-downloads)
22
+ - [Model File Requirements](#model-file-requirements)
23
+ - [Installation](#installation)
24
+ - [Usage](#usage)
25
+ - [Qualcomm CPU Detection](#qualcomm-cpu-detection)
26
+ - [VAD Pre-buffer](#vad-pre-buffer)
27
+ - [Voice Cloning](#voice-cloning)
28
+ - [Threading](#threading)
29
+ - [Testing](#testing)
30
+ - [License](#license)
31
+
32
+ ## Architecture
33
+
34
+ ```
35
+ ┌─────────────────────────────────────────────────────────────┐
36
+ │ JS / TS │
37
+ │ getOnnxSpeech() → createVad() / createTts() / createAsr() │
38
+ └──────────────────────┬──────────────────────────────────────┘
39
+ │ react-native-nitro-modules (zero-copy ArrayBuffer)
40
+ ┌──────────────────────┴──────────────────────────────────────┐
41
+ │ C++ │
42
+ │ OnnxSpeechImpl → VadEngine / AsrEngine / TtsEngine / ... │
43
+ │ ModelSingleton caches heavy recognizer / TTS instances │
44
+ └──────────────────────┬──────────────────────────────────────┘
45
+ │ sherpa-onnx C API
46
+ ┌──────────────────────┴──────────────────────────────────────┐
47
+ │ ONNX Runtime │
48
+ └─────────────────────────────────────────────────────────────┘
49
+ ```
50
+
51
+ Key design decisions:
52
+
53
+ - **Singleton preloading:** `OfflineAsrEngine`, `StreamingAsrEngine`, `TtsEngine` and `SpeakerEngine` use `ModelSingleton` keyed by model directory and type. Loading the same model twice returns the same native instance.
54
+ - **Background inference:** Every heavy operation runs on a fixed `ThreadPool` so the JS thread never blocks.
55
+ - **External model download:** The module does not bundle an internal downloader. Download model files in the background with a library such as [`@kesha-antonov/react-native-background-downloader`](https://github.com/kesha-antonov/react-native-background-downloader), then pass the local file paths to `load()` / `initialize()`.
56
+ - **Zero-copy audio:** `ArrayBuffer` is the only audio transport format; samples are expected to be 16 kHz mono little-endian f32 PCM.
57
+
58
+ ## Supported Models
59
+
60
+ ### ASR
61
+
62
+ | Model family | Type slug | Architecture | Best for | Streaming | Notes |
63
+ |---|---|---|---|---|---|
64
+ | Whisper | `whisper` | encoder-decoder | Multi-language, accuracy | No | Needs `encoder.onnx`, `decoder.onnx`, `tokens.txt` |
65
+ | Transducer | `transducer` | RNN-T / RNNT | Streaming accuracy | Yes | Needs `encoder.onnx`, `decoder.onnx`, `joiner.onnx` |
66
+ | Paraformer | `paraformer` | non-autoregressive | Fast offline Chinese/English | No | Single `model.onnx` |
67
+ | Zipformer | `zipformer` | fast conformer variant | Streaming, low latency | Yes | Transducer triple |
68
+ | Conformer | `conformer` | attention-convolution | Streaming accuracy | Yes | Transducer triple |
69
+ | Wenet | `wenet` | U2++ / CTC | Chinese industrial | No | Single `model.onnx` |
70
+ | Telespeech | `telespeech` | telephony ASR | 8 kHz telco audio | No | Single `model.onnx` |
71
+ | Moonshine | `moonshine` | lightweight | Edge devices | No | Single `model.onnx` |
72
+ | Dolphin | `dolphin` | CTC | English | No | Single `model.onnx` |
73
+ | NeMo | `nemo` | CTC / RNNT | NVIDIA NeMo exported models | No | `model.onnx` + config |
74
+ | SenseVoice | `sense_voice` | multilingual | Alibaba SenseVoice | No | Single `model.onnx` |
75
+
76
+ ### TTS
77
+
78
+ | Model family | Type slug | Vocoder | Quality | Speed | Notes |
79
+ |---|---|---|---|---|---|
80
+ | Kokoro | `kokoro` | internal | High, multi-speaker | Medium | Needs `model.onnx`, `voices.bin`, `tokens.txt`, `lexicon.txt` |
81
+ | VITS | `vits` | internal | High quality | Medium | Needs `model.onnx`, `tokens.txt`, optional lexicon |
82
+ | Matcha | `matcha` | external (e.g. Hifigan) | Fast, natural | Fast | Needs acoustic model + vocoder ONNX |
83
+ | Pocket | `pocket` | internal | Lightweight zero-shot | Very fast | Needs `model.onnx` + config JSON |
84
+ | ZipVoice | `zipvoice` | internal | Placeholder type | - | Maps to VITS-like config until sherpa-onnx exposes dedicated ZipVoice support |
85
+
86
+ ### VAD
87
+
88
+ The module uses sherpa-onnx's Silero VAD implementation. The `silero_vad.onnx` model is bundled with the module, so you can initialize VAD without specifying `modelPath`. You may still pass a custom `modelPath` if you want to use your own ONNX VAD model.
89
+
90
+ ## Model Downloads
91
+
92
+ Pretrained sherpa-onnx model families are published as GitHub release assets. Download the required files to the device (for example with [`@kesha-antonov/react-native-background-downloader`](https://github.com/kesha-antonov/react-native-background-downloader)) and pass the local paths to `load()` / `initialize()`.
93
+
94
+ - **TTS models:** https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
95
+ - **ASR models:** https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models
96
+
97
+ > The Silero VAD model (`silero_vad.onnx`) is bundled with the module, so no separate download is required for VAD.
98
+
99
+ ## Model File Requirements
100
+
101
+ Each model family expects a specific set of ONNX and metadata files. When you call `load()` / `initialize()`, you must point every relevant path at the downloaded files on the device. The exact files depend on the model family:
102
+
103
+ ### Whisper (offline ASR)
104
+
105
+ ```
106
+ whisper/
107
+ encoder.onnx
108
+ decoder.onnx
109
+ tokens.txt
110
+ ```
111
+
112
+ ### Transducer / Zipformer / Conformer (streaming ASR)
113
+
114
+ ```
115
+ transducer/
116
+ encoder.onnx
117
+ decoder.onnx
118
+ joiner.onnx
119
+ tokens.txt
120
+ ```
121
+
122
+ ### Paraformer / Wenet / Telespeech / Moonshine / Dolphin / SenseVoice (offline ASR)
123
+
124
+ ```
125
+ model/
126
+ model.onnx
127
+ tokens.txt
128
+ ```
129
+
130
+ ### NeMo (offline ASR)
131
+
132
+ ```
133
+ nemo/
134
+ model.onnx
135
+ config.yaml
136
+ tokens.txt
137
+ ```
138
+
139
+ ### Kokoro (TTS)
140
+
141
+ ```
142
+ kokoro/
143
+ model.onnx
144
+ voices.bin
145
+ tokens.txt
146
+ lexicon.txt
147
+ ```
148
+
149
+ ### VITS (TTS)
150
+
151
+ ```
152
+ vits/
153
+ model.onnx
154
+ tokens.txt
155
+ lexicon.txt (optional)
156
+ ```
157
+
158
+ ### Matcha (TTS)
159
+
160
+ ```
161
+ matcha/
162
+ acoustic_model.onnx
163
+ vocoder.onnx
164
+ tokens.txt
165
+ lexicon.txt
166
+ ```
167
+
168
+ ### Pocket (TTS)
169
+
170
+ ```
171
+ pocket/
172
+ model.onnx
173
+ config.json
174
+ ```
175
+
176
+ ### Silero VAD
177
+
178
+ `silero_vad.onnx` is bundled with the module and used by default. A custom model can be provided via `VadConfig.modelPath`.
179
+
180
+ ### Speaker embedding (voice cloning)
181
+
182
+ ```
183
+ speaker/
184
+ model.onnx
185
+ ```
186
+
187
+ ## Installation
188
+
189
+ ```bash
190
+ yarn add react-native-nitro-onnx
191
+ # or
192
+ npm install react-native-nitro-onnx
193
+ ```
194
+
195
+ Build requirements:
196
+
197
+ - React Native >= 0.78
198
+ - react-native-nitro-modules >= 0.35.8
199
+ - Xcode 15 / Android NDK 26
200
+ - The `prepare-sherpa-onnx.js` postinstall script downloads the sherpa-onnx
201
+ prebuilt tree (host static libraries, Android shared libraries, iOS
202
+ xcframework, and C API headers) into `cpp/sherpa-onnx-prebuilt`.
203
+
204
+ iOS:
205
+
206
+ ```bash
207
+ cd ios && pod install
208
+ ```
209
+
210
+ Android:
211
+
212
+ ```bash
213
+ cd android
214
+ ./gradlew assembleDebug
215
+ ```
216
+
217
+ ## Usage
218
+
219
+ ```typescript
220
+ import { getOnnxSpeech, DEFAULT_AUDIO_FORMAT } from "react-native-nitro-onnx";
221
+
222
+ const speech = getOnnxSpeech();
223
+
224
+ // 1. Load a model by specifying the local file paths.
225
+ // Download the files first, e.g. with @kesha-antonov/react-native-background-downloader.
226
+ const modelDir = `${RNFS.DocumentDirectoryPath}/whisper-tiny`;
227
+ const asr = speech.createOfflineAsr();
228
+ await asr.load({
229
+ type: "whisper",
230
+ modelDir,
231
+ tokensPath: `${modelDir}/tokens.txt`,
232
+ whisperEncoder: `${modelDir}/encoder.onnx`,
233
+ whisperDecoder: `${modelDir}/decoder.onnx`,
234
+ numThreads: 4,
235
+ language: "en",
236
+ });
237
+
238
+ // 2. Recognize speech.
239
+ const result = await asr.recognize(pcmArrayBuffer);
240
+ console.log(result.text);
241
+ ```
242
+
243
+ ### VAD example
244
+
245
+ ```typescript
246
+ const vad = speech.createVad();
247
+ await vad.initialize({
248
+ // modelPath is optional; the bundled silero_vad.onnx is used by default.
249
+ threshold: 0.5,
250
+ minSilenceDurationMs: 500,
251
+ minSpeechDurationMs: 250,
252
+ preBufferMs: 300,
253
+ });
254
+
255
+ // Subscribe to events.
256
+ vad.onSpeechStart = (segment) => {
257
+ console.log("speech started at", segment.startMs);
258
+ };
259
+ vad.onSpeechEnd = (segment) => {
260
+ console.log("speech ended at", segment.endMs, "samples", segment.samples);
261
+ };
262
+
263
+ // Feed microphone chunks.
264
+ await vad.process(microphoneChunk);
265
+ ```
266
+
267
+ ### TTS example
268
+
269
+ ```typescript
270
+ const tts = speech.createTts();
271
+ const modelDir = `${RNFS.DocumentDirectoryPath}/kokoro`;
272
+ await tts.load({
273
+ type: "kokoro",
274
+ modelDir,
275
+ acousticModel: `${modelDir}/model.onnx`,
276
+ voices: `${modelDir}/voices.bin`,
277
+ tokens: `${modelDir}/tokens.txt`,
278
+ lexicon: `${modelDir}/lexicon.txt`,
279
+ numThreads: 4,
280
+ outputSampleRate: 16000,
281
+ speed: 1.0, // default speed
282
+ });
283
+
284
+ // Synthesize with optional per-call speed override
285
+ const audio = await tts.synthesize("Hello, this is a test.", 0.9);
286
+ // audio.samples is an ArrayBuffer of f32 PCM at audio.sampleRate.
287
+
288
+ // Save to WAV file
289
+ await tts.saveWav(audio, "/path/to/output.wav");
290
+ ```
291
+
292
+ > **Important:** The synthesized audio is **f32le PCM** (32-bit float, little-endian). When playing with `react-native-audio-api`, you must specify the correct `sampleRate` when creating the `AudioContext`:
293
+ >
294
+ > ```typescript
295
+ > import { AudioContext } from "react-native-audio-api";
296
+ >
297
+ > const audio = await tts.synthesize("Hello");
298
+ > const ctx = new AudioContext({ sampleRate: audio.sampleRate });
299
+ > const buffer = ctx.createBuffer(1, audio.samples.byteLength / 4, audio.sampleRate);
300
+ > buffer.copyToChannel(new Float32Array(audio.samples), 0);
301
+ > const source = ctx.createBufferSource();
302
+ > source.buffer = buffer;
303
+ > source.connect(ctx.destination);
304
+ > source.start();
305
+ > ```
306
+
307
+ ## Qualcomm CPU Detection
308
+
309
+ Use `isQualcommCpu()` to detect Qualcomm chipsets and select optimized QNN execution providers:
310
+
311
+ ```typescript
312
+ const speech = getOnnxSpeech();
313
+ const isQualcomm = speech.isQualcommCpu();
314
+
315
+ if (isQualcomm) {
316
+ // Use QNN execution provider for better performance on Qualcomm chips
317
+ await asr.load({
318
+ type: "whisper",
319
+ // ... other config
320
+ provider: "qnn",
321
+ });
322
+ } else {
323
+ // Use CPU or other execution provider
324
+ await asr.load({
325
+ type: "whisper",
326
+ // ... other config
327
+ provider: "cpu",
328
+ });
329
+ }
330
+ ```
331
+
332
+ > **Note:** `isQualcommCpu()` returns `false` on iOS. It checks `/proc/cpuinfo` on Android to detect Qualcomm processors.
333
+
334
+ ## VAD Pre-buffer
335
+
336
+ A common problem with streaming VAD is that `onSpeechStart` is delivered asynchronously, so the first few frames of speech are already inside the native VAD before JS is notified. When JS later receives the segment at `onSpeechEnd`, the leading audio is clipped.
337
+
338
+ This module solves that with a **sliding pre-buffer queue**:
339
+
340
+ 1. Every incoming chunk is appended to a fixed-size ring buffer (`preBufferMs`).
341
+ 2. When sherpa-onnx detects speech, the pre-buffer is captured as the start of the active segment.
342
+ 3. `onSpeechStart` is emitted immediately with the buffered leading audio.
343
+ 4. Subsequent chunks are accumulated into the same segment.
344
+ 5. When speech ends, the full accumulated segment is emitted via `onSpeechEnd` and made available through `pullSegments()`.
345
+
346
+ The result is that no speech frames are lost between detection and JS delivery.
347
+
348
+ ## Voice Cloning
349
+
350
+ Two voice-cloning paths are exposed:
351
+
352
+ 1. **Speaker embedding registration** - compute an embedding from reference audio, store it locally, and pass the speaker ID to TTS models that accept a speaker index (e.g. Kokoro multi-speaker, VITS multi-speaker).
353
+ 2. **Reference-audio TTS** - models that support prompt-based or zero-shot synthesis (e.g. Pocket) receive the reference embedding directly during synthesis.
354
+
355
+ ```typescript
356
+ const speaker = speech.createSpeakerManager();
357
+ await speaker.load({ modelDir: "/path/to/speaker", model: "model.onnx", numThreads: 4 });
358
+
359
+ const embedding = await speaker.computeEmbedding(referenceAudio);
360
+ const registered = await speaker.registerSpeaker("speaker-1", "Alice", embedding);
361
+
362
+ // Use the registered speaker with TTS (optional speed override)
363
+ const cloned = await tts.synthesizeWithSpeaker("Hello, I am Alice.", registered.id, 1.1);
364
+ ```
365
+
366
+ ## Threading
367
+
368
+ Every native inference task runs on a background thread pool:
369
+
370
+ - VAD processing
371
+ - Offline / streaming ASR decode
372
+ - TTS synthesis
373
+ - Speaker embedding extraction
374
+
375
+ JS calls return promises that resolve on the JS thread when the background work completes. Native event callbacks (VAD speech start/end, streaming partial/final results, ASR/TTS errors) are dispatched to JS without blocking inference.
376
+
377
+ ## Testing
378
+
379
+ TypeScript tests:
380
+
381
+ ```bash
382
+ yarn test
383
+ ```
384
+
385
+ C++ tests (do not require a real model):
386
+
387
+ ```bash
388
+ yarn test:cpp
389
+ ```
390
+
391
+ The C++ test suite covers:
392
+
393
+ - Audio sample / millisecond conversions
394
+ - Float vector / byte buffer round-trip
395
+
396
+ ## License
397
+
398
+ MIT
@@ -0,0 +1,117 @@
1
+ cmake_minimum_required(VERSION 3.13)
2
+ project(NitroOnnxSpeech)
3
+
4
+ set(CMAKE_CXX_STANDARD 20)
5
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
6
+ set(CMAKE_C_STANDARD 11)
7
+
8
+ # ------------------------------------------------------------------------------
9
+ # Optional C++ unit-test target. Built separately from the React Native library.
10
+ # ------------------------------------------------------------------------------
11
+ option(BUILD_ONNX_SPEECH_TESTS "Build C++ unit tests" OFF)
12
+
13
+ if(BUILD_ONNX_SPEECH_TESTS)
14
+ enable_testing()
15
+ add_executable(NitroOnnxSpeechTests
16
+ ../cpp/tests/test_core.cpp
17
+ ../cpp/ThreadPool.cpp
18
+ ../cpp/AudioUtils.cpp
19
+ )
20
+ target_include_directories(NitroOnnxSpeechTests PRIVATE ../cpp)
21
+ find_package(Threads REQUIRED)
22
+ target_link_libraries(NitroOnnxSpeechTests Threads::Threads)
23
+ add_test(NAME NitroOnnxSpeechTests COMMAND NitroOnnxSpeechTests)
24
+ return()
25
+ endif()
26
+
27
+ # ------------------------------------------------------------------------------
28
+ # React Native Nitro module build
29
+ # ------------------------------------------------------------------------------
30
+ find_package(react-native-nitro-modules REQUIRED CONFIG)
31
+
32
+ # Sherpa ONNX: the prebuilt tree is fixed at cpp/sherpa-onnx-prebuilt.
33
+ # If it is missing, run the prepare script automatically at configure time.
34
+ set(_ONNX_PREBUILT_DIR "${CMAKE_SOURCE_DIR}/../cpp/sherpa-onnx-prebuilt")
35
+ if(NOT EXISTS "${_ONNX_PREBUILT_DIR}")
36
+ find_program(NODE_EXECUTABLE node REQUIRED)
37
+ message(STATUS "sherpa-onnx prebuilt not found; running prepare-sherpa-onnx.js...")
38
+ execute_process(
39
+ COMMAND ${NODE_EXECUTABLE} ../scripts/prepare-sherpa-onnx.js
40
+ WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
41
+ RESULT_VARIABLE _PREPARE_RESULT
42
+ OUTPUT_VARIABLE _PREPARE_OUTPUT
43
+ ERROR_VARIABLE _PREPARE_ERROR
44
+ )
45
+ if(NOT _PREPARE_RESULT EQUAL 0)
46
+ message(FATAL_ERROR
47
+ "Failed to run prepare-sherpa-onnx.js:\n${_PREPARE_OUTPUT}\n${_PREPARE_ERROR}"
48
+ )
49
+ endif()
50
+ endif()
51
+ if(NOT EXISTS "${_ONNX_PREBUILT_DIR}")
52
+ message(FATAL_ERROR
53
+ "sherpa-onnx prebuilt is missing at cpp/sherpa-onnx-prebuilt. "
54
+ "Run 'yarn install' to download a prebuilt library."
55
+ )
56
+ endif()
57
+
58
+ file(GLOB_RECURSE ONNX_SPEECH_SOURCES "../cpp/*.cpp")
59
+ list(FILTER ONNX_SPEECH_SOURCES EXCLUDE REGEX "cpp/tests/.*")
60
+ list(FILTER ONNX_SPEECH_SOURCES EXCLUDE REGEX "cpp/build-.*/.*")
61
+
62
+ add_library(${PROJECT_NAME} SHARED
63
+ ${ONNX_SPEECH_SOURCES}
64
+ src/main/cpp/cpp-adapter.cpp
65
+ )
66
+
67
+ target_include_directories(${PROJECT_NAME} PRIVATE
68
+ "${_ONNX_PREBUILT_DIR}/include"
69
+ "../cpp"
70
+ "../nitrogen/generated/android"
71
+ "../nitrogen/generated/shared/c++"
72
+ )
73
+
74
+ include(${CMAKE_SOURCE_DIR}/../nitrogen/generated/android/NitroOnnxSpeech+autolinking.cmake)
75
+
76
+ if(ANDROID)
77
+ # Android prebuilt ships shared libraries per ABI.
78
+ set(_ANDROID_ABI_DIR "${_ONNX_PREBUILT_DIR}/android/jniLibs/${ANDROID_ABI}")
79
+ if(NOT EXISTS "${_ANDROID_ABI_DIR}")
80
+ message(FATAL_ERROR
81
+ "Missing sherpa-onnx Android libraries for ABI ${ANDROID_ABI}. "
82
+ "Run 'yarn install' to download the prebuilt tree."
83
+ )
84
+ endif()
85
+
86
+ add_library(sherpa-onnx-c-api SHARED IMPORTED)
87
+ set_target_properties(sherpa-onnx-c-api PROPERTIES
88
+ IMPORTED_LOCATION "${_ANDROID_ABI_DIR}/libsherpa-onnx-c-api.so"
89
+ )
90
+
91
+ add_library(onnxruntime SHARED IMPORTED)
92
+ set_target_properties(onnxruntime PROPERTIES
93
+ IMPORTED_LOCATION "${_ANDROID_ABI_DIR}/libonnxruntime.so"
94
+ )
95
+
96
+ target_link_libraries(${PROJECT_NAME}
97
+ sherpa-onnx-c-api
98
+ onnxruntime
99
+ log
100
+ )
101
+ else()
102
+ # Host static library layout. Mainly useful for local C++ development.
103
+ target_link_libraries(${PROJECT_NAME}
104
+ sherpa-onnx-c-api
105
+ sherpa-onnx-core
106
+ onnxruntime
107
+ )
108
+ endif()
109
+
110
+ if(ANDROID)
111
+ # Skip CMake's pthread feature probe on Android. Bionic has pthread built-in
112
+ # but lacks pthread_cancel; the probe fails under -Werror=implicit-function-declaration.
113
+ set(THREADS_HAVE_PTHREAD_ARG FALSE)
114
+ set(CMAKE_HAVE_LIBC_PTHREAD TRUE)
115
+ endif()
116
+ find_package(Threads REQUIRED)
117
+ target_link_libraries(${PROJECT_NAME} Threads::Threads)
@@ -0,0 +1,65 @@
1
+ buildscript {
2
+ ext.safeExtGet = {prop, fallback ->
3
+ rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
4
+ }
5
+ }
6
+
7
+ def reactNativeArchitectures() {
8
+ def value = rootProject.getProperties().get("reactNativeArchitectures")
9
+ return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
10
+ }
11
+
12
+ apply plugin: 'com.android.library'
13
+ apply plugin: 'kotlin-android'
14
+
15
+ android {
16
+ namespace "com.margelo.nitro.onnx.speech"
17
+ compileSdk safeExtGet("compileSdkVersion", 34)
18
+
19
+ defaultConfig {
20
+ minSdk safeExtGet("minSdkVersion", 21)
21
+ externalNativeBuild {
22
+ cmake {
23
+ cppFlags "-std=c++20 -O3 -fexceptions -frtti"
24
+ arguments "-DANDROID_STL=c++_shared"
25
+ abiFilters (*reactNativeArchitectures())
26
+
27
+ buildTypes {
28
+ debug {
29
+ cppFlags "-O1 -g"
30
+ }
31
+ release {
32
+ cppFlags "-O2"
33
+ }
34
+ }
35
+ }
36
+ }
37
+ }
38
+
39
+ buildFeatures {
40
+ buildConfig false
41
+ prefab true
42
+ }
43
+
44
+ compileOptions {
45
+ sourceCompatibility JavaVersion.VERSION_17
46
+ targetCompatibility JavaVersion.VERSION_17
47
+ }
48
+
49
+ kotlinOptions {
50
+ jvmTarget = "17"
51
+ }
52
+
53
+ externalNativeBuild {
54
+ cmake {
55
+ path file("CMakeLists.txt")
56
+ }
57
+ }
58
+ }
59
+
60
+ dependencies {
61
+ implementation "com.facebook.react:react-native:+"
62
+ implementation project(':react-native-nitro-modules')
63
+ }
64
+
65
+ apply from: "../nitrogen/generated/android/NitroOnnxSpeech+autolinking.gradle"
@@ -0,0 +1,6 @@
1
+ <manifest xmlns:android="http://schemas.android.com/apk/res/android">
2
+ <uses-permission android:name="android.permission.INTERNET" />
3
+ <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
4
+ <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
5
+ <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
6
+ </manifest>
@@ -0,0 +1,24 @@
1
+ #include <jni.h>
2
+ #include <fbjni/fbjni.h>
3
+ #include "NitroOnnxSpeechOnLoad.hpp"
4
+ #include "ResourceDir.hpp"
5
+
6
+ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
7
+ return facebook::jni::initialize(vm, []() {
8
+ margelo::nitro::onnx::speech::registerAllNatives();
9
+ });
10
+ }
11
+
12
+ extern "C" JNIEXPORT void JNICALL
13
+ Java_com_margelo_nitro_onnx_speech_OnnxSpeechPackage_setResourceDir(JNIEnv* env, jclass, jstring dir) {
14
+ const char* utf = env->GetStringUTFChars(dir, nullptr);
15
+ margelo::nitro::onnx::speech::setResourceDir(utf);
16
+ env->ReleaseStringUTFChars(dir, utf);
17
+ }
18
+
19
+ extern "C" JNIEXPORT void JNICALL
20
+ Java_com_margelo_nitro_onnx_speech_OnnxSpeechPackage_setCacheDir(JNIEnv* env, jclass, jstring dir) {
21
+ const char* utf = env->GetStringUTFChars(dir, nullptr);
22
+ margelo::nitro::onnx::speech::setCacheDir(utf);
23
+ env->ReleaseStringUTFChars(dir, utf);
24
+ }
@@ -0,0 +1,67 @@
1
+ package com.margelo.nitro.onnx.speech
2
+
3
+ import android.content.Context
4
+ import android.util.Log
5
+ import com.facebook.react.ReactPackage
6
+ import com.facebook.react.bridge.NativeModule
7
+ import com.facebook.react.bridge.ReactApplicationContext
8
+ import com.facebook.react.uimanager.ViewManager
9
+ import java.io.File
10
+ import java.io.FileOutputStream
11
+
12
+ class OnnxSpeechPackage : ReactPackage {
13
+ companion object {
14
+ private const val TAG = "OnnxSpeechPackage"
15
+
16
+ init {
17
+ NitroOnnxSpeechOnLoad.initializeNative()
18
+ }
19
+
20
+ @JvmStatic
21
+ private external fun setResourceDir(dir: String)
22
+
23
+ @JvmStatic
24
+ private external fun setCacheDir(dir: String)
25
+
26
+ /**
27
+ * Copies bundled model files from APK assets to internal storage so that
28
+ * the C++ layer can access them via real file paths (fopen-compatible).
29
+ * Called lazily when createNativeModules is first invoked.
30
+ */
31
+ internal fun ensureResources(context: Context) {
32
+ val filesDir = context.filesDir
33
+ val marker = File(filesDir, ".resources_extracted")
34
+ if (marker.exists()) {
35
+ setResourceDir(filesDir.absolutePath)
36
+ setCacheDir(filesDir.absolutePath)
37
+ return
38
+ }
39
+ try {
40
+ copyAsset(context, "silero_vad.onnx", filesDir)
41
+ marker.createNewFile()
42
+ setResourceDir(filesDir.absolutePath)
43
+ setCacheDir(filesDir.absolutePath)
44
+ } catch (e: Exception) {
45
+ Log.e(TAG, "Failed to extract bundled resources", e)
46
+ }
47
+ }
48
+
49
+ private fun copyAsset(context: Context, name: String, destDir: File) {
50
+ val dest = File(destDir, name)
51
+ if (dest.exists()) return
52
+ context.assets.open(name).use { input ->
53
+ FileOutputStream(dest).use { output ->
54
+ input.copyTo(output)
55
+ }
56
+ }
57
+ }
58
+ }
59
+
60
+ override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
61
+ ensureResources(reactContext)
62
+ return emptyList()
63
+ }
64
+
65
+ @Suppress("OVERRIDE_DEPRECATION")
66
+ override fun createViewManagers(reactContext: ReactApplicationContext): List<ViewManager<*, *>> = emptyList()
67
+ }
Binary file