react-native-nitro-onnx 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +241 -0
- package/NitroOnnxSpeech.podspec +51 -0
- package/Package.swift +52 -0
- package/README.md +398 -0
- package/android/CMakeLists.txt +117 -0
- package/android/build.gradle +65 -0
- package/android/src/main/AndroidManifest.xml +6 -0
- package/android/src/main/assets/silero_vad.onnx +0 -0
- package/android/src/main/cpp/cpp-adapter.cpp +24 -0
- package/android/src/main/java/com/margelo/nitro/onnx/speech/OnnxSpeechPackage.kt +67 -0
- package/assets/silero_vad.onnx +0 -0
- package/cpp/AndroidPthreadCompat.cpp +16 -0
- package/cpp/AsrEngine.cpp +277 -0
- package/cpp/AsrEngine.hpp +110 -0
- package/cpp/AudioFileReader.cpp +177 -0
- package/cpp/AudioFileReader.hpp +26 -0
- package/cpp/AudioUtils.cpp +28 -0
- package/cpp/AudioUtils.hpp +38 -0
- package/cpp/ModelSingleton.hpp +51 -0
- package/cpp/NitroOnnxSpeech.cpp +68 -0
- package/cpp/NitroOnnxSpeech.hpp +42 -0
- package/cpp/OfflineAsr.cpp +81 -0
- package/cpp/OfflineAsr.hpp +33 -0
- package/cpp/ResourceDir.cpp +29 -0
- package/cpp/ResourceDir.hpp +18 -0
- package/cpp/SpeakerEngine.cpp +170 -0
- package/cpp/SpeakerEngine.hpp +70 -0
- package/cpp/SpeakerManager.cpp +93 -0
- package/cpp/SpeakerManager.hpp +43 -0
- package/cpp/StreamingAsr.cpp +123 -0
- package/cpp/StreamingAsr.hpp +55 -0
- package/cpp/ThreadPool.cpp +41 -0
- package/cpp/ThreadPool.hpp +62 -0
- package/cpp/Tts.cpp +132 -0
- package/cpp/Tts.hpp +38 -0
- package/cpp/TtsEngine.cpp +222 -0
- package/cpp/TtsEngine.hpp +77 -0
- package/cpp/Vad.cpp +126 -0
- package/cpp/Vad.hpp +62 -0
- package/cpp/VadEngine.cpp +217 -0
- package/cpp/VadEngine.hpp +126 -0
- package/ios/OnnxSpeechInitializer.mm +37 -0
- package/ios/PrivacyInfo.xcprivacy +14 -0
- package/lib/index.d.ts +11 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +19 -0
- package/lib/index.js.map +1 -0
- package/lib/specs/OnnxSpeech.nitro.d.ts +264 -0
- package/lib/specs/OnnxSpeech.nitro.d.ts.map +1 -0
- package/lib/specs/OnnxSpeech.nitro.js +6 -0
- package/lib/specs/OnnxSpeech.nitro.js.map +1 -0
- package/nitro.json +19 -0
- package/nitrogen/generated/.gitattributes +1 -0
- package/nitrogen/generated/android/NitroOnnxSpeech+autolinking.cmake +86 -0
- package/nitrogen/generated/android/NitroOnnxSpeech+autolinking.gradle +27 -0
- package/nitrogen/generated/android/NitroOnnxSpeechOnLoad.cpp +49 -0
- package/nitrogen/generated/android/NitroOnnxSpeechOnLoad.hpp +34 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/onnx/speech/NitroOnnxSpeechOnLoad.kt +35 -0
- package/nitrogen/generated/ios/NitroOnnxSpeech+autolinking.rb +62 -0
- package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Bridge.cpp +17 -0
- package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Bridge.hpp +27 -0
- package/nitrogen/generated/ios/NitroOnnxSpeech-Swift-Cxx-Umbrella.hpp +38 -0
- package/nitrogen/generated/ios/NitroOnnxSpeechAutolinking.mm +35 -0
- package/nitrogen/generated/ios/NitroOnnxSpeechAutolinking.swift +16 -0
- package/nitrogen/generated/shared/c++/AsrModelConfig.hpp +142 -0
- package/nitrogen/generated/shared/c++/AsrModelType.hpp +112 -0
- package/nitrogen/generated/shared/c++/AsrResult.hpp +105 -0
- package/nitrogen/generated/shared/c++/HybridOfflineAsrSpec.cpp +25 -0
- package/nitrogen/generated/shared/c++/HybridOfflineAsrSpec.hpp +73 -0
- package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.cpp +27 -0
- package/nitrogen/generated/shared/c++/HybridOnnxSpeechSpec.hpp +82 -0
- package/nitrogen/generated/shared/c++/HybridSpeakerManagerSpec.cpp +28 -0
- package/nitrogen/generated/shared/c++/HybridSpeakerManagerSpec.hpp +77 -0
- package/nitrogen/generated/shared/c++/HybridStreamingAsrSpec.cpp +32 -0
- package/nitrogen/generated/shared/c++/HybridStreamingAsrSpec.hpp +81 -0
- package/nitrogen/generated/shared/c++/HybridTtsSpec.cpp +26 -0
- package/nitrogen/generated/shared/c++/HybridTtsSpec.hpp +74 -0
- package/nitrogen/generated/shared/c++/HybridVadSpec.cpp +31 -0
- package/nitrogen/generated/shared/c++/HybridVadSpec.hpp +81 -0
- package/nitrogen/generated/shared/c++/RegisteredSpeaker.hpp +91 -0
- package/nitrogen/generated/shared/c++/SpeakerEmbeddingConfig.hpp +91 -0
- package/nitrogen/generated/shared/c++/TtsModelConfig.hpp +178 -0
- package/nitrogen/generated/shared/c++/TtsModelType.hpp +88 -0
- package/nitrogen/generated/shared/c++/TtsResult.hpp +91 -0
- package/nitrogen/generated/shared/c++/VadConfig.hpp +100 -0
- package/nitrogen/generated/shared/c++/VadSegment.hpp +91 -0
- package/package.json +62 -0
- package/scripts/prepare-sherpa-onnx.js +236 -0
- package/scripts/test-cpp.js +24 -0
- package/src/index.ts +45 -0
- package/src/specs/OnnxSpeech.nitro.ts +326 -0
package/README.md
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
# react-native-nitro-onnx
|
|
2
|
+
|
|
3
|
+
A React Native [Nitro Module](https://nitro.margelo.com) that wraps [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) for on-device speech processing:
|
|
4
|
+
|
|
5
|
+
- **ASR** (Automatic Speech Recognition) - offline and streaming
|
|
6
|
+
- **TTS** (Text-to-Speech)
|
|
7
|
+
- **VAD** (Voice Activity Detection) with sliding pre-buffer
|
|
8
|
+
- **Voice cloning** via speaker embeddings and reference-audio TTS
|
|
9
|
+
|
|
10
|
+
All audio I/O uses zero-copy `ArrayBuffer` with **16 kHz mono f32 PCM**.
|
|
11
|
+
|
|
12
|
+
> **Note:** This repository is a structural scaffold. Every model family has a typed config slot and a singleton-backed engine. Only one model per category is fully wired in the reference implementation; the remaining model types map to the correct sherpa-onnx C API fields and are ready for incremental completion.
|
|
13
|
+
|
|
14
|
+
## Table of Contents
|
|
15
|
+
|
|
16
|
+
- [Architecture](#architecture)
|
|
17
|
+
- [Supported Models](#supported-models)
|
|
18
|
+
- [ASR](#asr)
|
|
19
|
+
- [TTS](#tts)
|
|
20
|
+
- [VAD](#vad)
|
|
21
|
+
- [Model Downloads](#model-downloads)
|
|
22
|
+
- [Model File Requirements](#model-file-requirements)
|
|
23
|
+
- [Installation](#installation)
|
|
24
|
+
- [Usage](#usage)
|
|
25
|
+
- [Qualcomm CPU Detection](#qualcomm-cpu-detection)
|
|
26
|
+
- [VAD Pre-buffer](#vad-pre-buffer)
|
|
27
|
+
- [Voice Cloning](#voice-cloning)
|
|
28
|
+
- [Threading](#threading)
|
|
29
|
+
- [Testing](#testing)
|
|
30
|
+
- [License](#license)
|
|
31
|
+
|
|
32
|
+
## Architecture
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
┌─────────────────────────────────────────────────────────────┐
|
|
36
|
+
│ JS / TS │
|
|
37
|
+
│ getOnnxSpeech() → createVad() / createTts() / createAsr() │
|
|
38
|
+
└──────────────────────┬──────────────────────────────────────┘
|
|
39
|
+
│ react-native-nitro-modules (zero-copy ArrayBuffer)
|
|
40
|
+
┌──────────────────────┴──────────────────────────────────────┐
|
|
41
|
+
│ C++ │
|
|
42
|
+
│ OnnxSpeechImpl → VadEngine / AsrEngine / TtsEngine / ... │
|
|
43
|
+
│ ModelSingleton caches heavy recognizer / TTS instances │
|
|
44
|
+
└──────────────────────┬──────────────────────────────────────┘
|
|
45
|
+
│ sherpa-onnx C API
|
|
46
|
+
┌──────────────────────┴──────────────────────────────────────┐
|
|
47
|
+
│ ONNX Runtime │
|
|
48
|
+
└─────────────────────────────────────────────────────────────┘
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Key design decisions:
|
|
52
|
+
|
|
53
|
+
- **Singleton preloading:** `OfflineAsrEngine`, `StreamingAsrEngine`, `TtsEngine` and `SpeakerEngine` use `ModelSingleton` keyed by model directory and type. Loading the same model twice returns the same native instance.
|
|
54
|
+
- **Background inference:** Every heavy operation runs on a fixed `ThreadPool` so the JS thread never blocks.
|
|
55
|
+
- **External model download:** The module does not bundle an internal downloader. Download model files in the background with a library such as [`@kesha-antonov/react-native-background-downloader`](https://github.com/kesha-antonov/react-native-background-downloader), then pass the local file paths to `load()` / `initialize()`.
|
|
56
|
+
- **Zero-copy audio:** `ArrayBuffer` is the only audio transport format; samples are expected to be 16 kHz mono little-endian f32 PCM.
|
|
57
|
+
|
|
58
|
+
## Supported Models
|
|
59
|
+
|
|
60
|
+
### ASR
|
|
61
|
+
|
|
62
|
+
| Model family | Type slug | Architecture | Best for | Streaming | Notes |
|
|
63
|
+
|---|---|---|---|---|---|
|
|
64
|
+
| Whisper | `whisper` | encoder-decoder | Multi-language, accuracy | No | Needs `encoder.onnx`, `decoder.onnx`, `tokens.txt` |
|
|
65
|
+
| Transducer | `transducer` | RNN-T / RNNT | Streaming accuracy | Yes | Needs `encoder.onnx`, `decoder.onnx`, `joiner.onnx` |
|
|
66
|
+
| Paraformer | `paraformer` | non-autoregressive | Fast offline Chinese/English | No | Single `model.onnx` |
|
|
67
|
+
| Zipformer | `zipformer` | fast conformer variant | Streaming, low latency | Yes | Transducer triple |
|
|
68
|
+
| Conformer | `conformer` | attention-convolution | Streaming accuracy | Yes | Transducer triple |
|
|
69
|
+
| Wenet | `wenet` | U2++ / CTC | Chinese industrial | No | Single `model.onnx` |
|
|
70
|
+
| Telespeech | `telespeech` | telephony ASR | 8 kHz telco audio | No | Single `model.onnx` |
|
|
71
|
+
| Moonshine | `moonshine` | lightweight | Edge devices | No | Single `model.onnx` |
|
|
72
|
+
| Dolphin | `dolphin` | CTC | English | No | Single `model.onnx` |
|
|
73
|
+
| NeMo | `nemo` | CTC / RNNT | NVIDIA NeMo exported models | No | `model.onnx` + config |
|
|
74
|
+
| SenseVoice | `sense_voice` | multilingual | Alibaba SenseVoice | No | Single `model.onnx` |
|
|
75
|
+
|
|
76
|
+
### TTS
|
|
77
|
+
|
|
78
|
+
| Model family | Type slug | Vocoder | Quality | Speed | Notes |
|
|
79
|
+
|---|---|---|---|---|---|
|
|
80
|
+
| Kokoro | `kokoro` | internal | High, multi-speaker | Medium | Needs `model.onnx`, `voices.bin`, `tokens.txt`, `lexicon.txt` |
|
|
81
|
+
| VITS | `vits` | internal | High quality | Medium | Needs `model.onnx`, `tokens.txt`, optional lexicon |
|
|
82
|
+
| Matcha | `matcha` | external (e.g. Hifigan) | Fast, natural | Fast | Needs acoustic model + vocoder ONNX |
|
|
83
|
+
| Pocket | `pocket` | internal | Lightweight zero-shot | Very fast | Needs `model.onnx` + config JSON |
|
|
84
|
+
| ZipVoice | `zipvoice` | internal | Placeholder type | - | Maps to VITS-like config until sherpa-onnx exposes dedicated ZipVoice support |
|
|
85
|
+
|
|
86
|
+
### VAD
|
|
87
|
+
|
|
88
|
+
The module uses sherpa-onnx's Silero VAD implementation. The `silero_vad.onnx` model is bundled with the module, so you can initialize VAD without specifying `modelPath`. You may still pass a custom `modelPath` if you want to use your own ONNX VAD model.
|
|
89
|
+
|
|
90
|
+
## Model Downloads
|
|
91
|
+
|
|
92
|
+
Pretrained sherpa-onnx model families are published as GitHub release assets. Download the required files to the device (for example with [`@kesha-antonov/react-native-background-downloader`](https://github.com/kesha-antonov/react-native-background-downloader)) and pass the local paths to `load()` / `initialize()`.
|
|
93
|
+
|
|
94
|
+
- **TTS models:** https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
|
|
95
|
+
- **ASR models:** https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models
|
|
96
|
+
|
|
97
|
+
> The Silero VAD model (`silero_vad.onnx`) is bundled with the module, so no separate download is required for VAD.
|
|
98
|
+
|
|
99
|
+
## Model File Requirements
|
|
100
|
+
|
|
101
|
+
Each model family expects a specific set of ONNX and metadata files. When you call `load()` / `initialize()`, you must point every relevant path at the downloaded files on the device. The exact files depend on the model family:
|
|
102
|
+
|
|
103
|
+
### Whisper (offline ASR)
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
whisper/
|
|
107
|
+
encoder.onnx
|
|
108
|
+
decoder.onnx
|
|
109
|
+
tokens.txt
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Transducer / Zipformer / Conformer (streaming ASR)
|
|
113
|
+
|
|
114
|
+
```
|
|
115
|
+
transducer/
|
|
116
|
+
encoder.onnx
|
|
117
|
+
decoder.onnx
|
|
118
|
+
joiner.onnx
|
|
119
|
+
tokens.txt
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Paraformer / Wenet / Telespeech / Moonshine / Dolphin / SenseVoice (offline ASR)
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
model/
|
|
126
|
+
model.onnx
|
|
127
|
+
tokens.txt
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### NeMo (offline ASR)
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
nemo/
|
|
134
|
+
model.onnx
|
|
135
|
+
config.yaml
|
|
136
|
+
tokens.txt
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### Kokoro (TTS)
|
|
140
|
+
|
|
141
|
+
```
|
|
142
|
+
kokoro/
|
|
143
|
+
model.onnx
|
|
144
|
+
voices.bin
|
|
145
|
+
tokens.txt
|
|
146
|
+
lexicon.txt
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
### VITS (TTS)
|
|
150
|
+
|
|
151
|
+
```
|
|
152
|
+
vits/
|
|
153
|
+
model.onnx
|
|
154
|
+
tokens.txt
|
|
155
|
+
lexicon.txt (optional)
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
### Matcha (TTS)
|
|
159
|
+
|
|
160
|
+
```
|
|
161
|
+
matcha/
|
|
162
|
+
acoustic_model.onnx
|
|
163
|
+
vocoder.onnx
|
|
164
|
+
tokens.txt
|
|
165
|
+
lexicon.txt
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
### Pocket (TTS)
|
|
169
|
+
|
|
170
|
+
```
|
|
171
|
+
pocket/
|
|
172
|
+
model.onnx
|
|
173
|
+
config.json
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
### Silero VAD
|
|
177
|
+
|
|
178
|
+
`silero_vad.onnx` is bundled with the module and used by default. A custom model can be provided via `VadConfig.modelPath`.
|
|
179
|
+
|
|
180
|
+
### Speaker embedding (voice cloning)
|
|
181
|
+
|
|
182
|
+
```
|
|
183
|
+
speaker/
|
|
184
|
+
model.onnx
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## Installation
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
yarn add react-native-nitro-onnx
|
|
191
|
+
# or
|
|
192
|
+
npm install react-native-nitro-onnx
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Build requirements:
|
|
196
|
+
|
|
197
|
+
- React Native >= 0.78
|
|
198
|
+
- react-native-nitro-modules >= 0.35.8
|
|
199
|
+
- Xcode 15 / Android NDK 26
|
|
200
|
+
- The `prepare-sherpa-onnx.js` postinstall script downloads the sherpa-onnx
|
|
201
|
+
prebuilt tree (host static libraries, Android shared libraries, iOS
|
|
202
|
+
xcframework, and C API headers) into `cpp/sherpa-onnx-prebuilt`.
|
|
203
|
+
|
|
204
|
+
iOS:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
cd ios && pod install
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Android:
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
cd android
|
|
214
|
+
./gradlew assembleDebug
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
## Usage
|
|
218
|
+
|
|
219
|
+
```typescript
|
|
220
|
+
import { getOnnxSpeech, DEFAULT_AUDIO_FORMAT } from "react-native-nitro-onnx";
|
|
221
|
+
|
|
222
|
+
const speech = getOnnxSpeech();
|
|
223
|
+
|
|
224
|
+
// 1. Load a model by specifying the local file paths.
|
|
225
|
+
// Download the files first, e.g. with @kesha-antonov/react-native-background-downloader.
|
|
226
|
+
const modelDir = `${RNFS.DocumentDirectoryPath}/whisper-tiny`;
|
|
227
|
+
const asr = speech.createOfflineAsr();
|
|
228
|
+
await asr.load({
|
|
229
|
+
type: "whisper",
|
|
230
|
+
modelDir,
|
|
231
|
+
tokensPath: `${modelDir}/tokens.txt`,
|
|
232
|
+
whisperEncoder: `${modelDir}/encoder.onnx`,
|
|
233
|
+
whisperDecoder: `${modelDir}/decoder.onnx`,
|
|
234
|
+
numThreads: 4,
|
|
235
|
+
language: "en",
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
// 2. Recognize speech.
|
|
239
|
+
const result = await asr.recognize(pcmArrayBuffer);
|
|
240
|
+
console.log(result.text);
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
### VAD example
|
|
244
|
+
|
|
245
|
+
```typescript
|
|
246
|
+
const vad = speech.createVad();
|
|
247
|
+
await vad.initialize({
|
|
248
|
+
// modelPath is optional; the bundled silero_vad.onnx is used by default.
|
|
249
|
+
threshold: 0.5,
|
|
250
|
+
minSilenceDurationMs: 500,
|
|
251
|
+
minSpeechDurationMs: 250,
|
|
252
|
+
preBufferMs: 300,
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
// Subscribe to events.
|
|
256
|
+
vad.onSpeechStart = (segment) => {
|
|
257
|
+
console.log("speech started at", segment.startMs);
|
|
258
|
+
};
|
|
259
|
+
vad.onSpeechEnd = (segment) => {
|
|
260
|
+
console.log("speech ended at", segment.endMs, "samples", segment.samples);
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
// Feed microphone chunks.
|
|
264
|
+
await vad.process(microphoneChunk);
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
### TTS example
|
|
268
|
+
|
|
269
|
+
```typescript
|
|
270
|
+
const tts = speech.createTts();
|
|
271
|
+
const modelDir = `${RNFS.DocumentDirectoryPath}/kokoro`;
|
|
272
|
+
await tts.load({
|
|
273
|
+
type: "kokoro",
|
|
274
|
+
modelDir,
|
|
275
|
+
acousticModel: `${modelDir}/model.onnx`,
|
|
276
|
+
voices: `${modelDir}/voices.bin`,
|
|
277
|
+
tokens: `${modelDir}/tokens.txt`,
|
|
278
|
+
lexicon: `${modelDir}/lexicon.txt`,
|
|
279
|
+
numThreads: 4,
|
|
280
|
+
outputSampleRate: 16000,
|
|
281
|
+
speed: 1.0, // default speed
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
// Synthesize with optional per-call speed override
|
|
285
|
+
const audio = await tts.synthesize("Hello, this is a test.", 0.9);
|
|
286
|
+
// audio.samples is an ArrayBuffer of f32 PCM at audio.sampleRate.
|
|
287
|
+
|
|
288
|
+
// Save to WAV file
|
|
289
|
+
await tts.saveWav(audio, "/path/to/output.wav");
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
> **Important:** The synthesized audio is **f32le PCM** (32-bit float, little-endian). When playing with `react-native-audio-api`, you must specify the correct `sampleRate` when creating the `AudioContext`:
|
|
293
|
+
>
|
|
294
|
+
> ```typescript
|
|
295
|
+
> import { AudioContext } from "react-native-audio-api";
|
|
296
|
+
>
|
|
297
|
+
> const audio = await tts.synthesize("Hello");
|
|
298
|
+
> const ctx = new AudioContext({ sampleRate: audio.sampleRate });
|
|
299
|
+
> const buffer = ctx.createBuffer(1, audio.samples.byteLength / 4, audio.sampleRate);
|
|
300
|
+
> buffer.copyToChannel(new Float32Array(audio.samples), 0);
|
|
301
|
+
> const source = ctx.createBufferSource();
|
|
302
|
+
> source.buffer = buffer;
|
|
303
|
+
> source.connect(ctx.destination);
|
|
304
|
+
> source.start();
|
|
305
|
+
> ```
|
|
306
|
+
|
|
307
|
+
## Qualcomm CPU Detection
|
|
308
|
+
|
|
309
|
+
Use `isQualcommCpu()` to detect Qualcomm chipsets and select optimized QNN execution providers:
|
|
310
|
+
|
|
311
|
+
```typescript
|
|
312
|
+
const speech = getOnnxSpeech();
|
|
313
|
+
const isQualcomm = speech.isQualcommCpu();
|
|
314
|
+
|
|
315
|
+
if (isQualcomm) {
|
|
316
|
+
// Use QNN execution provider for better performance on Qualcomm chips
|
|
317
|
+
await asr.load({
|
|
318
|
+
type: "whisper",
|
|
319
|
+
// ... other config
|
|
320
|
+
provider: "qnn",
|
|
321
|
+
});
|
|
322
|
+
} else {
|
|
323
|
+
// Use CPU or other execution provider
|
|
324
|
+
await asr.load({
|
|
325
|
+
type: "whisper",
|
|
326
|
+
// ... other config
|
|
327
|
+
provider: "cpu",
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
> **Note:** `isQualcommCpu()` returns `false` on iOS. It checks `/proc/cpuinfo` on Android to detect Qualcomm processors.
|
|
333
|
+
|
|
334
|
+
## VAD Pre-buffer
|
|
335
|
+
|
|
336
|
+
A common problem with streaming VAD is that `onSpeechStart` is delivered asynchronously, so the first few frames of speech are already inside the native VAD before JS is notified. When JS later receives the segment at `onSpeechEnd`, the leading audio is clipped.
|
|
337
|
+
|
|
338
|
+
This module solves that with a **sliding pre-buffer queue**:
|
|
339
|
+
|
|
340
|
+
1. Every incoming chunk is appended to a fixed-size ring buffer (`preBufferMs`).
|
|
341
|
+
2. When sherpa-onnx detects speech, the pre-buffer is captured as the start of the active segment.
|
|
342
|
+
3. `onSpeechStart` is emitted immediately with the buffered leading audio.
|
|
343
|
+
4. Subsequent chunks are accumulated into the same segment.
|
|
344
|
+
5. When speech ends, the full accumulated segment is emitted via `onSpeechEnd` and made available through `pullSegments()`.
|
|
345
|
+
|
|
346
|
+
The result is that no speech frames are lost between detection and JS delivery.
|
|
347
|
+
|
|
348
|
+
## Voice Cloning
|
|
349
|
+
|
|
350
|
+
Two voice-cloning paths are exposed:
|
|
351
|
+
|
|
352
|
+
1. **Speaker embedding registration** - compute an embedding from reference audio, store it locally, and pass the speaker ID to TTS models that accept a speaker index (e.g. Kokoro multi-speaker, VITS multi-speaker).
|
|
353
|
+
2. **Reference-audio TTS** - models that support prompt-based or zero-shot synthesis (e.g. Pocket) receive the reference embedding directly during synthesis.
|
|
354
|
+
|
|
355
|
+
```typescript
|
|
356
|
+
const speaker = speech.createSpeakerManager();
|
|
357
|
+
await speaker.load({ modelDir: "/path/to/speaker", model: "model.onnx", numThreads: 4 });
|
|
358
|
+
|
|
359
|
+
const embedding = await speaker.computeEmbedding(referenceAudio);
|
|
360
|
+
const registered = await speaker.registerSpeaker("speaker-1", "Alice", embedding);
|
|
361
|
+
|
|
362
|
+
// Use the registered speaker with TTS (optional speed override)
|
|
363
|
+
const cloned = await tts.synthesizeWithSpeaker("Hello, I am Alice.", registered.id, 1.1);
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
## Threading
|
|
367
|
+
|
|
368
|
+
Every native inference task runs on a background thread pool:
|
|
369
|
+
|
|
370
|
+
- VAD processing
|
|
371
|
+
- Offline / streaming ASR decode
|
|
372
|
+
- TTS synthesis
|
|
373
|
+
- Speaker embedding extraction
|
|
374
|
+
|
|
375
|
+
JS calls return promises that resolve on the JS thread when the background work completes. Native event callbacks (VAD speech start/end, streaming partial/final results, ASR/TTS errors) are dispatched to JS without blocking inference.
|
|
376
|
+
|
|
377
|
+
## Testing
|
|
378
|
+
|
|
379
|
+
TypeScript tests:
|
|
380
|
+
|
|
381
|
+
```bash
|
|
382
|
+
yarn test
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
C++ tests (do not require a real model):
|
|
386
|
+
|
|
387
|
+
```bash
|
|
388
|
+
yarn test:cpp
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
The C++ test suite covers:
|
|
392
|
+
|
|
393
|
+
- Audio sample / millisecond conversions
|
|
394
|
+
- Float vector / byte buffer round-trip
|
|
395
|
+
|
|
396
|
+
## License
|
|
397
|
+
|
|
398
|
+
MIT
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.13)
|
|
2
|
+
project(NitroOnnxSpeech)
|
|
3
|
+
|
|
4
|
+
set(CMAKE_CXX_STANDARD 20)
|
|
5
|
+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|
6
|
+
set(CMAKE_C_STANDARD 11)
|
|
7
|
+
|
|
8
|
+
# ------------------------------------------------------------------------------
|
|
9
|
+
# Optional C++ unit-test target. Built separately from the React Native library.
|
|
10
|
+
# ------------------------------------------------------------------------------
|
|
11
|
+
option(BUILD_ONNX_SPEECH_TESTS "Build C++ unit tests" OFF)
|
|
12
|
+
|
|
13
|
+
if(BUILD_ONNX_SPEECH_TESTS)
|
|
14
|
+
enable_testing()
|
|
15
|
+
add_executable(NitroOnnxSpeechTests
|
|
16
|
+
../cpp/tests/test_core.cpp
|
|
17
|
+
../cpp/ThreadPool.cpp
|
|
18
|
+
../cpp/AudioUtils.cpp
|
|
19
|
+
)
|
|
20
|
+
target_include_directories(NitroOnnxSpeechTests PRIVATE ../cpp)
|
|
21
|
+
find_package(Threads REQUIRED)
|
|
22
|
+
target_link_libraries(NitroOnnxSpeechTests Threads::Threads)
|
|
23
|
+
add_test(NAME NitroOnnxSpeechTests COMMAND NitroOnnxSpeechTests)
|
|
24
|
+
return()
|
|
25
|
+
endif()
|
|
26
|
+
|
|
27
|
+
# ------------------------------------------------------------------------------
|
|
28
|
+
# React Native Nitro module build
|
|
29
|
+
# ------------------------------------------------------------------------------
|
|
30
|
+
find_package(react-native-nitro-modules REQUIRED CONFIG)
|
|
31
|
+
|
|
32
|
+
# Sherpa ONNX: the prebuilt tree is fixed at cpp/sherpa-onnx-prebuilt.
|
|
33
|
+
# If it is missing, run the prepare script automatically at configure time.
|
|
34
|
+
set(_ONNX_PREBUILT_DIR "${CMAKE_SOURCE_DIR}/../cpp/sherpa-onnx-prebuilt")
|
|
35
|
+
if(NOT EXISTS "${_ONNX_PREBUILT_DIR}")
|
|
36
|
+
find_program(NODE_EXECUTABLE node REQUIRED)
|
|
37
|
+
message(STATUS "sherpa-onnx prebuilt not found; running prepare-sherpa-onnx.js...")
|
|
38
|
+
execute_process(
|
|
39
|
+
COMMAND ${NODE_EXECUTABLE} ../scripts/prepare-sherpa-onnx.js
|
|
40
|
+
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
|
41
|
+
RESULT_VARIABLE _PREPARE_RESULT
|
|
42
|
+
OUTPUT_VARIABLE _PREPARE_OUTPUT
|
|
43
|
+
ERROR_VARIABLE _PREPARE_ERROR
|
|
44
|
+
)
|
|
45
|
+
if(NOT _PREPARE_RESULT EQUAL 0)
|
|
46
|
+
message(FATAL_ERROR
|
|
47
|
+
"Failed to run prepare-sherpa-onnx.js:\n${_PREPARE_OUTPUT}\n${_PREPARE_ERROR}"
|
|
48
|
+
)
|
|
49
|
+
endif()
|
|
50
|
+
endif()
|
|
51
|
+
if(NOT EXISTS "${_ONNX_PREBUILT_DIR}")
|
|
52
|
+
message(FATAL_ERROR
|
|
53
|
+
"sherpa-onnx prebuilt is missing at cpp/sherpa-onnx-prebuilt. "
|
|
54
|
+
"Run 'yarn install' to download a prebuilt library."
|
|
55
|
+
)
|
|
56
|
+
endif()
|
|
57
|
+
|
|
58
|
+
file(GLOB_RECURSE ONNX_SPEECH_SOURCES "../cpp/*.cpp")
|
|
59
|
+
list(FILTER ONNX_SPEECH_SOURCES EXCLUDE REGEX "cpp/tests/.*")
|
|
60
|
+
list(FILTER ONNX_SPEECH_SOURCES EXCLUDE REGEX "cpp/build-.*/.*")
|
|
61
|
+
|
|
62
|
+
add_library(${PROJECT_NAME} SHARED
|
|
63
|
+
${ONNX_SPEECH_SOURCES}
|
|
64
|
+
src/main/cpp/cpp-adapter.cpp
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
target_include_directories(${PROJECT_NAME} PRIVATE
|
|
68
|
+
"${_ONNX_PREBUILT_DIR}/include"
|
|
69
|
+
"../cpp"
|
|
70
|
+
"../nitrogen/generated/android"
|
|
71
|
+
"../nitrogen/generated/shared/c++"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
include(${CMAKE_SOURCE_DIR}/../nitrogen/generated/android/NitroOnnxSpeech+autolinking.cmake)
|
|
75
|
+
|
|
76
|
+
if(ANDROID)
|
|
77
|
+
# Android prebuilt ships shared libraries per ABI.
|
|
78
|
+
set(_ANDROID_ABI_DIR "${_ONNX_PREBUILT_DIR}/android/jniLibs/${ANDROID_ABI}")
|
|
79
|
+
if(NOT EXISTS "${_ANDROID_ABI_DIR}")
|
|
80
|
+
message(FATAL_ERROR
|
|
81
|
+
"Missing sherpa-onnx Android libraries for ABI ${ANDROID_ABI}. "
|
|
82
|
+
"Run 'yarn install' to download the prebuilt tree."
|
|
83
|
+
)
|
|
84
|
+
endif()
|
|
85
|
+
|
|
86
|
+
add_library(sherpa-onnx-c-api SHARED IMPORTED)
|
|
87
|
+
set_target_properties(sherpa-onnx-c-api PROPERTIES
|
|
88
|
+
IMPORTED_LOCATION "${_ANDROID_ABI_DIR}/libsherpa-onnx-c-api.so"
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
add_library(onnxruntime SHARED IMPORTED)
|
|
92
|
+
set_target_properties(onnxruntime PROPERTIES
|
|
93
|
+
IMPORTED_LOCATION "${_ANDROID_ABI_DIR}/libonnxruntime.so"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
target_link_libraries(${PROJECT_NAME}
|
|
97
|
+
sherpa-onnx-c-api
|
|
98
|
+
onnxruntime
|
|
99
|
+
log
|
|
100
|
+
)
|
|
101
|
+
else()
|
|
102
|
+
# Host static library layout. Mainly useful for local C++ development.
|
|
103
|
+
target_link_libraries(${PROJECT_NAME}
|
|
104
|
+
sherpa-onnx-c-api
|
|
105
|
+
sherpa-onnx-core
|
|
106
|
+
onnxruntime
|
|
107
|
+
)
|
|
108
|
+
endif()
|
|
109
|
+
|
|
110
|
+
if(ANDROID)
|
|
111
|
+
# Skip CMake's pthread feature probe on Android. Bionic has pthread built-in
|
|
112
|
+
# but lacks pthread_cancel; the probe fails under -Werror=implicit-function-declaration.
|
|
113
|
+
set(THREADS_HAVE_PTHREAD_ARG FALSE)
|
|
114
|
+
set(CMAKE_HAVE_LIBC_PTHREAD TRUE)
|
|
115
|
+
endif()
|
|
116
|
+
find_package(Threads REQUIRED)
|
|
117
|
+
target_link_libraries(${PROJECT_NAME} Threads::Threads)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
buildscript {
|
|
2
|
+
ext.safeExtGet = {prop, fallback ->
|
|
3
|
+
rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
|
|
4
|
+
}
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
def reactNativeArchitectures() {
|
|
8
|
+
def value = rootProject.getProperties().get("reactNativeArchitectures")
|
|
9
|
+
return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
apply plugin: 'com.android.library'
|
|
13
|
+
apply plugin: 'kotlin-android'
|
|
14
|
+
|
|
15
|
+
android {
|
|
16
|
+
namespace "com.margelo.nitro.onnx.speech"
|
|
17
|
+
compileSdk safeExtGet("compileSdkVersion", 34)
|
|
18
|
+
|
|
19
|
+
defaultConfig {
|
|
20
|
+
minSdk safeExtGet("minSdkVersion", 21)
|
|
21
|
+
externalNativeBuild {
|
|
22
|
+
cmake {
|
|
23
|
+
cppFlags "-std=c++20 -O3 -fexceptions -frtti"
|
|
24
|
+
arguments "-DANDROID_STL=c++_shared"
|
|
25
|
+
abiFilters (*reactNativeArchitectures())
|
|
26
|
+
|
|
27
|
+
buildTypes {
|
|
28
|
+
debug {
|
|
29
|
+
cppFlags "-O1 -g"
|
|
30
|
+
}
|
|
31
|
+
release {
|
|
32
|
+
cppFlags "-O2"
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
buildFeatures {
|
|
40
|
+
buildConfig false
|
|
41
|
+
prefab true
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
compileOptions {
|
|
45
|
+
sourceCompatibility JavaVersion.VERSION_17
|
|
46
|
+
targetCompatibility JavaVersion.VERSION_17
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
kotlinOptions {
|
|
50
|
+
jvmTarget = "17"
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
externalNativeBuild {
|
|
54
|
+
cmake {
|
|
55
|
+
path file("CMakeLists.txt")
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
dependencies {
|
|
61
|
+
implementation "com.facebook.react:react-native:+"
|
|
62
|
+
implementation project(':react-native-nitro-modules')
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
apply from: "../nitrogen/generated/android/NitroOnnxSpeech+autolinking.gradle"
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
|
2
|
+
<uses-permission android:name="android.permission.INTERNET" />
|
|
3
|
+
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
|
4
|
+
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
|
|
5
|
+
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
|
|
6
|
+
</manifest>
|
|
Binary file
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#include <jni.h>
|
|
2
|
+
#include <fbjni/fbjni.h>
|
|
3
|
+
#include "NitroOnnxSpeechOnLoad.hpp"
|
|
4
|
+
#include "ResourceDir.hpp"
|
|
5
|
+
|
|
6
|
+
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
|
|
7
|
+
return facebook::jni::initialize(vm, []() {
|
|
8
|
+
margelo::nitro::onnx::speech::registerAllNatives();
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
extern "C" JNIEXPORT void JNICALL
|
|
13
|
+
Java_com_margelo_nitro_onnx_speech_OnnxSpeechPackage_setResourceDir(JNIEnv* env, jclass, jstring dir) {
|
|
14
|
+
const char* utf = env->GetStringUTFChars(dir, nullptr);
|
|
15
|
+
margelo::nitro::onnx::speech::setResourceDir(utf);
|
|
16
|
+
env->ReleaseStringUTFChars(dir, utf);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
extern "C" JNIEXPORT void JNICALL
|
|
20
|
+
Java_com_margelo_nitro_onnx_speech_OnnxSpeechPackage_setCacheDir(JNIEnv* env, jclass, jstring dir) {
|
|
21
|
+
const char* utf = env->GetStringUTFChars(dir, nullptr);
|
|
22
|
+
margelo::nitro::onnx::speech::setCacheDir(utf);
|
|
23
|
+
env->ReleaseStringUTFChars(dir, utf);
|
|
24
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
package com.margelo.nitro.onnx.speech
|
|
2
|
+
|
|
3
|
+
import android.content.Context
|
|
4
|
+
import android.util.Log
|
|
5
|
+
import com.facebook.react.ReactPackage
|
|
6
|
+
import com.facebook.react.bridge.NativeModule
|
|
7
|
+
import com.facebook.react.bridge.ReactApplicationContext
|
|
8
|
+
import com.facebook.react.uimanager.ViewManager
|
|
9
|
+
import java.io.File
|
|
10
|
+
import java.io.FileOutputStream
|
|
11
|
+
|
|
12
|
+
class OnnxSpeechPackage : ReactPackage {
|
|
13
|
+
companion object {
|
|
14
|
+
private const val TAG = "OnnxSpeechPackage"
|
|
15
|
+
|
|
16
|
+
init {
|
|
17
|
+
NitroOnnxSpeechOnLoad.initializeNative()
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
@JvmStatic
|
|
21
|
+
private external fun setResourceDir(dir: String)
|
|
22
|
+
|
|
23
|
+
@JvmStatic
|
|
24
|
+
private external fun setCacheDir(dir: String)
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Copies bundled model files from APK assets to internal storage so that
|
|
28
|
+
* the C++ layer can access them via real file paths (fopen-compatible).
|
|
29
|
+
* Called lazily when createNativeModules is first invoked.
|
|
30
|
+
*/
|
|
31
|
+
internal fun ensureResources(context: Context) {
|
|
32
|
+
val filesDir = context.filesDir
|
|
33
|
+
val marker = File(filesDir, ".resources_extracted")
|
|
34
|
+
if (marker.exists()) {
|
|
35
|
+
setResourceDir(filesDir.absolutePath)
|
|
36
|
+
setCacheDir(filesDir.absolutePath)
|
|
37
|
+
return
|
|
38
|
+
}
|
|
39
|
+
try {
|
|
40
|
+
copyAsset(context, "silero_vad.onnx", filesDir)
|
|
41
|
+
marker.createNewFile()
|
|
42
|
+
setResourceDir(filesDir.absolutePath)
|
|
43
|
+
setCacheDir(filesDir.absolutePath)
|
|
44
|
+
} catch (e: Exception) {
|
|
45
|
+
Log.e(TAG, "Failed to extract bundled resources", e)
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
private fun copyAsset(context: Context, name: String, destDir: File) {
|
|
50
|
+
val dest = File(destDir, name)
|
|
51
|
+
if (dest.exists()) return
|
|
52
|
+
context.assets.open(name).use { input ->
|
|
53
|
+
FileOutputStream(dest).use { output ->
|
|
54
|
+
input.copyTo(output)
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
|
|
61
|
+
ensureResources(reactContext)
|
|
62
|
+
return emptyList()
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
@Suppress("OVERRIDE_DEPRECATION")
|
|
66
|
+
override fun createViewManagers(reactContext: ReactApplicationContext): List<ViewManager<*, *>> = emptyList()
|
|
67
|
+
}
|
|
Binary file
|