react-native-sherpa-onnx 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/README.md +95 -265
  2. package/SherpaOnnx.podspec +68 -64
  3. package/android/build.gradle +182 -192
  4. package/android/codegen.gradle +57 -0
  5. package/android/prebuilt-download.gradle +428 -0
  6. package/android/prebuilt-versions.gradle +43 -0
  7. package/android/proguard-rules.pro +10 -0
  8. package/android/src/main/assets/testModels/add_mul_add.onnx +28 -0
  9. package/android/src/main/assets/testModels/nnapi_internal_uint8_support.onnx +0 -0
  10. package/android/src/main/assets/testModels/qnn_multi_ctx_embed.onnx +0 -0
  11. package/android/src/main/cpp/CMakeLists.txt +126 -81
  12. package/android/src/main/cpp/CMakePresets.json +54 -0
  13. package/android/src/main/cpp/crypto/sha256.cpp +174 -0
  14. package/android/src/main/cpp/crypto/sha256.h +16 -0
  15. package/android/src/main/cpp/jni/archive/sherpa-onnx-archive-helper.cpp +404 -0
  16. package/android/src/main/cpp/jni/archive/sherpa-onnx-archive-helper.h +56 -0
  17. package/android/src/main/cpp/jni/archive/sherpa-onnx-archive-jni.cpp +181 -0
  18. package/android/src/main/cpp/jni/audio/sherpa-onnx-audio-convert-jni.cpp +888 -0
  19. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-common.h +18 -0
  20. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-detect-jni-common.cpp +86 -0
  21. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-detect-jni-common.h +20 -0
  22. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-model-detect-helper.cpp +423 -0
  23. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-model-detect-helper.h +55 -0
  24. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-model-detect-stt.cpp +399 -0
  25. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-model-detect-tts.cpp +238 -0
  26. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-model-detect.h +122 -0
  27. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-stt-wrapper.cpp +99 -0
  28. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-stt-wrapper.h +16 -0
  29. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-tts-wrapper.cpp +78 -0
  30. package/android/src/main/cpp/jni/model_detect/sherpa-onnx-tts-wrapper.h +16 -0
  31. package/android/src/main/cpp/jni/module/sherpa-onnx-module-jni.cpp +190 -0
  32. package/android/src/main/cpp/jni/tts/sherpa-onnx-tts-zipvoice-jni.cpp +301 -0
  33. package/android/src/main/java/com/sherpaonnx/SherpaOnnxArchiveHelper.kt +94 -0
  34. package/android/src/main/java/com/sherpaonnx/SherpaOnnxAssetHelper.kt +350 -0
  35. package/android/src/main/java/com/sherpaonnx/SherpaOnnxModule.kt +675 -200
  36. package/android/src/main/java/com/sherpaonnx/SherpaOnnxSttHelper.kt +699 -0
  37. package/android/src/main/java/com/sherpaonnx/SherpaOnnxTtsHelper.kt +1123 -0
  38. package/android/src/main/java/com/sherpaonnx/ZipvoiceTtsWrapper.kt +187 -0
  39. package/ios/SherpaOnnx+Assets.h +11 -0
  40. package/ios/SherpaOnnx+Assets.mm +325 -0
  41. package/ios/SherpaOnnx+STT.mm +455 -0
  42. package/ios/SherpaOnnx+TTS.mm +1101 -0
  43. package/ios/SherpaOnnx.h +13 -1
  44. package/ios/SherpaOnnx.mm +146 -233
  45. package/ios/SherpaOnnx.xcconfig +19 -19
  46. package/ios/SherpaOnnxCoreMLHelper.swift +24 -0
  47. package/ios/archive/sherpa-onnx-archive-helper.h +21 -0
  48. package/ios/archive/sherpa-onnx-archive-helper.mm +296 -0
  49. package/ios/libarchive_darwin_config.h +153 -0
  50. package/ios/model_detect/sherpa-onnx-common.h +18 -0
  51. package/ios/model_detect/sherpa-onnx-model-detect-helper.h +49 -0
  52. package/ios/model_detect/sherpa-onnx-model-detect-helper.mm +210 -0
  53. package/ios/model_detect/sherpa-onnx-model-detect-stt.mm +344 -0
  54. package/ios/model_detect/sherpa-onnx-model-detect-tts.mm +201 -0
  55. package/ios/model_detect/sherpa-onnx-model-detect.h +117 -0
  56. package/ios/scripts/patch-libarchive-includes.sh +61 -0
  57. package/ios/scripts/setup-ios-libarchive.sh +98 -0
  58. package/ios/stt/sherpa-onnx-stt-wrapper.h +129 -0
  59. package/ios/stt/sherpa-onnx-stt-wrapper.mm +523 -0
  60. package/ios/tts/sherpa-onnx-tts-wrapper.h +90 -0
  61. package/ios/tts/sherpa-onnx-tts-wrapper.mm +376 -0
  62. package/lib/module/NativeSherpaOnnx.js +3 -0
  63. package/lib/module/NativeSherpaOnnx.js.map +1 -1
  64. package/lib/module/audio/index.js +22 -0
  65. package/lib/module/audio/index.js.map +1 -0
  66. package/lib/module/diarization/index.js +1 -1
  67. package/lib/module/diarization/index.js.map +1 -1
  68. package/lib/module/download/ModelDownloadManager.js +918 -0
  69. package/lib/module/download/ModelDownloadManager.js.map +1 -0
  70. package/lib/module/download/extractTarBz2.js +53 -0
  71. package/lib/module/download/extractTarBz2.js.map +1 -0
  72. package/lib/module/download/index.js +6 -0
  73. package/lib/module/download/index.js.map +1 -0
  74. package/lib/module/download/validation.js +178 -0
  75. package/lib/module/download/validation.js.map +1 -0
  76. package/lib/module/enhancement/index.js +1 -1
  77. package/lib/module/enhancement/index.js.map +1 -1
  78. package/lib/module/index.js +47 -11
  79. package/lib/module/index.js.map +1 -1
  80. package/lib/module/separation/index.js +1 -1
  81. package/lib/module/separation/index.js.map +1 -1
  82. package/lib/module/stt/index.js +119 -49
  83. package/lib/module/stt/index.js.map +1 -1
  84. package/lib/module/stt/sttModelLanguages.js +512 -0
  85. package/lib/module/stt/sttModelLanguages.js.map +1 -0
  86. package/lib/module/stt/types.js +53 -1
  87. package/lib/module/stt/types.js.map +1 -1
  88. package/lib/module/tts/index.js +241 -26
  89. package/lib/module/tts/index.js.map +1 -1
  90. package/lib/module/tts/types.js +89 -0
  91. package/lib/module/tts/types.js.map +1 -0
  92. package/lib/module/types.js.map +1 -1
  93. package/lib/module/utils.js +62 -17
  94. package/lib/module/utils.js.map +1 -1
  95. package/lib/module/vad/index.js +1 -1
  96. package/lib/module/vad/index.js.map +1 -1
  97. package/lib/typescript/src/NativeSherpaOnnx.d.ts +332 -12
  98. package/lib/typescript/src/NativeSherpaOnnx.d.ts.map +1 -1
  99. package/lib/typescript/src/audio/index.d.ts +13 -0
  100. package/lib/typescript/src/audio/index.d.ts.map +1 -0
  101. package/lib/typescript/src/diarization/index.d.ts +3 -2
  102. package/lib/typescript/src/diarization/index.d.ts.map +1 -1
  103. package/lib/typescript/src/download/ModelDownloadManager.d.ts +108 -0
  104. package/lib/typescript/src/download/ModelDownloadManager.d.ts.map +1 -0
  105. package/lib/typescript/src/download/extractTarBz2.d.ts +14 -0
  106. package/lib/typescript/src/download/extractTarBz2.d.ts.map +1 -0
  107. package/lib/typescript/src/download/index.d.ts +7 -0
  108. package/lib/typescript/src/download/index.d.ts.map +1 -0
  109. package/lib/typescript/src/download/validation.d.ts +57 -0
  110. package/lib/typescript/src/download/validation.d.ts.map +1 -0
  111. package/lib/typescript/src/enhancement/index.d.ts +3 -2
  112. package/lib/typescript/src/enhancement/index.d.ts.map +1 -1
  113. package/lib/typescript/src/index.d.ts +26 -4
  114. package/lib/typescript/src/index.d.ts.map +1 -1
  115. package/lib/typescript/src/separation/index.d.ts +3 -2
  116. package/lib/typescript/src/separation/index.d.ts.map +1 -1
  117. package/lib/typescript/src/stt/index.d.ts +37 -40
  118. package/lib/typescript/src/stt/index.d.ts.map +1 -1
  119. package/lib/typescript/src/stt/sttModelLanguages.d.ts +52 -0
  120. package/lib/typescript/src/stt/sttModelLanguages.d.ts.map +1 -0
  121. package/lib/typescript/src/stt/types.d.ts +196 -9
  122. package/lib/typescript/src/stt/types.d.ts.map +1 -1
  123. package/lib/typescript/src/tts/index.d.ts +47 -29
  124. package/lib/typescript/src/tts/index.d.ts.map +1 -1
  125. package/lib/typescript/src/tts/types.d.ts +311 -0
  126. package/lib/typescript/src/tts/types.d.ts.map +1 -0
  127. package/lib/typescript/src/types.d.ts +0 -32
  128. package/lib/typescript/src/types.d.ts.map +1 -1
  129. package/lib/typescript/src/utils.d.ts +59 -12
  130. package/lib/typescript/src/utils.d.ts.map +1 -1
  131. package/lib/typescript/src/vad/index.d.ts +3 -2
  132. package/lib/typescript/src/vad/index.d.ts.map +1 -1
  133. package/package.json +44 -15
  134. package/scripts/check-qnn-support.sh +78 -0
  135. package/scripts/setup-ios-framework.sh +379 -282
  136. package/scripts/switch-registry.js +8 -8
  137. package/src/NativeSherpaOnnx.ts +445 -15
  138. package/src/audio/index.ts +32 -0
  139. package/src/diarization/index.ts +4 -2
  140. package/src/download/ModelDownloadManager.ts +1325 -0
  141. package/src/download/extractTarBz2.ts +78 -0
  142. package/src/download/index.ts +43 -0
  143. package/src/download/validation.ts +279 -0
  144. package/src/enhancement/index.ts +4 -2
  145. package/src/index.tsx +61 -13
  146. package/src/separation/index.ts +4 -2
  147. package/src/stt/index.ts +222 -56
  148. package/src/stt/sttModelLanguages.ts +237 -0
  149. package/src/stt/types.ts +263 -9
  150. package/src/tts/index.ts +441 -38
  151. package/src/tts/types.ts +373 -0
  152. package/src/types.ts +0 -44
  153. package/src/utils.ts +70 -22
  154. package/src/vad/index.ts +4 -2
  155. package/third_party/ffmpeg_prebuilt/ANDROID_RELEASE_TAG +1 -0
  156. package/third_party/libarchive_prebuilt/ANDROID_RELEASE_TAG +1 -0
  157. package/third_party/libarchive_prebuilt/IOS_RELEASE_TAG +1 -0
  158. package/third_party/sherpa-onnx-prebuilt/ANDROID_RELEASE_TAG +1 -0
  159. package/third_party/sherpa-onnx-prebuilt/IOS_RELEASE_TAG +1 -0
  160. package/android/src/main/cpp/include/sherpa-onnx/c-api/c-api.h +0 -1918
  161. package/android/src/main/cpp/include/sherpa-onnx/c-api/cxx-api.h +0 -841
  162. package/android/src/main/cpp/jni/sherpa-onnx-jni.cpp +0 -129
  163. package/android/src/main/cpp/jni/sherpa-onnx-wrapper.cpp +0 -649
  164. package/android/src/main/cpp/jni/sherpa-onnx-wrapper.h +0 -56
  165. package/ios/Frameworks/sherpa_onnx.xcframework.zip +0 -0
  166. package/ios/include/sherpa-onnx/c-api/c-api.h +0 -1918
  167. package/ios/include/sherpa-onnx/c-api/cxx-api.h +0 -841
  168. package/ios/sherpa-onnx-wrapper.h +0 -57
  169. package/ios/sherpa-onnx-wrapper.mm +0 -432
  170. package/scripts/copy-headers.js +0 -184
  171. package/scripts/setup-assets.js +0 -323
package/README.md CHANGED
@@ -2,50 +2,75 @@
2
2
 
3
3
  React Native SDK for sherpa-onnx - providing offline speech processing capabilities
4
4
 
5
+ <div align="center">
6
+ <img src="./docs/images/banner.png" alt="Banner" width="560" />
7
+ </div>
8
+
9
+ <div align="center">
10
+
5
11
  [![npm version](https://img.shields.io/npm/v/react-native-sherpa-onnx.svg)](https://www.npmjs.com/package/react-native-sherpa-onnx)
6
12
  [![npm downloads](https://img.shields.io/npm/dm/react-native-sherpa-onnx.svg)](https://www.npmjs.com/package/react-native-sherpa-onnx)
7
13
  [![npm license](https://img.shields.io/npm/l/react-native-sherpa-onnx.svg)](https://www.npmjs.com/package/react-native-sherpa-onnx)
8
14
  [![Android](https://img.shields.io/badge/Android-Supported-green)](https://www.android.com/)
9
15
  [![iOS](https://img.shields.io/badge/iOS-Supported-blue)](https://www.apple.com/ios/)
10
16
 
17
+ </div>
18
+
19
+ > **⚠️ SDK 0.3.0 – Breaking changes from 0.2.0**
20
+ > Since the last release I have restructured and improved the SDK significantly: full iOS support, smoother behaviour, fewer failure points, and a much smaller footprint (~95% size reduction). As a result, **logic and the public API have changed**. If you are upgrading from 0.2.x, please follow the [Breaking changes (upgrading to 0.3.0)](docs/migration.md#breaking-changes-upgrading-to-030) section and the updated API documentation
21
+
11
22
  A React Native TurboModule that provides offline speech processing capabilities using [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx). The SDK aims to support all functionalities that sherpa-onnx offers, including offline speech-to-text, text-to-speech, speaker diarization, speech enhancement, source separation, and VAD (Voice Activity Detection).
12
23
 
24
+ ## Table of contents
25
+
26
+ - [Feature Support](#feature-support)
27
+ - [Platform Support Status](#platform-support-status)
28
+ - [Supported Model Types](#supported-model-types)
29
+ - [Speech-to-Text (STT) Models](#speech-to-text-stt-models)
30
+ - [Text-to-Speech (TTS) Models](#text-to-speech-tts-models)
31
+ - [Installation](#installation)
32
+ - [Android](#android)
33
+ - [iOS](#ios)
34
+ - [Documentation](#documentation)
35
+ - [Requirements](#requirements)
36
+ - [Breaking changes (upgrading to 0.3.0)](#breaking-changes-upgrading-to-030)
37
+ - [Instance-based API (TTS + STT)](#instance-based-api-tts--stt)
38
+ - [Speech-to-Text (STT)](#speech-to-text-stt)
39
+ - [Text-to-Speech (TTS)](#text-to-speech-tts)
40
+ - [Example Apps](#example-apps)
41
+ - [Example App (Audio to Text)](#example-app-audio-to-text)
42
+ - [Video to Text Comparison App](#video-to-text-comparison-app)
43
+ - [Contributing](#contributing)
44
+ - [License](#license)
45
+
13
46
  ## Feature Support
14
47
 
15
- | Feature | Status |
16
- |---------|--------|
17
- | Offline Speech-to-Text | ✅ Supported |
18
- | Text-to-Speech | Not yet supported |
19
- | Speaker Diarization | Not yet supported |
20
- | Speech Enhancement | Not yet supported |
21
- | Source Separation | Not yet supported |
22
- | VAD (Voice Activity Detection) | Not yet supported |
48
+ | Feature | Status | Notes |
49
+ |---------|--------|-------|
50
+ | Offline Speech-to-Text | ✅ **Supported** | No internet required; multiple model types (Zipformer, Paraformer, Whisper, etc.). See [Supported Model Types](#supported-model-types). |
51
+ | Text-to-Speech | **Supported** | Multiple model types (VITS, Matcha, Kokoro, etc.). See [Supported Model Types](#supported-model-types). |
52
+ | Execution providers (CPU, NNAPI, XNNPACK, Core ML, QNN) | **Supported** | See [Execution provider support](./docs/execution-providers.md). |
53
+ | Play Asset Delivery (PAD) | **Supported** | Android only. See [Model Setup](./docs/MODEL_SETUP.md). |
54
+ | Automatic Model type detection | **Supported** | `detectSttModel()` and `detectTtsModel()` for a path. See [Model Setup: Model type detection](./docs/MODEL_SETUP.md#model-type-detection-without-initialization). |
55
+ | Model quantization | **Supported** | Automatic detection and preference for quantized (int8) models. |
56
+ | Flexible model loading | ✅ **Supported** | Asset models, file system models, or auto-detection. |
57
+ | TypeScript | ✅ **Supported** | Full type definitions included. |
58
+ | Speaker Diarization | ❌ Not yet supported | Scheduled for release 0.4.0 |
59
+ | Speech Enhancement | ❌ Not yet supported | Scheduled for release 0.5.0 |
60
+ | Source Separation | ❌ Not yet supported | Scheduled for release 0.6.0 |
61
+ | VAD (Voice Activity Detection) | ❌ Not yet supported | Scheduled for release 0.7.0 |
23
62
 
24
63
  ## Platform Support Status
25
64
 
26
65
  | Platform | Status | Notes |
27
66
  |----------|--------|-------|
28
- | **Android** | ✅ **Production Ready** | Fully tested, CI/CD automated, multiple models supported |
29
- | **iOS** | 🟡 **Beta / Experimental** | XCFramework + Podspec ready<br/>✅ GitHub Actions builds pass<br/>❌ **No local Xcode testing** *(Windows-only dev)* |
30
-
31
- ### 🔧 **iOS Contributors WANTED!** 🙌
32
-
33
- **Full iOS support is a priority!** Help bring sherpa-onnx to iOS devices.
34
-
35
- **What's ready:**
36
- - ✅ XCFramework integration
37
- - ✅ Podspec configuration
38
- - ✅ GitHub Actions CI (macOS runner)
39
- - ✅ TypeScript bindings
40
-
41
- **What's needed:**
42
- - **Local Xcode testing** (Simulator + Device)
43
- - **iOS example app** (beyond CI)
44
- - **TurboModule iOS testing**
45
- - **Edge case testing**
67
+ | **Android** | ✅ **Production Ready** | CI/CD automated, multiple models supported |
68
+ | **iOS** | **Production Ready** | CI/CD automated, multiple models supported |
46
69
 
47
70
  ## Supported Model Types
48
71
 
72
+ ### Speech-to-Text (STT) Models
73
+
49
74
  | Model Type | `modelType` Value | Description | Download Links |
50
75
  | ------------------------ | ----------------- | ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ |
51
76
  | **Zipformer/Transducer** | `'transducer'` | Requires `encoder.onnx`, `decoder.onnx`, `joiner.onnx`, and `tokens.txt` | [Download](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/index.html) |
@@ -56,16 +81,16 @@ A React Native TurboModule that provides offline speech processing capabilities
56
81
  | **SenseVoice** | `'sense_voice'` | Requires `model.onnx` (or `model.int8.onnx`) and `tokens.txt` | [Download](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/sense-voice/index.html) |
57
82
  | **FunASR Nano** | `'funasr_nano'` | Requires `encoder_adaptor.onnx`, `llm.onnx`, `embedding.onnx`, and `tokenizer` directory | [Download](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/funasr-nano/index.html) |
58
83
 
59
- ## Features
84
+ ### Text-to-Speech (TTS) Models
60
85
 
61
- - **Offline Speech-to-Text** - No internet connection required for speech recognition
62
- - **Multiple Model Types** - Supports Zipformer/Transducer, Paraformer, NeMo CTC, Whisper, WeNet CTC, SenseVoice, and FunASR Nano models
63
- - **Model Quantization** - Automatic detection and preference for quantized (int8) models
64
- - **Flexible Model Loading** - Asset models, file system models, or auto-detection
65
- - **Android Support** - Fully supported on Android
66
- - **iOS Support** - Fully supported on iOS (requires sherpa-onnx XCFramework)
67
- - **TypeScript Support** - Full TypeScript definitions included
68
- - 🚧 **Additional Features Coming Soon** - Text-to-Speech, Speaker Diarization, Speech Enhancement, Source Separation, and VAD support are planned for future releases
86
+ | Model Type | `modelType` Value | Description | Download Links |
87
+ | ---------------- | ----------------- | ---------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- |
88
+ | **VITS** | `'vits'` | Fast, high-quality TTS. Includes Piper, Coqui, MeloTTS, MMS variants. Requires `model.onnx`, `tokens.txt` | [Download](https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models) |
89
+ | **Matcha** | `'matcha'` | High-quality acoustic model + vocoder. Requires `acoustic_model.onnx`, `vocoder.onnx`, `tokens.txt` | [Download](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html) |
90
+ | **Kokoro** | `'kokoro'` | Multi-speaker, multi-language. Requires `model.onnx`, `voices.bin`, `tokens.txt`, `espeak-ng-data/` | [Download](https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models) |
91
+ | **KittenTTS** | `'kitten'` | Lightweight, multi-speaker. Requires `model.onnx`, `voices.bin`, `tokens.txt`, `espeak-ng-data/` | [Download](https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models) |
92
+ | **Zipvoice** | `'zipvoice'` | Voice cloning capable. Requires `encoder.onnx`, `decoder.onnx`, `vocoder.onnx`, `tokens.txt` | [Download](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/zipvoice.html) |
93
+ | **Pocket** | `'pocket'` | Flow-matching TTS. Requires `lm_flow.onnx`, `lm_main.onnx`, `encoder.onnx`, `decoder.onnx`, `text_conditioner.onnx`, `vocab.json`, `token_scores.json` | [Download](https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models) |
69
94
 
70
95
  ## Installation
71
96
 
@@ -88,255 +113,51 @@ YARN_NODE_LINKER=node-modules yarn install
88
113
 
89
114
  ### Android
90
115
 
91
- No additional setup required. The library automatically handles native dependencies via Gradle.
116
+ No additional setup required. The library automatically handles native dependencies via Gradle. For execution provider support (CPU, NNAPI, XNNPACK, QNN) and optional QNN setup, see [Execution provider support](./docs/execution-providers.md). For building Android native libs yourself, see [sherpa-onnx-prebuilt](third_party/sherpa-onnx-prebuilt/README.md).
117
+
92
118
 
93
119
  ### iOS
94
120
 
95
- The sherpa-onnx XCFramework is **not included in the repository or npm package** due to its size (~80MB), but **no manual action is required**! The framework is automatically downloaded during `pod install`.
121
+ The sherpa-onnx **XCFramework is not shipped in the repo or npm** (size ~80MB). It is **downloaded automatically** when you run `pod install`; no manual steps are required. The version used is pinned in `third_party/sherpa-onnx-prebuilt/IOS_RELEASE_TAG` and the archive is fetched from [GitHub Releases](https://github.com/XDcobra/react-native-sherpa-onnx/releases?q=framework).
96
122
 
97
- #### Quick Setup
123
+ #### Setup
98
124
 
99
125
  ```sh
100
- cd example
126
+ cd your-app/ios
101
127
  bundle install
102
- bundle exec pod install --project-directory=ios
128
+ bundle exec pod install
103
129
  ```
104
130
 
105
- That's it! The `Podfile` automatically:
106
- 1. Copies required header files from the git submodule
107
- 2. Downloads the latest XCFramework from [GitHub Releases](https://github.com/XDcobra/react-native-sherpa-onnx/releases?q=framework)
108
- 3. Verifies everything is in place before building
131
+ The podspec runs `scripts/setup-ios-framework.sh`, which downloads the XCFramework (and, if needed, libarchive sources) so the Pod builds correctly. Libarchive is compiled from source as part of the Pod; its version is pinned in `third_party/libarchive_prebuilt/IOS_RELEASE_TAG`.
109
132
 
110
133
  #### For Advanced Users: Building the Framework Locally
134
+ #### Advanced: Building the iOS framework yourself
111
135
 
112
- If you want to build the XCFramework yourself instead of using the prebuilt release:
113
-
114
- ```sh
115
- # Clone sherpa-onnx repository
116
- git clone https://github.com/k2-fsa/sherpa-onnx.git
117
- cd sherpa-onnx
118
- git checkout v1.12.23
119
-
120
- # Build the iOS XCFramework (requires macOS, Xcode, CMake, and ONNX Runtime)
121
- ./build-ios.sh
122
-
123
- # Copy to your project
124
- cp -r build-ios/sherpa_onnx.xcframework /path/to/react-native-sherpa-onnx/ios/Frameworks/
125
- ```
126
-
127
- Then run `pod install` as usual.
128
-
129
- **Note:** The iOS implementation uses the same C++ wrapper as Android, ensuring consistent behavior across platforms.
136
+ If you need a custom sherpa-onnx build (e.g. different version or patches), you can build the XCFramework and place it in `ios/Frameworks/` before running `pod install`. The repo does not include an iOS build script; use one of:
130
137
 
131
- ## Quick Start
138
+ - **This repo's CI:** The [build-sherpa-onnx-ios-framework](.github/workflows/build-sherpa-onnx-ios-framework.yml) workflow produces the XCFramework and publishes it as a GitHub Release. You can run equivalent steps locally or inspect the workflow for the exact build and merge steps (including `libsherpa-onnx-cxx-api.a` and libarchive).
139
+ - **Version and layout:** Pinned version and release layout are documented in [third_party/sherpa-onnx-prebuilt](third_party/sherpa-onnx-prebuilt/README.md) (Android focus; for iOS, see `IOS_RELEASE_TAG` and the [iOS framework workflow](.github/workflows/build-sherpa-onnx-ios-framework.yml)).
132
140
 
133
- ```typescript
134
- import { resolveModelPath } from 'react-native-sherpa-onnx';
135
- import {
136
- initializeSTT,
137
- transcribeFile,
138
- unloadSTT,
139
- } from 'react-native-sherpa-onnx/stt';
141
+ The XCFramework must include the C++ API (`libsherpa-onnx-cxx-api.a` merged or linked) so that the iOS Obj-C++ code can use `sherpa_onnx::cxx::*`. The workflow's build script ensures this; if you use upstream `build-ios.sh` from sherpa-onnx, you may need to merge the C++ API into the static library yourself.
140
142
 
141
- // Initialize with a model
142
- const modelPath = await resolveModelPath({
143
- type: 'asset',
144
- path: 'models/sherpa-onnx-model',
145
- });
143
+ ## Documentation
146
144
 
147
- await initializeSTT({
148
- modelPath: modelPath,
149
- preferInt8: true, // Optional: prefer quantized models
150
- });
151
-
152
- // Transcribe an audio file
153
- const transcription = await transcribeFile('path/to/audio.wav');
154
- console.log('Transcription:', transcription);
155
-
156
- // Release resources when done
157
- await unloadSTT();
158
- ```
159
-
160
- ## Usage
161
-
162
- ### Initialization
163
-
164
- ```typescript
165
- import {
166
- initializeSherpaOnnx,
167
- assetModelPath,
168
- autoModelPath,
169
- } from 'react-native-sherpa-onnx';
170
-
171
- // Option 1: Asset model (bundled in app)
172
- await initializeSherpaOnnx({
173
- modelPath: assetModelPath('models/sherpa-onnx-model'),
174
- preferInt8: true, // Prefer quantized models
175
- });
176
-
177
- // Option 2: Auto-detect (tries asset, then file system)
178
- await initializeSherpaOnnx({
179
- modelPath: autoModelPath('models/sherpa-onnx-model'),
180
- });
181
-
182
- // Option 3: Simple string (backward compatible)
183
- await initializeSherpaOnnx('models/sherpa-onnx-model');
184
- ```
185
-
186
- ### Transcription (Speech-to-Text)
187
-
188
- ```typescript
189
- import { transcribeFile } from 'react-native-sherpa-onnx/stt';
190
-
191
- // Transcribe a WAV file (16kHz, mono, 16-bit PCM)
192
- const result = await transcribeFile('path/to/audio.wav');
193
- console.log('Transcription:', result);
194
- ```
195
-
196
- ### Model Quantization
197
-
198
- Control whether to prefer quantized (int8) or regular models:
199
-
200
- ```typescript
201
- import { initializeSTT } from 'react-native-sherpa-onnx/stt';
202
- import { resolveModelPath } from 'react-native-sherpa-onnx';
203
-
204
- const modelPath = await resolveModelPath({
205
- type: 'asset',
206
- path: 'models/my-model',
207
- });
208
-
209
- // Default: try int8 first, then regular
210
- await initializeSTT({ modelPath });
211
-
212
- // Explicitly prefer int8 models (smaller, faster)
213
- await initializeSTT({
214
- modelPath,
215
- preferInt8: true,
216
- });
217
-
218
- // Explicitly prefer regular models (higher accuracy)
219
- await initializeSTT({
220
- modelPath,
221
- preferInt8: false,
222
- });
223
- ```
145
+ - [Text-to-Speech (TTS)](./docs/tts.md)
146
+ - [Execution provider support (QNN, NNAPI, XNNPACK, Core ML)](./docs/execution-providers.md) – Checking and using acceleration backends
147
+ - [Voice Activity Detection (VAD)](./docs/vad.md)
148
+ - [Speaker Diarization](./docs/diarization.md)
149
+ - [Speech Enhancement](./docs/enhancement.md)
150
+ - [Source Separation](./docs/separation.md)
151
+ - [Model Setup](./docs/MODEL_SETUP.md) – Bundled assets, Play Asset Delivery (PAD), model discovery APIs, and troubleshooting
152
+ - [Model Download Manager](./docs/download-manager.md)
224
153
 
225
- ### Explicit Model Type
226
-
227
- For robustness, you can explicitly specify the model type to avoid auto-detection issues:
228
-
229
- ```typescript
230
- import { initializeSTT } from 'react-native-sherpa-onnx/stt';
231
- import { resolveModelPath } from 'react-native-sherpa-onnx';
232
-
233
- const modelPath = await resolveModelPath({
234
- type: 'asset',
235
- path: 'models/sherpa-onnx-nemo-parakeet-tdt-ctc-en',
236
- });
237
-
238
- // Explicitly specify model type
239
- await initializeSTT({
240
- modelPath,
241
- modelType: 'nemo_ctc', // 'transducer', 'paraformer', 'nemo_ctc', 'whisper', 'wenet_ctc', 'sense_voice', 'funasr_nano'
242
- });
243
-
244
- // Auto-detection (default behavior)
245
- await initializeSTT({
246
- modelPath,
247
- // modelType defaults to 'auto'
248
- });
249
- ```
250
-
251
- ### Cleanup (Speech-to-Text)
252
-
253
- ```typescript
254
- import { unloadSTT } from 'react-native-sherpa-onnx/stt';
255
-
256
- // Release resources when done
257
- await unloadSTT();
258
- ```
259
-
260
- ## Model Setup
261
-
262
- The library does **not** bundle models. You must provide your own models. See [MODEL_SETUP.md](./MODEL_SETUP.md) for detailed setup instructions.
263
-
264
- ### Model File Requirements
265
-
266
- - **Zipformer/Transducer**: Requires `encoder.onnx`, `decoder.onnx`, `joiner.onnx`, and `tokens.txt`
267
- - **Paraformer**: Requires `model.onnx` (or `model.int8.onnx`) and `tokens.txt`
268
- - **NeMo CTC**: Requires `model.onnx` (or `model.int8.onnx`) and `tokens.txt`
269
- - **Whisper**: Requires `encoder.onnx`, `decoder.onnx`, and `tokens.txt`
270
- - **WeNet CTC**: Requires `model.onnx` (or `model.int8.onnx`) and `tokens.txt`
271
- - **SenseVoice**: Requires `model.onnx` (or `model.int8.onnx`) and `tokens.txt`
272
-
273
- ### Model Files
274
-
275
- Place models in:
276
-
277
- - **Android**: `android/app/src/main/assets/models/`
278
- - **iOS**: Add to Xcode project as folder reference
279
-
280
- ## API Reference
281
-
282
- ### Speech-to-Text (STT) Module
283
-
284
- Import from `react-native-sherpa-onnx/stt`:
285
-
286
- #### `initializeSTT(options)`
287
-
288
- Initialize the speech-to-text engine with a model.
289
-
290
- **Parameters:**
291
-
292
- - `options.modelPath`: Absolute path to the model directory
293
- - `options.preferInt8` (optional): Prefer quantized models (`true`), regular models (`false`), or auto-detect (`undefined`, default)
294
- - `options.modelType` (optional): Explicit model type (`'transducer'`, `'paraformer'`, `'nemo_ctc'`, `'whisper'`, `'wenet_ctc'`, `'sense_voice'`, `'funasr_nano'`), or auto-detect (`'auto'`, default)
295
-
296
- **Returns:** `Promise<void>`
297
-
298
- #### `transcribeFile(filePath)`
299
-
300
- Transcribe an audio file.
301
-
302
- **Parameters:**
303
-
304
- - `filePath`: Path to WAV file (16kHz, mono, 16-bit PCM)
305
-
306
- **Returns:** `Promise<string>` - Transcribed text
307
-
308
- #### `unloadSTT()`
309
-
310
- Release resources and unload the speech-to-text model.
311
-
312
- **Returns:** `Promise<void>`
313
-
314
- ### Utility Functions
315
-
316
- Import from `react-native-sherpa-onnx`:
317
-
318
- #### `resolveModelPath(config)`
319
-
320
- Resolve a model path configuration to an absolute path.
321
-
322
- **Parameters:**
323
-
324
- - `config.type`: Path type (`'asset'`, `'file'`, or `'auto'`)
325
- - `config.path`: Path to resolve (relative for assets, absolute for files)
326
-
327
- **Returns:** `Promise<string>` - Absolute path to model directory
328
-
329
- #### `testSherpaInit()`
330
-
331
- Test that the sherpa-onnx native module is properly loaded.
332
-
333
- **Returns:** `Promise<string>` - Test message confirming module is loaded
154
+ Note: For when to use `listAssetModels()` vs `listModelsAtPath()` and how to combine bundled and PAD/file-based models, see [Model Setup](./docs/MODEL_SETUP.md).
334
155
 
335
156
  ## Requirements
336
157
 
337
158
  - React Native >= 0.70
338
159
  - Android API 24+ (Android 7.0+)
339
- - iOS 13.0+ (requires sherpa-onnx XCFramework - see iOS Setup below)
160
+ - iOS 13.0+
340
161
 
341
162
  ## Example Apps
342
163
 
@@ -360,9 +181,17 @@ yarn android # or yarn ios
360
181
  ```
361
182
 
362
183
  <div align="center">
363
- <img src="./docs/images/example_home_screen.png" alt="Model selection home screen" width="30%" />
364
- <img src="./docs/images/example_english.png" alt="Transcribe english audio" width="30%" />
365
- <img src="./docs/images/example_multilanguage.png" alt="Transcribe english and chinese audio" width="30%" />
184
+ <table>
185
+ <tr>
186
+ <td><img src="./docs/images/example_home_screen.png" alt="Model selection home screen" width="240" /></td>
187
+ <td><img src="./docs/images/example_stt_1.png" alt="Transcribe english audio" width="240" /></td>
188
+ <td><img src="./docs/images/example_stt_2.png" alt="Transcribe cantonese audio" width="240" /></td>
189
+ </tr>
190
+ <tr>
191
+ <td><img src="./docs/images/example_tts.png" alt="Text to speech generation" width="240" /></td>
192
+ <td><img src="./docs/images/example_provider.png" alt="Text to speech generation" width="240" /></td>
193
+ </tr>
194
+ </table>
366
195
  </div>
367
196
 
368
197
  ### Video to Text Comparison App
@@ -400,3 +229,4 @@ MIT
400
229
  ---
401
230
 
402
231
  Made with [create-react-native-library](https://github.com/callstack/react-native-builder-bob)
232
+
@@ -1,11 +1,41 @@
1
1
  require "json"
2
2
 
3
3
  package = JSON.parse(File.read(File.join(__dir__, "package.json")))
4
-
5
- # Compute absolute paths
6
4
  pod_root = __dir__
7
- ios_include_path = File.join(pod_root, 'ios', 'include')
8
- framework_path = File.join(pod_root, 'ios', 'Frameworks', 'sherpa_onnx.xcframework')
5
+ # Prefer libarchive_prebuilt layout (output of third_party/libarchive_prebuilt/build_libarchive_ios.sh).
6
+ # Fallback: download via setup-ios-libarchive.sh to ios/Downloads/libarchive (e.g. when using SDK from npm).
7
+ libarchive_prebuilt = File.join(pod_root, "third_party", "libarchive_prebuilt", "libarchive-ios-layout")
8
+ libarchive_downloads = File.join(pod_root, "ios", "Downloads", "libarchive")
9
+ unless File.directory?(libarchive_prebuilt) && Dir.glob(File.join(libarchive_prebuilt, "*.c")).any?
10
+ libarchive_script = File.join(pod_root, "ios", "scripts", "setup-ios-libarchive.sh")
11
+ if File.exist?(libarchive_script)
12
+ unless system("bash", libarchive_script)
13
+ abort("[SherpaOnnx] setup-ios-libarchive.sh failed. Check that third_party/libarchive_prebuilt/IOS_RELEASE_TAG exists and the release is available (network). Run the script manually: bash #{libarchive_script}")
14
+ end
15
+ end
16
+ end
17
+ libarchive_dir = (File.directory?(libarchive_prebuilt) && Dir.glob(File.join(libarchive_prebuilt, "*.c")).any?) ? libarchive_prebuilt : libarchive_downloads
18
+ # Patch libarchive .c files (copy to ios/patched_libarchive with stdio.h/unistd.h added) so we don't modify the submodule.
19
+ patched_dir = File.join(pod_root, "ios", "patched_libarchive")
20
+ patch_script = File.join(pod_root, "ios", "scripts", "patch-libarchive-includes.sh")
21
+ if File.directory?(libarchive_dir) && File.exist?(patch_script)
22
+ unless system("bash", patch_script, libarchive_dir)
23
+ abort("[SherpaOnnx] patch-libarchive-includes.sh failed. Check that #{libarchive_dir} contains libarchive .c/.h files.")
24
+ end
25
+ end
26
+ # Libarchive C sources: use patched copies (same exclude as before: test, windows, linux, sunos, freebsd).
27
+ libarchive_sources = if File.directory?(patched_dir)
28
+ Dir.glob(File.join(patched_dir, "*.c")).reject { |f|
29
+ base = File.basename(f, ".c")
30
+ File.basename(f) =~ /^test\./ || base.include?("windows") || base.include?("linux") || base.include?("sunos") || base.include?("freebsd")
31
+ }.map { |f| Pathname.new(f).relative_path_from(Pathname.new(pod_root)).to_s.gsub("\\", "/") }
32
+ else
33
+ []
34
+ end
35
+
36
+ if libarchive_sources.empty?
37
+ abort("[SherpaOnnx] Libarchive sources missing. Ensure third_party/libarchive_prebuilt/libarchive-ios-layout exists (run third_party/libarchive_prebuilt/build_libarchive_ios.sh) or ios/scripts/setup-ios-libarchive.sh has run, and that ios/scripts/patch-libarchive-includes.sh succeeds. Check pod install logs for patch script errors.")
38
+ end
9
39
 
10
40
  Pod::Spec.new do |s|
11
41
  s.name = "SherpaOnnx"
@@ -17,68 +47,42 @@ Pod::Spec.new do |s|
17
47
 
18
48
  s.platforms = { :ios => min_ios_version_supported }
19
49
  s.source = { :git => "https://github.com/XDcobra/react-native-sherpa-onnx.git", :tag => "#{s.version}" }
20
-
21
- # Source files (implementation)
22
- # Include .cc for cxx-api.cc (C++ wrapper around C API)
23
- s.source_files = "ios/**/*.{h,m,mm,swift,cpp,cc}"
24
-
25
- # Private headers (our wrapper headers)
26
- s.private_header_files = [
27
- "ios/*.h",
28
- "ios/include/**/*.h"
29
- ]
30
-
31
- # Link with required frameworks and libraries
32
- # CoreML is required by ONNX Runtime's CoreML execution provider
33
- s.frameworks = 'Foundation', 'Accelerate', 'CoreML'
34
- s.libraries = 'c++'
35
-
36
- # Note: Header files and framework are set up by postinstall script (yarn setup-assets)
37
- # This runs automatically after yarn/npm install and handles all setup tasks
38
-
39
- # Verify XCFramework exists
40
- unless File.exist?(framework_path)
41
- raise <<~MSG
42
- [SherpaOnnx] ERROR: iOS Framework not found.
43
-
44
- The sherpa-onnx XCFramework should have been downloaded automatically during pod install.
45
- If the automatic download failed, you can manually download it by running:
46
-
47
- yarn download-ios-framework
48
-
49
- Or download from GitHub Releases:
50
- https://github.com/XDcobra/react-native-sherpa-onnx/releases?q=framework
51
-
52
- Then extract to: #{framework_path}
53
- MSG
54
- end
55
-
56
- # Log paths for debugging (visible during pod install)
57
- puts "[SherpaOnnx] Pod root: #{pod_root}"
58
- puts "[SherpaOnnx] Include path: #{ios_include_path}"
59
- puts "[SherpaOnnx] Framework path: #{framework_path}"
60
- framework_version = File.read(File.join(pod_root, 'ios', 'Frameworks', '.framework-version')).strip rescue 'unknown'
61
- puts "[SherpaOnnx] Framework version: #{framework_version}"
62
-
63
- # Use vendored_frameworks for the XCFramework
64
- s.vendored_frameworks = 'ios/Frameworks/sherpa_onnx.xcframework'
65
-
66
- # Preserve headers and config files
67
- s.preserve_paths = [
68
- 'ios/SherpaOnnx.xcconfig',
69
- 'ios/include/**/*'
70
- ]
71
-
50
+
51
+ # Download sherpa-onnx XCFramework from GitHub Releases before pod install (uses IOS_RELEASE_TAG for pinned version).
52
+ setup_script = File.join(pod_root, "scripts", "setup-ios-framework.sh")
53
+ s.prepare_command = "bash \"#{setup_script}\""
54
+
55
+ s.source_files = ["ios/**/*.{h,m,mm,swift,cpp}", *libarchive_sources]
56
+ s.private_header_files = "ios/**/*.h"
57
+
58
+ s.frameworks = "Foundation", "Accelerate", "CoreML"
59
+ s.vendored_frameworks = "ios/Frameworks/sherpa_onnx.xcframework"
60
+ # Absolute paths so headers are found regardless of PODS_TARGET_SRCROOT (e.g. when building via React Native CLI).
61
+ xcframework_root = File.join(pod_root, "ios", "Frameworks", "sherpa_onnx.xcframework")
62
+ simulator_headers = File.join(xcframework_root, "ios-arm64_x86_64-simulator", "Headers")
63
+ device_headers = File.join(xcframework_root, "ios-arm64", "Headers")
64
+ simulator_slice = File.join(xcframework_root, "ios-arm64_x86_64-simulator")
65
+ device_slice = File.join(xcframework_root, "ios-arm64")
66
+
72
67
  s.pod_target_xcconfig = {
73
- 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17',
74
- 'CLANG_CXX_LIBRARY' => 'libc++',
75
- 'HEADER_SEARCH_PATHS' => "$(inherited) \"#{ios_include_path}\"",
68
+ "HEADER_SEARCH_PATHS" => "$(inherited) \"#{pod_root}/ios\" \"#{pod_root}/ios/archive\" \"#{pod_root}/ios/model_detect\" \"#{pod_root}/ios/stt\" \"#{pod_root}/ios/tts\" \"#{libarchive_dir}\" \"#{device_headers}\" \"#{simulator_headers}\"",
69
+ "GCC_PREPROCESSOR_DEFINITIONS" => '$(inherited) PLATFORM_CONFIG_H=\\"libarchive_darwin_config.h\\"',
70
+ "CLANG_CXX_LANGUAGE_STANDARD" => "c++17",
71
+ "CLANG_CXX_LIBRARY" => "libc++",
72
+ "LIBRARY_SEARCH_PATHS[sdk=iphoneos*]" => "$(inherited) \"#{device_slice}\"",
73
+ "LIBRARY_SEARCH_PATHS[sdk=iphonesimulator*]" => "$(inherited) \"#{simulator_slice}\"",
74
+ "OTHER_LDFLAGS" => "$(inherited) -lsherpa-onnx"
76
75
  }
77
-
76
+
78
77
  s.user_target_xcconfig = {
79
- 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17',
80
- 'CLANG_CXX_LIBRARY' => 'libc++',
78
+ "CLANG_CXX_LANGUAGE_STANDARD" => "c++17",
79
+ "CLANG_CXX_LIBRARY" => "libc++",
80
+ "LIBRARY_SEARCH_PATHS[sdk=iphoneos*]" => "$(inherited) \"#{device_slice}\"",
81
+ "LIBRARY_SEARCH_PATHS[sdk=iphonesimulator*]" => "$(inherited) \"#{simulator_slice}\"",
82
+ "OTHER_LDFLAGS" => "$(inherited) -lsherpa-onnx"
81
83
  }
82
-
84
+
85
+ s.libraries = "c++", "z"
86
+
83
87
  install_modules_dependencies(s)
84
- end
88
+ end