react-native-sherpa-onnx 0.3.6 → 0.3.7

package/ios/tts/sherpa-onnx-tts-wrapper.mm

@@ -38,6 +38,7 @@ class TtsWrapper::Impl {
 public:
     bool initialized = false;
     std::string modelDir;
+    TtsModelKind modelKind = TtsModelKind::kUnknown;
     std::optional<sherpa_onnx::cxx::OfflineTts> tts;
 };
 
@@ -86,6 +87,7 @@ TtsInitializeResult TtsWrapper::initialize(
 
         auto detect = DetectTtsModel(modelDir, modelType);
         if (!detect.ok) {
+            result.error = detect.error;
             LOGE("%s", detect.error.c_str());
             return result;
         }
@@ -144,12 +146,24 @@ TtsInitializeResult TtsWrapper::initialize(
                 config.model.zipvoice.vocoder = detect.paths.vocoder;
                 config.model.zipvoice.tokens = detect.paths.tokens;
                 config.model.zipvoice.data_dir = detect.paths.dataDir;
+                if (!detect.paths.lexicon.empty()) {
+                    config.model.zipvoice.lexicon = detect.paths.lexicon;
+                }
+                // Limit peak RAM (same idea as Android Zipvoice init).
+                config.model.num_threads = 1;
                 break;
             case TtsModelKind::kPocket:
-                LOGE("TTS: Pocket model type is detected but not yet supported on iOS");
-                return result;
+                config.model.pocket.lm_flow = detect.paths.lmFlow;
+                config.model.pocket.lm_main = detect.paths.lmMain;
+                config.model.pocket.encoder = detect.paths.encoder;
+                config.model.pocket.decoder = detect.paths.decoder;
+                config.model.pocket.text_conditioner = detect.paths.textConditioner;
+                config.model.pocket.vocab_json = detect.paths.vocabJson;
+                config.model.pocket.token_scores_json = detect.paths.tokenScoresJson;
+                break;
             case TtsModelKind::kUnknown:
             default:
+                result.error = "TTS: Unknown model type: " + modelType;
                 LOGE("TTS: Unknown model type: %s", modelType.c_str());
                 return result;
         }
@@ -167,16 +181,35 @@ TtsInitializeResult TtsWrapper::initialize(
             config.silence_scale = *silenceScale;
         }
 
+        // Log paths passed to sherpa-onnx C++ API to diagnose /usr/share/espeak-ng-data fallback.
+        LOGI("TTS: modelDir=%s", modelDir.c_str());
+        switch (detect.selectedKind) {
+            case TtsModelKind::kVits:
+                LOGI("TTS: vits data_dir=%s (empty=%d)", detect.paths.dataDir.empty() ? "(empty)" : detect.paths.dataDir.c_str(), (int)detect.paths.dataDir.empty());
+                break;
+            case TtsModelKind::kMatcha:
+                LOGI("TTS: matcha data_dir=%s (empty=%d)", detect.paths.dataDir.empty() ? "(empty)" : detect.paths.dataDir.c_str(), (int)detect.paths.dataDir.empty());
+                break;
+            case TtsModelKind::kKokoro:
+            case TtsModelKind::kKitten:
+            case TtsModelKind::kZipvoice:
+                LOGI("TTS: data_dir=%s (empty=%d)", detect.paths.dataDir.empty() ? "(empty)" : detect.paths.dataDir.c_str(), (int)detect.paths.dataDir.empty());
+                break;
+            default:
+                break;
+        }
         LOGI("TTS: Creating OfflineTts instance...");
         pImpl->tts = sherpa_onnx::cxx::OfflineTts::Create(config);
 
         if (!pImpl->tts.has_value()) {
-            LOGE("TTS: Failed to create OfflineTts instance");
+            result.error = "TTS: Failed to create OfflineTts instance (e.g. missing espeak-ng data or invalid model)";
+            LOGE("%s", result.error.c_str());
             return result;
         }
 
         pImpl->initialized = true;
         pImpl->modelDir = modelDir;
+        pImpl->modelKind = detect.selectedKind;
 
         LOGI("TTS: Initialization successful");
         LOGI("TTS: Sample rate: %d Hz", pImpl->tts.value().SampleRate());
@@ -186,9 +219,11 @@ TtsInitializeResult TtsWrapper::initialize(
         result.detectedModels = detect.detectedModels;
         return result;
     } catch (const std::exception& e) {
+        result.error = std::string("TTS init exception: ") + e.what();
         LOGE("TTS: Exception during initialization: %s", e.what());
         return result;
     } catch (...) {
+        result.error = "TTS: Unknown exception during initialization";
         LOGE("TTS: Unknown exception during initialization");
         return result;
     }
@@ -234,6 +269,55 @@ TtsWrapper::AudioResult TtsWrapper::generate(
     }
 }
 
+TtsWrapper::AudioResult TtsWrapper::generate(
+    const std::string& text,
+    int32_t sid,
+    float speed,
+    const std::optional<VoiceCloneOptions>& cloning
+) {
+    if (!cloning.has_value() || cloning->reference_audio.empty() ||
+        cloning->reference_sample_rate <= 0) {
+        return generate(text, sid, speed);
+    }
+
+    AudioResult result;
+    result.sampleRate = 0;
+
+    if (!pImpl->initialized || !pImpl->tts.has_value()) {
+        LOGE("TTS: Not initialized. Call initialize() first.");
+        return result;
+    }
+
+    if (text.empty()) {
+        LOGE("TTS: Input text is empty");
+        return result;
+    }
+
+    try {
+        sherpa_onnx::cxx::GenerationConfig gc;
+        gc.silence_scale = cloning->silence_scale;
+        gc.speed = speed;
+        gc.sid = sid;
+        gc.reference_audio = cloning->reference_audio;
+        gc.reference_sample_rate = cloning->reference_sample_rate;
+        gc.reference_text = cloning->reference_text;
+        gc.num_steps = cloning->num_steps;
+        gc.extra = cloning->extra;
+
+        auto audio = pImpl->tts.value().Generate(text, gc);
+        result.samples = std::move(audio.samples);
+        result.sampleRate = audio.sample_rate;
+        LOGI("TTS: Generated (voice clone) %zu samples at %d Hz", result.samples.size(), result.sampleRate);
+        return result;
+    } catch (const std::exception& e) {
+        LOGE("TTS: Exception during generation (clone): %s", e.what());
+        return result;
+    } catch (...) {
+        LOGE("TTS: Unknown exception during generation (clone)");
+        return result;
+    }
+}
+
 bool TtsWrapper::generateStream(
     const std::string& text,
     int32_t sid,
@@ -279,6 +363,63 @@ bool TtsWrapper::generateStream(
     }
 }
 
+bool TtsWrapper::generateStream(
+    const std::string& text,
+    int32_t sid,
+    float speed,
+    const TtsStreamCallback& callback,
+    const std::optional<VoiceCloneOptions>& cloning
+) {
+    if (!cloning.has_value() || cloning->reference_audio.empty() ||
+        cloning->reference_sample_rate <= 0) {
+        return generateStream(text, sid, speed, callback);
+    }
+
+    if (!pImpl->initialized || !pImpl->tts.has_value()) {
+        LOGE("TTS: Not initialized. Call initialize() first.");
+        return false;
+    }
+
+    if (text.empty()) {
+        LOGE("TTS: Input text is empty");
+        return false;
+    }
+
+    try {
+        auto callbackCopy = callback;
+        auto shim = [](const float *samples, int32_t numSamples, float progress, void *arg) -> int32_t {
+            auto *cb = reinterpret_cast<TtsStreamCallback*>(arg);
+            if (!cb || !(*cb)) return 0;
+            return (*cb)(samples, numSamples, progress);
+        };
+
+        sherpa_onnx::cxx::GenerationConfig gc;
+        gc.silence_scale = cloning->silence_scale;
+        gc.speed = speed;
+        gc.sid = sid;
+        gc.reference_audio = cloning->reference_audio;
+        gc.reference_sample_rate = cloning->reference_sample_rate;
+        gc.reference_text = cloning->reference_text;
+        gc.num_steps = cloning->num_steps;
+        gc.extra = cloning->extra;
+
+        pImpl->tts.value().Generate(
+            text,
+            gc,
+            callbackCopy ? shim : nullptr,
+            callbackCopy ? &callbackCopy : nullptr
+        );
+
+        return true;
+    } catch (const std::exception& e) {
+        LOGE("TTS: Exception during streaming generation (clone): %s", e.what());
+        return false;
+    } catch (...) {
+        LOGE("TTS: Unknown exception during streaming generation (clone)");
+        return false;
+    }
+}
+
 int32_t TtsWrapper::getSampleRate() const {
     if (!pImpl->initialized || !pImpl->tts.has_value()) {
         LOGE("TTS: Not initialized. Call initialize() first.");
@@ -304,10 +445,15 @@ void TtsWrapper::release() {
         pImpl->tts.reset();
         pImpl->initialized = false;
         pImpl->modelDir.clear();
+        pImpl->modelKind = TtsModelKind::kUnknown;
         LOGI("TTS: Resources released");
     }
 }
 
+TtsModelKind TtsWrapper::getModelKind() const {
+    return pImpl->modelKind;
+}
+
 bool TtsWrapper::saveToWavFile(
     const std::vector<float>& samples,
     int32_t sampleRate,

package/lib/module/NativeSherpaOnnx.js.map

@@ -1 +1 @@
-{"version":3,"names":["TurboModuleRegistry","getEnforcing"],"sourceRoot":"../../src","sources":["NativeSherpaOnnx.ts"],"mappings":";;AAAA,SAASA,mBAAmB,QAA0B,cAAc;;AAEpE;;AAgpBA,eAAeA,mBAAmB,CAACC,YAAY,CAAO,YAAY,CAAC","ignoreList":[]}
+{"version":3,"names":["TurboModuleRegistry","getEnforcing"],"sourceRoot":"../../src","sources":["NativeSherpaOnnx.ts"],"mappings":";;AAAA,SAASA,mBAAmB,QAA0B,cAAc;;AAEpE;;AA+qBA,eAAeA,mBAAmB,CAACC,YAAY,CAAO,YAAY,CAAC","ignoreList":[]}

package/lib/module/audio/index.js

@@ -73,4 +73,12 @@ export function convertAudioToFormat(inputPath, outputPath, format, outputSample
 export function convertAudioToWav16k(inputPath, outputPath) {
   return SherpaOnnx.convertAudioToWav16k(inputPath, outputPath);
 }
+/**
+ * Decode a supported audio file to mono float PCM in [-1, 1] plus sample rate.
+ * Same decode coverage as {@link convertAudioToFormat} (FFmpeg-backed on Android when not WAV).
+ * @param targetSampleRateHz - Resample to this rate when > 0; use native decoded rate when 0 or omitted.
+ */
+export function decodeAudioFileToFloatSamples(inputPath, targetSampleRateHz) {
+  return SherpaOnnx.decodeAudioFileToFloatSamples(inputPath, targetSampleRateHz ?? 0);
+}
 //# sourceMappingURL=index.js.map

package/lib/module/audio/index.js.map

@@ -1 +1 @@
-{"version":3,"names":["Buffer","DeviceEventEmitter","SherpaOnnx","base64PcmToFloatArray","base64","bytes","from","view","DataView","buffer","byteOffset","byteLength","len","out","Float32Array","i","getInt16","createPcmLiveStream","options","sampleRate","channelCount","bufferSizeFrames","start","startPcmLiveStream","stop","stopPcmLiveStream","onData","callback","sub","addListener","event","base64Pcm","sr","samples","remove","onError","message","convertAudioToFormat","inputPath","outputPath","format","outputSampleRateHz","convertAudioToWav16k"],"sourceRoot":"../../../src","sources":["audio/index.ts"],"mappings":";;AAAA,SAASA,MAAM,QAAQ,QAAQ;AAC/B,SAASC,kBAAkB,QAAQ,cAAc;AACjD,OAAOC,UAAU,MAAM,wBAAqB;;AAE5C;AACA;AACA;AACA;AACA,SAASC,qBAAqBA,CAACC,MAAc,EAAgB;EAC3D,MAAMC,KAAK,GAAGL,MAAM,CAACM,IAAI,CAACF,MAAM,EAAE,QAAQ,CAAC;EAC3C,MAAMG,IAAI,GAAG,IAAIC,QAAQ,CAACH,KAAK,CAACI,MAAM,EAAEJ,KAAK,CAACK,UAAU,EAAEL,KAAK,CAACM,UAAU,CAAC;EAC3E,MAAMC,GAAG,GAAGP,KAAK,CAACM,UAAU,GAAG,CAAC;EAChC,MAAME,GAAG,GAAG,IAAIC,YAAY,CAACF,GAAG,CAAC;EACjC,KAAK,IAAIG,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5BF,GAAG,CAACE,CAAC,CAAC,GAAGR,IAAI,CAACS,QAAQ,CAACD,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,KAAK;EAC7C;EACA,OAAOF,GAAG;AACZ;AAiBA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASI,mBAAmBA,CACjCC,OAA8B,EACT;EACrB,MAAMC,UAAU,GAAGD,OAAO,EAAEC,UAAU,IAAI,KAAK;EAC/C,MAAMC,YAAY,GAAGF,OAAO,EAAEE,YAAY,IAAI,CAAC;EAC/C,MAAMC,gBAAgB,GAAGH,OAAO,EAAEG,gBAAgB,IAAI,CAAC;EAEvD,OAAO;IACLC,KAAK,EAAEA,CAAA,KACLpB,UAAU,CAACqB,kBAAkB,CAAC;MAC5BJ,UAAU;MACVC,YAAY;MACZC;IACF,CAAC,CAAC;IAEJG,IAAI,EAAEA,CAAA,KAAMtB,UAAU,CAACuB,iBAAiB,CAAC,CAAC;IAE1CC,MAAM,EAAGC,QAA6D,IAAK;MACzE,MAAMC,GAAG,GAAG3B,kBAAkB,CAAC4B,WAAW,CACxC,mBAAmB,EAClBC,KAAkD,IAAK;QACtD,MAAM1B,MAAM,GAAG0B,KAAK,EAAEC,SAAS,IAAI,EAAE;QACrC,MAAMC,EAAE,GAAGF,KAAK,EAAEX,UAAU,IAAIA,UAAU;QAC1C,IAAIf,MAAM,EAAE;UACV,MAAM6B,OAAO,GAAG9B,qBAAqB,CAACC,MAAM,CAAC;UAC7CuB,QAAQ,CAACM,OAAO,EAAED,EAAE,CAAC;QACvB;MACF,CACF,CAAC;MACD,OAAO,MAAMJ,GAAG,CAACM,MAAM,CAAC,CAAC;IAC3B,CAAC;IAEDC,OAAO,EAAGR,QAAmC,IAAK;MAChD,MAAMC,GAAG,GAAG3B,kBAAkB,CAAC4B,WAAW,CACxC,oBAAoB,EACnBC,KAA2B,IAAK;QAC/BH,QAAQ,CAACG,KAAK,EAAEM,OAAO,IAAI,eAAe,CAAC;MAC7C,CACF,CAAC;MACD,OAAO,MAAMR,GAAG,CAACM,MAAM,CAAC,CAAC;IAC3B;EACF,CAAC;AACH;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASG,oBAAoBA,CAClCC,SAAiB,EACjBC,UAAkB,EAClBC,MAAc,EACdC,kBAA2B,EACZ;EACf,OAAOvC,UAAU,CAACmC,oBAAoB,CACpCC,SAAS,EACTC,UAAU,EACVC,MAAM,EACNC,kBAAkB,IAAI,CACxB,CAAC;AACH;;AAEA;AACA;AACA;AACA;AACA,OAAO,SAASC,oBAAoBA,CAClCJ,SAAiB,EACjBC,UAAkB,EACH;EACf,OAAOrC,UAAU,CAACwC,oBAAoB,CAACJ,SAAS,EAAEC,UAAU,CAAC;AAC/D","ignoreList":[]}
+{"version":3,"names":["Buffer","DeviceEventEmitter","SherpaOnnx","base64PcmToFloatArray","base64","bytes","from","view","DataView","buffer","byteOffset","byteLength","len","out","Float32Array","i","getInt16","createPcmLiveStream","options","sampleRate","channelCount","bufferSizeFrames","start","startPcmLiveStream","stop","stopPcmLiveStream","onData","callback","sub","addListener","event","base64Pcm","sr","samples","remove","onError","message","convertAudioToFormat","inputPath","outputPath","format","outputSampleRateHz","convertAudioToWav16k","decodeAudioFileToFloatSamples","targetSampleRateHz"],"sourceRoot":"../../../src","sources":["audio/index.ts"],"mappings":";;AAAA,SAASA,MAAM,QAAQ,QAAQ;AAC/B,SAASC,kBAAkB,QAAQ,cAAc;AACjD,OAAOC,UAAU,MAAM,wBAAqB;;AAE5C;AACA;AACA;AACA;AACA,SAASC,qBAAqBA,CAACC,MAAc,EAAgB;EAC3D,MAAMC,KAAK,GAAGL,MAAM,CAACM,IAAI,CAACF,MAAM,EAAE,QAAQ,CAAC;EAC3C,MAAMG,IAAI,GAAG,IAAIC,QAAQ,CAACH,KAAK,CAACI,MAAM,EAAEJ,KAAK,CAACK,UAAU,EAAEL,KAAK,CAACM,UAAU,CAAC;EAC3E,MAAMC,GAAG,GAAGP,KAAK,CAACM,UAAU,GAAG,CAAC;EAChC,MAAME,GAAG,GAAG,IAAIC,YAAY,CAACF,GAAG,CAAC;EACjC,KAAK,IAAIG,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5BF,GAAG,CAACE,CAAC,CAAC,GAAGR,IAAI,CAACS,QAAQ,CAACD,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,KAAK;EAC7C;EACA,OAAOF,GAAG;AACZ;AAiBA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASI,mBAAmBA,CACjCC,OAA8B,EACT;EACrB,MAAMC,UAAU,GAAGD,OAAO,EAAEC,UAAU,IAAI,KAAK;EAC/C,MAAMC,YAAY,GAAGF,OAAO,EAAEE,YAAY,IAAI,CAAC;EAC/C,MAAMC,gBAAgB,GAAGH,OAAO,EAAEG,gBAAgB,IAAI,CAAC;EAEvD,OAAO;IACLC,KAAK,EAAEA,CAAA,KACLpB,UAAU,CAACqB,kBAAkB,CAAC;MAC5BJ,UAAU;MACVC,YAAY;MACZC;IACF,CAAC,CAAC;IAEJG,IAAI,EAAEA,CAAA,KAAMtB,UAAU,CAACuB,iBAAiB,CAAC,CAAC;IAE1CC,MAAM,EAAGC,QAA6D,IAAK;MACzE,MAAMC,GAAG,GAAG3B,kBAAkB,CAAC4B,WAAW,CACxC,mBAAmB,EAClBC,KAAkD,IAAK;QACtD,MAAM1B,MAAM,GAAG0B,KAAK,EAAEC,SAAS,IAAI,EAAE;QACrC,MAAMC,EAAE,GAAGF,KAAK,EAAEX,UAAU,IAAIA,UAAU;QAC1C,IAAIf,MAAM,EAAE;UACV,MAAM6B,OAAO,GAAG9B,qBAAqB,CAACC,MAAM,CAAC;UAC7CuB,QAAQ,CAACM,OAAO,EAAED,EAAE,CAAC;QACvB;MACF,CACF,CAAC;MACD,OAAO,MAAMJ,GAAG,CAACM,MAAM,CAAC,CAAC;IAC3B,CAAC;IAEDC,OAAO,EAAGR,QAAmC,IAAK;MAChD,MAAMC,GAAG,GAAG3B,kBAAkB,CAAC4B,WAAW,CACxC,oBAAoB,EACnBC,KAA2B,IAAK;QAC/BH,QAAQ,CAACG,KAAK,EAAEM,OAAO,IAAI,eAAe,CAAC;MAC7C,CACF,CAAC;MACD,OAAO,MAAMR,GAAG,CAACM,MAAM,CAAC,CAAC;IAC3B;EACF,CAAC;AACH;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASG,oBAAoBA,CAClCC,SAAiB,EACjBC,UAAkB,EAClBC,MAAc,EACdC,kBAA2B,EACZ;EACf,OAAOvC,UAAU,CAACmC,oBAAoB,CACpCC,SAAS,EACTC,UAAU,EACVC,MAAM,EACNC,kBAAkB,IAAI,CACxB,CAAC;AACH;;AAEA;AACA;AACA;AACA;AACA,OAAO,SAASC,oBAAoBA,CAClCJ,SAAiB,EACjBC,UAAkB,EACH;EACf,OAAOrC,UAAU,CAACwC,oBAAoB,CAACJ,SAAS,EAAEC,UAAU,CAAC;AAC/D;AAOA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASI,6BAA6BA,CAC3CL,SAAiB,EACjBM,kBAA2B,EACQ;EACnC,OAAO1C,UAAU,CAACyC,6BAA6B,CAC7CL,SAAS,EACTM,kBAAkB,IAAI,CACxB,CAAC;AACH","ignoreList":[]}