react-native-sherpa-onnx 0.3.6 → 0.3.7

package/ios/SherpaOnnx+OnlineSTT.mm

@@ -70,6 +70,7 @@ static sherpaonnx::OnlineSttWrapper* getOnlineSttInstanceForStream(NSString* str
     NSString *provider = options.provider();
     NSString *ruleFsts = options.ruleFsts();
     NSString *ruleFars = options.ruleFars();
+    auto dither = options.dither();
     auto blankPenalty = options.blankPenalty();
     auto debug = options.debug();
     auto rule1MustContainNonSilence = options.rule1MustContainNonSilence();
@@ -102,6 +103,7 @@ static sherpaonnx::OnlineSttWrapper* getOnlineSttInstanceForStream(NSString* str
             provider != nil ? [provider UTF8String] : "cpu",
             ruleFsts != nil ? [ruleFsts UTF8String] : "",
             ruleFars != nil ? [ruleFars UTF8String] : "",
+            dither.has_value() ? (float)dither.value() : 0.f,
             blankPenalty.has_value() ? (float)blankPenalty.value() : 0.f,
             debug.has_value() && debug.value(),
             rule1MustContainNonSilence.has_value() && rule1MustContainNonSilence.value(),

package/ios/SherpaOnnx+PcmLiveStream.mm

@@ -143,37 +143,10 @@ static void pcmLiveStopQueue(void) {
 
 @implementation SherpaOnnx (PcmLiveStream)
 
-- (void)startPcmLiveStream:(id __unsafe_unretained)optionsArg
+#if __has_include(<SherpaOnnxSpec/SherpaOnnxSpec.h>)
+- (void)startPcmLiveStream:(JS::NativeSherpaOnnx::SpecStartPcmLiveStreamOptions &)options
                    resolve:(RCTPromiseResolveBlock)resolve
                     reject:(RCTPromiseRejectBlock)reject
-{
-  int targetRate = 16000;
-  UInt32 bufferSizeFrames = 0;
-
-  // Parse optionsArg coming from JS (fallback / non-codegen path).
-  if ([optionsArg isKindOfClass:[NSDictionary class]]) {
-    NSDictionary *dict = (NSDictionary *)optionsArg;
-
-    id sampleRateValue = dict[@"sampleRate"];
-    if ([sampleRateValue respondsToSelector:@selector(intValue)]) {
-      int v = (int)[sampleRateValue intValue];
-      if (v > 0) targetRate = v;
-    }
-
-    id bufferSizeValue = dict[@"bufferSizeFrames"];
-    if ([bufferSizeValue respondsToSelector:@selector(doubleValue)]) {
-      double v = [bufferSizeValue doubleValue];
-      if (v > 0) bufferSizeFrames = (UInt32)v;
-    }
-  }
-
-  [self _startPcmLiveStreamWithTargetRate:targetRate bufferSizeFrames:bufferSizeFrames resolve:resolve reject:reject];
-}
-
-#if __has_include(<SherpaOnnxSpec/SherpaOnnxSpec.h>)
-- (void)startPcmLiveStreamWithOptions:(JS::NativeSherpaOnnx::SpecStartPcmLiveStreamOptions &)options
-                             resolve:(RCTPromiseResolveBlock)resolve
-                              reject:(RCTPromiseRejectBlock)reject
 {
   int targetRate = 16000;
   if (options.sampleRate()) {

package/ios/SherpaOnnx+TTS.mm

@@ -17,6 +17,7 @@
 #include <condition_variable>
 #include <memory>
 #include <mutex>
+#include <optional>
 #include <sstream>
 #include <string>
 #include <unordered_map>
@@ -55,6 +56,7 @@ static NSString *ttsModelKindToNSString(sherpaonnx::TtsModelKind kind) {
         case K::kMatcha: return @"matcha";
         case K::kKokoro: return @"kokoro";
         case K::kKitten: return @"kitten";
+        case K::kPocket: return @"pocket";
         case K::kZipvoice: return @"zipvoice";
         default: return @"unknown";
     }
@@ -73,8 +75,60 @@ std::vector<std::string> SplitTtsTokens(const std::string &text) {
     }
     return tokens;
 }
+
+/** When options omit numSteps, matches Android SherpaOnnxTtsHelper / upstream GenerationConfig default. */
+constexpr int32_t kDefaultVoiceCloneNumSteps = 5;
+
+/** Non-null optional when referenceAudio is non-empty array and referenceSampleRate > 0. */
+static std::optional<sherpaonnx::VoiceCloneOptions> VoiceCloneOptionsFromNSDictionary(NSDictionary *options, int32_t defaultNumSteps) {
+    if (options == nil) return std::nullopt;
+    NSArray *refArr = options[@"referenceAudio"];
+    if (![refArr isKindOfClass:[NSArray class]] || [refArr count] == 0) return std::nullopt;
+    NSNumber *srNum = options[@"referenceSampleRate"];
+    if (srNum == nil || [srNum doubleValue] <= 0) return std::nullopt;
+
+    sherpaonnx::VoiceCloneOptions vo;
+    vo.reference_sample_rate = static_cast<int32_t>([srNum doubleValue]);
+    vo.reference_audio.reserve([refArr count]);
+    for (id elem in refArr) {
+        float v = 0.f;
+        if ([elem isKindOfClass:[NSNumber class]]) {
+            v = static_cast<float>([(NSNumber *)elem doubleValue]);
+        }
+        vo.reference_audio.push_back(v);
+    }
+    NSString *rt = options[@"referenceText"];
+    if (rt != nil && [rt length] > 0) {
+        vo.reference_text = std::string([rt UTF8String]);
+    }
+    if (options[@"numSteps"] != nil) {
+        vo.num_steps = static_cast<int32_t>([options[@"numSteps"] doubleValue]);
+    } else {
+        vo.num_steps = defaultNumSteps;
+    }
+    if (options[@"silenceScale"] != nil) {
+        vo.silence_scale = static_cast<float>([options[@"silenceScale"] doubleValue]);
+    }
+    id extra = options[@"extra"];
+    if ([extra isKindOfClass:[NSDictionary class]]) {
+        NSDictionary *ex = (NSDictionary *)extra;
+        for (NSString *k in ex) {
+            id v = ex[k];
+            if ([v isKindOfClass:[NSString class]]) {
+                vo.extra[std::string([k UTF8String])] = std::string([(NSString *)v UTF8String]);
+            }
+        }
+    }
+    return vo;
+}
+
+static bool NSDictionaryHasValidReferenceAudio(NSDictionary *options) {
+    auto o = VoiceCloneOptionsFromNSDictionary(options, 1);
+    return o.has_value() && !o->reference_audio.empty() && o->reference_sample_rate > 0;
 }
 
+} // namespace
+
 @implementation SherpaOnnx (TTS)
 
 - (void)initializeTts:(NSString *)instanceId
@@ -195,8 +249,10 @@ std::vector<std::string> SplitTtsTokens(const std::string &text) {
 
             resolve(resultDict);
         } else {
-            NSString *errorMsg = @"Failed to initialize TTS";
-            RCTLogError(@"%@", errorMsg);
+            NSString *errorMsg = result.error.empty()
+                ? @"Failed to initialize TTS"
+                : [NSString stringWithUTF8String:result.error.c_str()];
+            RCTLogError(@"TTS init failed: %@", errorMsg);
             reject(@"TTS_INIT_ERROR", errorMsg, nil);
         }
     } @catch (NSException *exception) {
@@ -408,10 +464,37 @@ std::vector<std::string> SplitTtsTokens(const std::string &text) {
     @try {
         std::string textStr = [text UTF8String];
 
+        using Kind = sherpaonnx::TtsModelKind;
+        Kind kind = wrapper->getModelKind();
+        bool hasRef = NSDictionaryHasValidReferenceAudio(options);
+
+        if (hasRef && kind != Kind::kZipvoice && kind != Kind::kPocket) {
+            reject(@"TTS_GENERATE_ERROR", @"Reference audio is only supported for Zipvoice and Pocket TTS.", nil);
+            return;
+        }
+        if (kind == Kind::kPocket && !hasRef) {
+            reject(@"TTS_GENERATE_ERROR", @"Pocket TTS requires reference audio for voice cloning. Pass referenceAudio and referenceSampleRate (> 0) in options.", nil);
+            return;
+        }
+        if (hasRef && kind == Kind::kZipvoice) {
+            NSString *rt = options[@"referenceText"];
+            NSString *trimmed = rt != nil ? [rt stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] : @"";
+            if ([trimmed length] == 0) {
+                reject(@"TTS_GENERATE_ERROR", @"Zipvoice voice cloning requires non-empty referenceText (transcript of reference audio).", nil);
+                return;
+            }
+        }
+
+        std::optional<sherpaonnx::VoiceCloneOptions> cloneOpt;
+        if (hasRef) {
+            cloneOpt = VoiceCloneOptionsFromNSDictionary(options, kDefaultVoiceCloneNumSteps);
+        }
+
         auto result = wrapper->generate(
             textStr,
             static_cast<int32_t>(sid),
-            static_cast<float>(speed)
+            static_cast<float>(speed),
+            cloneOpt
         );
 
         if (result.samples.empty() || result.sampleRate == 0) {
@@ -469,10 +552,37 @@ std::vector<std::string> SplitTtsTokens(const std::string &text) {
     @try {
         std::string textStr = [text UTF8String];
 
+        using Kind = sherpaonnx::TtsModelKind;
+        Kind kind = wrapper->getModelKind();
+        bool hasRef = NSDictionaryHasValidReferenceAudio(options);
+
+        if (hasRef && kind != Kind::kZipvoice && kind != Kind::kPocket) {
+            reject(@"TTS_GENERATE_ERROR", @"Reference audio is only supported for Zipvoice and Pocket TTS.", nil);
+            return;
+        }
+        if (kind == Kind::kPocket && !hasRef) {
+            reject(@"TTS_GENERATE_ERROR", @"Pocket TTS requires reference audio for voice cloning. Pass referenceAudio and referenceSampleRate (> 0) in options.", nil);
+            return;
+        }
+        if (hasRef && kind == Kind::kZipvoice) {
+            NSString *rt = options[@"referenceText"];
+            NSString *trimmed = rt != nil ? [rt stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] : @"";
+            if ([trimmed length] == 0) {
+                reject(@"TTS_GENERATE_ERROR", @"Zipvoice voice cloning requires non-empty referenceText (transcript of reference audio).", nil);
+                return;
+            }
+        }
+
+        std::optional<sherpaonnx::VoiceCloneOptions> cloneOpt;
+        if (hasRef) {
+            cloneOpt = VoiceCloneOptionsFromNSDictionary(options, kDefaultVoiceCloneNumSteps);
+        }
+
         auto result = wrapper->generate(
             textStr,
             static_cast<int32_t>(sid),
-            static_cast<float>(speed)
+            static_cast<float>(speed),
+            cloneOpt
         );
 
         if (result.samples.empty() || result.sampleRate == 0) {
@@ -487,22 +597,32 @@ std::vector<std::string> SplitTtsTokens(const std::string &text) {
             [samplesArray addObject:@(sample)];
         }
 
-        std::vector<std::string> tokens = SplitTtsTokens(textStr);
         NSMutableArray *subtitlesArray = [NSMutableArray array];
-        if (!tokens.empty()) {
-            double totalSeconds = static_cast<double>(result.samples.size()) /
-                                  static_cast<double>(result.sampleRate);
-            double perToken = totalSeconds / static_cast<double>(tokens.size());
-
-            for (size_t i = 0; i < tokens.size(); ++i) {
-                double start = perToken * static_cast<double>(i);
-                double end = perToken * static_cast<double>(i + 1);
-                NSDictionary *item = @{
-                    @"text": [NSString stringWithUTF8String:tokens[i].c_str()],
-                    @"start": @(start),
-                    @"end": @(end)
-                };
-                [subtitlesArray addObject:item];
+        if (hasRef && !result.samples.empty() && result.sampleRate > 0) {
+            double durationSec = static_cast<double>(result.samples.size()) / static_cast<double>(result.sampleRate);
+            NSDictionary *subtitleMap = @{
+                @"text": text,
+                @"start": @0.0,
+                @"end": @(durationSec)
+            };
+            [subtitlesArray addObject:subtitleMap];
+        } else {
+            std::vector<std::string> tokens = SplitTtsTokens(textStr);
+            if (!tokens.empty()) {
+                double totalSeconds = static_cast<double>(result.samples.size()) /
+                                      static_cast<double>(result.sampleRate);
+                double perToken = totalSeconds / static_cast<double>(tokens.size());
+
+                for (size_t i = 0; i < tokens.size(); ++i) {
+                    double start = perToken * static_cast<double>(i);
+                    double end = perToken * static_cast<double>(i + 1);
+                    NSDictionary *item = @{
+                        @"text": [NSString stringWithUTF8String:tokens[i].c_str()],
+                        @"start": @(start),
+                        @"end": @(end)
+                    };
+                    [subtitlesArray addObject:item];
+                }
             }
         }
 
@@ -556,6 +676,43 @@ std::vector<std::string> SplitTtsTokens(const std::string &text) {
         instRef->streamRunning.store(true);
     }
 
+    using Kind = sherpaonnx::TtsModelKind;
+    Kind streamKind = instRef->wrapper->getModelKind();
+    bool streamHasRef = NSDictionaryHasValidReferenceAudio(options);
+
+    if (streamKind == Kind::kPocket && !streamHasRef) {
+        std::lock_guard<std::mutex> lock(g_tts_mutex);
+        auto it2 = g_tts_instances.find([instanceId UTF8String]);
+        if (it2 != g_tts_instances.end()) {
+            it2->second->streamRunning.store(false);
+        }
+        reject(@"TTS_STREAM_ERROR", @"Pocket TTS requires reference audio for voice cloning. Pass referenceAudio and referenceSampleRate (> 0) in options.", nil);
+        return;
+    }
+    if (streamHasRef && streamKind == Kind::kZipvoice) {
+        std::lock_guard<std::mutex> lock(g_tts_mutex);
+        auto it2 = g_tts_instances.find([instanceId UTF8String]);
+        if (it2 != g_tts_instances.end()) {
+            it2->second->streamRunning.store(false);
+        }
+        reject(@"TTS_STREAM_ERROR", @"Streaming with reference audio not supported for Zipvoice", nil);
+        return;
+    }
+    if (streamHasRef && streamKind != Kind::kPocket) {
+        std::lock_guard<std::mutex> lock(g_tts_mutex);
+        auto it2 = g_tts_instances.find([instanceId UTF8String]);
+        if (it2 != g_tts_instances.end()) {
+            it2->second->streamRunning.store(false);
+        }
+        reject(@"TTS_STREAM_ERROR", @"Reference audio streaming is only supported for Pocket TTS.", nil);
+        return;
+    }
+
+    std::optional<sherpaonnx::VoiceCloneOptions> streamCloneOpt;
+    if (streamHasRef) {
+        streamCloneOpt = VoiceCloneOptionsFromNSDictionary(options, kDefaultVoiceCloneNumSteps);
+    }
+
     std::string textStr = [text UTF8String];
     int32_t sampleRate = instRef->wrapper->getSampleRate();
     NSString *instanceIdCopy = [instanceId copy];
@@ -595,7 +752,8 @@ std::vector<std::string> SplitTtsTokens(const std::string &text) {
                     });
 
                     return instRef->streamCancelled.load() ? 0 : 1;
-                }
+                },
+                streamCloneOpt
             );
         } @catch (NSException *exception) {
             NSString *errorMsg = [NSString stringWithFormat:@"TTS streaming failed: %@", exception.reason];

package/ios/SherpaOnnx.mm

@@ -189,6 +189,14 @@
     resolve(nil);
 }
 
+- (void)cancelExtractBySourcePath:(NSString *)sourcePath
+                          resolve:(RCTPromiseResolveBlock)resolve
+                           reject:(RCTPromiseRejectBlock)reject
+{
+    [SherpaOnnxArchiveHelper cancelExtractForPath:sourcePath];
+    resolve(nil);
+}
+
 - (void)computeFileSha256:(NSString *)filePath
              resolve:(RCTPromiseResolveBlock)resolve
              reject:(RCTPromiseRejectBlock)reject
@@ -271,6 +279,26 @@
     resolve(nil);
 }
 
+- (void)decodeAudioFileToFloatSamples:(NSString *)inputPath
+                   targetSampleRateHz:(NSNumber *)targetSampleRateHz
+                              resolve:(RCTPromiseResolveBlock)resolve
+                               reject:(RCTPromiseRejectBlock)reject
+{
+    NSArray<NSNumber *> *samples = nil;
+    int sr = 0;
+    NSError *error = nil;
+    int rate = targetSampleRateHz != nil ? targetSampleRateHz.intValue : 0;
+    if (![SherpaOnnxAudioConvert decodeAudioFileToFloatSamples:inputPath
+                                            targetSampleRateHz:rate
+                                                    outSamples:&samples
+                                                 outSampleRate:&sr
+                                                         error:&error]) {
+        reject(@"DECODE_ERROR", error ? error.localizedDescription : @"Failed to decode audio", error);
+        return;
+    }
+    resolve(@{ @"samples": samples ?: @[], @"sampleRate": @(sr) });
+}
+
 - (void)getAvailableProviders:(RCTPromiseResolveBlock)resolve
                       reject:(RCTPromiseRejectBlock)reject
 {
@@ -286,4 +314,30 @@
     }
 }
 
+- (void)readAssetFileAsUtf8:(NSString *)assetPath
+                    resolve:(RCTPromiseResolveBlock)resolve
+                     reject:(RCTPromiseRejectBlock)reject
+{
+    // Validate assetPath to prevent path traversal: reject any path that
+    // contains "..", is absolute, or uses backslashes.
+    if ([assetPath containsString:@".."] ||
+        [assetPath hasPrefix:@"/"] ||
+        [assetPath hasPrefix:@"\\"] ||
+        [assetPath containsString:@"\\"]) {
+        reject(@"ASSET_READ_ERROR",
+               [NSString stringWithFormat:@"Invalid asset path: %@", assetPath],
+               nil);
+        return;
+    }
+    NSString *resourcePath = [[NSBundle mainBundle] resourcePath];
+    NSString *fullPath = [resourcePath stringByAppendingPathComponent:assetPath];
+    NSError *error = nil;
+    NSString *content = [NSString stringWithContentsOfFile:fullPath encoding:NSUTF8StringEncoding error:&error];
+    if (error) {
+        reject(@"ASSET_READ_ERROR", [NSString stringWithFormat:@"Failed to read asset %@: %@", assetPath, error.localizedDescription], error);
+    } else {
+        resolve(content);
+    }
+}
+
 @end

package/ios/SherpaOnnxAudioConvert.h

@@ -23,6 +23,16 @@ NS_ASSUME_NONNULL_BEGIN
           outputSampleRateHz:(int)outputSampleRateHz
                        error:(NSError **)error;
 
+/**
+ * Decode audio to mono float samples (approx. [-1, 1]) and sample rate.
+ * targetSampleRateHz <= 0 keeps the decoded stream rate.
+ */
++ (BOOL)decodeAudioFileToFloatSamples:(NSString *)inputPath
+                   targetSampleRateHz:(int)targetSampleRateHz
+                           outSamples:(NSArray<NSNumber *> **)outSamples
+                        outSampleRate:(int *)outSampleRate
+                                error:(NSError **)error;
+
 @end
 
 NS_ASSUME_NONNULL_END