react-native-sherpa-onnx 0.4.0 → 0.4.2

package/android/src/main/java/com/sherpaonnx/SherpaOnnxTextSegmenter.kt

@@ -0,0 +1,330 @@
+package com.sherpaonnx
+
+import kotlin.math.floor
+import kotlin.math.max
+
+internal data class SubtitleTimingItem(
+  val text: String,
+  val start: Double,
+  val end: Double
+)
+
+internal object SherpaOnnxTextSegmenter {
+  private val sentenceTerminators = setOf('.', '!', '?', ';', '。', '！', '？', '；')
+  private val trailingClosers = setOf('"', '\'', ')', ']', '}', '>', '”', '’', '」', '』', '】', '）')
+  private val commonAbbreviations = setOf(
+    "mr",
+    "mrs",
+    "ms",
+    "dr",
+    "prof",
+    "sr",
+    "jr",
+    "st",
+    "vs",
+    "etc",
+    "e.g",
+    "i.e"
+  )
+
+  fun splitIntoSentences(text: String): List<String> {
+    val normalized = text.trim()
+    if (normalized.isEmpty()) return emptyList()
+
+    val out = mutableListOf<String>()
+    var start = 0
+    var i = 0
+
+    while (i < normalized.length) {
+      val current = normalized[i]
+      if (!isSentenceTerminator(current)) {
+        i += 1
+        continue
+      }
+
+      if (current == '.' && !shouldSplitOnPeriod(normalized, i)) {
+        i += 1
+        continue
+      }
+
+      val end = sentenceBoundaryEnd(normalized, i)
+      val next = normalized.getOrNull(end)
+      if (next != null && !next.isWhitespace()) {
+        i += 1
+        continue
+      }
+
+      val sentence = normalized.substring(start, end).trim()
+      if (sentence.isNotEmpty()) {
+        out += sentence
+      }
+
+      start = end
+      while (start < normalized.length && normalized[start].isWhitespace()) {
+        start += 1
+      }
+      i = start
+    }
+
+    val tail = if (start < normalized.length) normalized.substring(start).trim() else ""
+    if (tail.isNotEmpty()) {
+      out += tail
+    }
+
+    return if (out.isNotEmpty()) out else listOf(normalized)
+  }
+
+  fun splitIntoWords(text: String): List<String> {
+    val normalized = text.trim()
+    if (normalized.isEmpty()) return emptyList()
+
+    val out = mutableListOf<String>()
+    val current = StringBuilder()
+
+    fun flushCurrent() {
+      val token = current.toString().trim()
+      if (token.isNotEmpty()) {
+        out += token
+      }
+      current.clear()
+    }
+
+    for (char in normalized) {
+      when {
+        char.isWhitespace() -> flushCurrent()
+        isCjkChar(char) -> {
+          flushCurrent()
+          out += char.toString()
+        }
+        isWordDelimiter(char) -> flushCurrent()
+        else -> current.append(char)
+      }
+    }
+
+    flushCurrent()
+    return if (out.isNotEmpty()) out else listOf(normalized)
+  }
+
+  fun buildSubtitlesFromChunks(
+    segments: List<String>,
+    chunkSampleCounts: List<Int>,
+    sampleRate: Int
+  ): List<SubtitleTimingItem> {
+    if (sampleRate <= 0) return emptyList()
+
+    val cleanedSegments = sanitizeSegments(segments)
+    if (cleanedSegments.isEmpty()) return emptyList()
+
+    val alignedCounts = alignChunkCountsToSegments(cleanedSegments, chunkSampleCounts)
+
+    val subtitles = mutableListOf<SubtitleTimingItem>()
+    var offsetSamples = 0
+
+    for (index in cleanedSegments.indices) {
+      val samples = alignedCounts.getOrElse(index) { 0 }.coerceAtLeast(0)
+      if (samples == 0 && offsetSamples == 0) {
+        continue
+      }
+
+      val startSec = offsetSamples.toDouble() / sampleRate.toDouble()
+      offsetSamples += samples
+      val endSec = offsetSamples.toDouble() / sampleRate.toDouble()
+
+      subtitles += SubtitleTimingItem(
+        text = cleanedSegments[index],
+        start = startSec,
+        end = endSec
+      )
+    }
+
+    return subtitles
+  }
+
+  fun buildWordSubtitlesFromSentenceChunks(
+    sentences: List<String>,
+    sentenceChunkSampleCounts: List<Int>,
+    sampleRate: Int
+  ): List<SubtitleTimingItem> {
+    val cleanedSentences = sanitizeSegments(sentences)
+    if (cleanedSentences.isEmpty()) return emptyList()
+
+    val alignedSentenceCounts = alignChunkCountsToSegments(
+      cleanedSentences,
+      sentenceChunkSampleCounts
+    )
+
+    val wordSegments = mutableListOf<String>()
+    val wordChunkCounts = mutableListOf<Int>()
+
+    for (index in cleanedSentences.indices) {
+      val sentence = cleanedSentences[index]
+      val sentenceSamples = alignedSentenceCounts.getOrElse(index) { 0 }.coerceAtLeast(0)
+      val words = splitIntoWords(sentence)
+      if (words.isEmpty()) continue
+
+      val distributed = distributeSamplesByTextWeight(sentenceSamples, words)
+      for (wordIndex in words.indices) {
+        wordSegments += words[wordIndex]
+        wordChunkCounts += distributed.getOrElse(wordIndex) { 0 }
+      }
+    }
+
+    return buildSubtitlesFromChunks(wordSegments, wordChunkCounts, sampleRate)
+  }
+
+  private fun sanitizeSegments(segments: List<String>): List<String> {
+    return segments
+      .map { it.trim() }
+      .filter { it.isNotEmpty() }
+  }
+
+  private fun alignChunkCountsToSegments(
+    segments: List<String>,
+    chunkSampleCounts: List<Int>
+  ): List<Int> {
+    if (segments.isEmpty()) return emptyList()
+
+    val counts = chunkSampleCounts.map { max(0, it) }
+    if (counts.size == segments.size) {
+      return counts
+    }
+
+    if (counts.size > segments.size) {
+      val merged = counts.take(segments.size).toMutableList()
+      val extra = counts.drop(segments.size).sum()
+      if (merged.isNotEmpty()) {
+        val lastIndex = merged.lastIndex
+        merged[lastIndex] = merged[lastIndex] + extra
+      }
+      return merged
+    }
+
+    return distributeSamplesByTextWeight(counts.sum(), segments)
+  }
+
+  private fun distributeSamplesByTextWeight(totalSamples: Int, segments: List<String>): List<Int> {
+    if (segments.isEmpty()) return emptyList()
+
+    val safeTotal = totalSamples.coerceAtLeast(0)
+    if (safeTotal == 0) {
+      return List(segments.size) { 0 }
+    }
+
+    val weights = segments.map { max(1, it.length) }
+    val weightSum = weights.sum().coerceAtLeast(1)
+
+    val base = MutableList(segments.size) { 0 }
+    val fractions = mutableListOf<Pair<Int, Double>>()
+
+    for (index in segments.indices) {
+      val exact = (safeTotal.toDouble() * weights[index].toDouble()) / weightSum.toDouble()
+      val floorValue = floor(exact).toInt()
+      base[index] = floorValue
+      fractions += index to (exact - floorValue.toDouble())
+    }
+
+    var assigned = base.sum()
+    var remaining = safeTotal - assigned
+
+    if (remaining > 0) {
+      val order = fractions.sortedByDescending { it.second }
+      var ptr = 0
+      while (remaining > 0 && order.isNotEmpty()) {
+        val target = order[ptr % order.size].first
+        base[target] = base[target] + 1
+        assigned += 1
+        remaining = safeTotal - assigned
+        ptr += 1
+      }
+    }
+
+    return base
+  }
+
+  private fun isSentenceTerminator(char: Char): Boolean {
+    return sentenceTerminators.contains(char)
+  }
+
+  private fun shouldSplitOnPeriod(text: String, periodIndex: Int): Boolean {
+    val prev = text.getOrNull(periodIndex - 1)
+    val next = text.getOrNull(periodIndex + 1)
+
+    if (prev != null && next != null && prev.isDigit() && next.isDigit()) {
+      return false
+    }
+
+    val tokenRaw = extractTokenBeforePeriod(text, periodIndex)
+    val tokenLower = tokenRaw.lowercase()
+    if (commonAbbreviations.contains(tokenLower)) {
+      return false
+    }
+
+    // Likely initial, e.g. "A. Smith" — use original case; tokenLower[0] is never uppercase.
+    if (tokenRaw.length == 1 && tokenRaw[0].isUpperCase()) {
+      return false
+    }
+
+    return true
+  }
+
+  private fun extractTokenBeforePeriod(text: String, periodIndex: Int): String {
+    var i = periodIndex - 1
+    while (i >= 0 && text[i].isWhitespace()) {
+      i -= 1
+    }
+
+    val end = i
+    while (i >= 0) {
+      val c = text[i]
+      if (c.isLetter() || c == '.') {
+        i -= 1
+        continue
+      }
+      break
+    }
+
+    if (end < i + 1) return ""
+
+    var token = text.substring(i + 1, end + 1)
+    while (token.endsWith('.')) {
+      token = token.dropLast(1)
+    }
+    return token
+  }
+
+  private fun sentenceBoundaryEnd(text: String, startIndex: Int): Int {
+    var end = startIndex + 1
+    while (end < text.length && isSentenceTerminator(text[end])) {
+      end += 1
+    }
+    while (end < text.length && trailingClosers.contains(text[end])) {
+      end += 1
+    }
+    return end
+  }
+
+  private fun isCjkChar(char: Char): Boolean {
+    val block = Character.UnicodeBlock.of(char)
+    return block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS ||
+      block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A ||
+      block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B ||
+      block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C ||
+      block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D ||
+      block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E ||
+      block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F ||
+      block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_G ||
+      block == Character.UnicodeBlock.HIRAGANA ||
+      block == Character.UnicodeBlock.KATAKANA ||
+      block == Character.UnicodeBlock.HANGUL_SYLLABLES
+  }
+
+  private fun isWordDelimiter(char: Char): Boolean {
+    return when (char) {
+      '.', ',', '!', '?', ';', ':', '(', ')', '[', ']', '{', '}',
+      '"', '\'', '`', '~', '<', '>', '/', '\\', '|', '@', '#', '$',
+      '%', '^', '&', '*', '+', '=', '…', '，', '。', '！', '？', '；',
+      '：', '、' -> true
+      else -> false
+    }
+  }
+}

package/android/src/main/java/com/sherpaonnx/SherpaOnnxTtsHelper.kt

@@ -111,6 +111,46 @@ internal class SherpaOnnxTtsHelper(
     }
   }
 
+  /**
+   * libsherpa-onnx-jni looks up `invoke([F)Ljava/lang/Integer` (see sherpa-onnx `offline-tts.cc` CallCallback).
+   * Kotlin `Function1<*, Int>` compiles to `invoke([F)I`, so GetMethodID fails and JNI aborts.
+   * Using [java.lang.Integer] as the type parameter yields the boxed JVM signature the JNI expects.
+   * The cast is only for the Kotlin API (`generateWithCallback` still declares `Function1<FloatArray, Int>`).
+   */
+  /** Box for JNI: must be real [java.lang.Integer], not Kotlin [Int] (primitive `invoke([F)I` breaks sherpa JNI). */
+  @Suppress("DEPRECATION")
+  private fun boxForTtsJni(n: Int): java.lang.Integer = java.lang.Integer(n)
+
+  @Suppress("UNCHECKED_CAST")
+  private fun ttsChunkCallbackForJni(
+    sentenceChunkSizes: MutableList<Int>
+  ): kotlin.Function1<FloatArray, Int> {
+    val boxed =
+      object : kotlin.jvm.functions.Function1<FloatArray, java.lang.Integer> {
+        override fun invoke(chunk: FloatArray): java.lang.Integer {
+          sentenceChunkSizes.add(chunk.size)
+          return boxForTtsJni(chunk.size)
+        }
+      }
+    return boxed as kotlin.Function1<FloatArray, Int>
+  }
+
+  @Suppress("UNCHECKED_CAST")
+  private fun ttsStreamChunkCallbackForJni(
+    cancelled: AtomicBoolean,
+    onChunk: (FloatArray) -> Unit
+  ): kotlin.Function1<FloatArray, Int> {
+    val boxed =
+      object : kotlin.jvm.functions.Function1<FloatArray, java.lang.Integer> {
+        override fun invoke(chunk: FloatArray): java.lang.Integer {
+          if (cancelled.get()) return boxForTtsJni(0)
+          onChunk(chunk)
+          return boxForTtsJni(chunk.size)
+        }
+      }
+    return boxed as kotlin.Function1<FloatArray, Int>
+  }
+
   /** Single-thread executor for TTS init so the RN bridge thread is not blocked (avoids Inspector/dev WebSocket races in debug builds). */
   private val ttsInitExecutor = Executors.newSingleThreadExecutor()
 
@@ -451,9 +491,66 @@ internal class SherpaOnnxTtsHelper(
         promise.reject("TTS_GENERATE_ERROR", "TTS not initialized")
         return
       }
+
+      val subtitleMode = getSubtitleMode(options)
+      val subtitleGranularity = getSubtitleGranularity(options)
+      if (isCharacterGranularityRequested(options) && subtitleMode != "accurate") {
+        Log.e(
+          "SherpaOnnxTts",
+          "TTS_SUBTITLE_ERROR: Character granularity is only supported when subtitleMode is 'accurate'"
+        )
+        promise.reject(
+          "TTS_SUBTITLE_ERROR",
+          "Character granularity is only supported when subtitleMode is 'accurate'."
+        )
+        return
+      }
+
       val sid = getSid(options)
       val speed = getSpeed(options)
+      val sentenceChunkSizes = mutableListOf<Int>()
       val audio = when {
+        subtitleMode == "off" -> {
+          when {
+            hasReferenceAudio(options) && (inst.isZipvoice || inst.isPocket) -> {
+              if (inst.isZipvoice) {
+                val promptText = options!!.getString("referenceText")?.trim().orEmpty()
+                if (promptText.isEmpty()) {
+                  Log.e("SherpaOnnxTts", "TTS_GENERATE_ERROR: Zipvoice voice cloning requires non-empty referenceText")
+                  promise.reject(
+                    "TTS_GENERATE_ERROR",
+                    "Zipvoice voice cloning requires non-empty referenceText (transcript of reference audio)."
+                  )
+                  return
+                }
+              }
+              val config = parseGenerationConfig(options) ?: GenerationConfig(speed = speed, sid = sid)
+              inst.tts!!.generateWithConfig(text, config)
+            }
+            hasReferenceAudio(options) -> {
+              Log.e("SherpaOnnxTts", "TTS_GENERATE_ERROR: Reference audio is not supported for this TTS model type")
+              promise.reject(
+                "TTS_GENERATE_ERROR",
+                "Reference audio is only supported for Zipvoice and Pocket TTS."
+              )
+              return
+            }
+            inst.isPocket -> {
+              Log.e("SherpaOnnxTts", "TTS_GENERATE_ERROR: Pocket TTS requires reference audio for voice cloning")
+              promise.reject(
+                "TTS_GENERATE_ERROR",
+                "Pocket TTS requires reference audio for voice cloning. Pass referenceAudio and referenceSampleRate (> 0) in options."
+              )
+              return
+            }
+            else -> dispatchGenerate(inst, text, sid, speed)
+              ?: run {
+                Log.e("SherpaOnnxTts", "TTS_GENERATE_ERROR: TTS not initialized")
+                promise.reject("TTS_GENERATE_ERROR", "TTS not initialized")
+                return
+              }
+          }
+        }
         hasReferenceAudio(options) && (inst.isZipvoice || inst.isPocket) -> {
           if (inst.isZipvoice) {
             val promptText = options!!.getString("referenceText")?.trim().orEmpty()
@@ -467,7 +564,11 @@ internal class SherpaOnnxTtsHelper(
             }
           }
           val config = parseGenerationConfig(options) ?: GenerationConfig(speed = speed, sid = sid)
-          inst.tts!!.generateWithConfig(text, config)
+          inst.tts!!.generateWithConfigAndCallback(
+            text,
+            config,
+            ttsChunkCallbackForJni(sentenceChunkSizes)
+          )
         }
         hasReferenceAudio(options) -> {
           Log.e("SherpaOnnxTts", "TTS_GENERATE_ERROR: Reference audio is not supported for this TTS model type")
@@ -485,13 +586,21 @@ internal class SherpaOnnxTtsHelper(
           )
           return
         }
-        else -> dispatchGenerate(inst, text, sid, speed)
-          ?: run {
+        else -> {
+          val tts = inst.tts
+          if (tts == null) {
             Log.e("SherpaOnnxTts", "TTS_GENERATE_ERROR: TTS not initialized")
             promise.reject("TTS_GENERATE_ERROR", "TTS not initialized")
             return
           }
+          tts.generateWithCallback(text, sid, speed, ttsChunkCallbackForJni(sentenceChunkSizes))
+        }
+      }
+
+      if (subtitleMode != "off" && sentenceChunkSizes.isEmpty() && audio.samples.isNotEmpty()) {
+        sentenceChunkSizes.add(audio.samples.size)
       }
+
       val map = Arguments.createMap()
       val samplesArray = Arguments.createArray()
       for (sample in audio.samples) {
@@ -499,17 +608,29 @@ internal class SherpaOnnxTtsHelper(
       }
       map.putArray("samples", samplesArray)
       map.putInt("sampleRate", audio.sampleRate)
-      val subtitlesArray = Arguments.createArray()
-      if (audio.samples.isNotEmpty() && audio.sampleRate > 0) {
-        val durationSec = audio.samples.size.toDouble() / audio.sampleRate
-        val subtitleMap = Arguments.createMap()
-        subtitleMap.putString("text", text)
-        subtitleMap.putDouble("start", 0.0)
-        subtitleMap.putDouble("end", durationSec)
-        subtitlesArray.pushMap(subtitleMap)
+
+      val subtitleItems = if (subtitleMode == "off") {
+        emptyList()
+      } else {
+        val sentenceSegments = SherpaOnnxTextSegmenter.splitIntoSentences(text)
+        if (subtitleGranularity == "word") {
+          SherpaOnnxTextSegmenter.buildWordSubtitlesFromSentenceChunks(
+            sentenceSegments,
+            sentenceChunkSizes,
+            audio.sampleRate
+          )
+        } else {
+          SherpaOnnxTextSegmenter.buildSubtitlesFromChunks(
+            sentenceSegments,
+            sentenceChunkSizes,
+            audio.sampleRate
+          )
+        }
       }
-      map.putArray("subtitles", subtitlesArray)
-      map.putBoolean("estimated", true)
+
+      map.putArray("subtitles", toSubtitleWritableArray(subtitleItems))
+      val timingMode = if (subtitleMode == "off") "off" else "estimated"
+      map.putString("timingMode", timingMode)
       promise.resolve(map)
     } catch (e: Exception) {
       Log.e("SherpaOnnxTts", "TTS_GENERATE_ERROR: ${e.message ?: "Failed to generate speech"}", e)
@@ -564,18 +685,23 @@ internal class SherpaOnnxTtsHelper(
         when {
           hasReferenceAudio(options) && inst.isPocket -> {
             val config = parseGenerationConfig(options) ?: GenerationConfig(speed = speed, sid = sid)
-            inst.tts!!.generateWithConfigAndCallback(text, config) { chunk ->
-              if (inst.ttsStreamCancelled.get()) return@generateWithConfigAndCallback 0
-              emitChunk(instanceId, requestId, chunk, sampleRate, 0f, false)
-              chunk.size
-            }
+            inst.tts!!.generateWithConfigAndCallback(
+              text,
+              config,
+              ttsStreamChunkCallbackForJni(inst.ttsStreamCancelled) { chunk ->
+                emitChunk(instanceId, requestId, chunk, sampleRate, 0f, false)
+              }
+            )
           }
           else -> {
-            inst.tts!!.generateWithCallback(text, sid, speed) { chunk ->
-              if (inst.ttsStreamCancelled.get()) return@generateWithCallback 0
-              emitChunk(instanceId, requestId, chunk, sampleRate, 0f, false)
-              chunk.size
-            }
+            inst.tts!!.generateWithCallback(
+              text,
+              sid,
+              speed,
+              ttsStreamChunkCallbackForJni(inst.ttsStreamCancelled) { chunk ->
+                emitChunk(instanceId, requestId, chunk, sampleRate, 0f, false)
+              }
+            )
           }
         }
         if (!inst.ttsStreamCancelled.get()) {
@@ -915,6 +1041,37 @@ internal class SherpaOnnxTtsHelper(
   private fun getSpeed(options: ReadableMap?): Float =
     if (options != null && options.hasKey("speed")) options.getDouble("speed").toFloat() else 1.0f
 
+  private fun getSubtitleMode(options: ReadableMap?): String {
+    val raw = options?.getString("subtitleMode")?.trim()?.lowercase()
+    return when (raw) {
+      "off", "fast", "accurate" -> raw
+      else -> "fast"
+    }
+  }
+
+  private fun getSubtitleGranularity(options: ReadableMap?): String {
+    val raw = options?.getString("subtitleGranularity")?.trim()?.lowercase()
+    return when (raw) {
+      "word", "sentence" -> raw
+      else -> "sentence"
+    }
+  }
+
+  private fun isCharacterGranularityRequested(options: ReadableMap?): Boolean {
+    val raw = options?.getString("subtitleGranularity")?.trim()?.lowercase()
+    return raw == "character"
+  }
+
+  private fun toSubtitleWritableArray(items: List<SubtitleTimingItem>) = Arguments.createArray().apply {
+    for (item in items) {
+      val subtitleMap = Arguments.createMap()
+      subtitleMap.putString("text", item.text)
+      subtitleMap.putDouble("start", item.start)
+      subtitleMap.putDouble("end", item.end)
+      pushMap(subtitleMap)
+    }
+  }
+
   /** Build Kotlin GenerationConfig from ReadableMap. Returns null only when options is null; otherwise returns a config with sid, speed, silenceScale, numSteps, and any reference/extra fields from options. */
   private fun parseGenerationConfig(options: ReadableMap?): GenerationConfig? {
     if (options == null) return null

package/ios/Resources/model_licenses/alignment-models-license-status.csv

@@ -0,0 +1,5 @@
+asset_name,license_type,commercial_use,confidence,detection_source,license_file
+wav2vec2-base-960h-fp16.tar.bz2,apache-2.0,yes,high,manual,https://huggingface.co/datasets/choosealicense/licenses/resolve/main/markdown/apache-2.0.md
+wav2vec2-base-960h-int8.tar.bz2,apache-2.0,yes,high,manual,https://huggingface.co/datasets/choosealicense/licenses/resolve/main/markdown/apache-2.0.md
+wav2vec2-base-960h-q4f16.tar.bz2,apache-2.0,yes,high,manual,https://huggingface.co/datasets/choosealicense/licenses/resolve/main/markdown/apache-2.0.md
+wav2vec2-base-960h.tar.bz2,apache-2.0,yes,high,manual,https://huggingface.co/datasets/choosealicense/licenses/resolve/main/markdown/apache-2.0.md