react-native-sherpa-onnx 0.4.1 → 0.4.2

package/README.md

@@ -121,6 +121,7 @@ Full step-by-step: [Download manager – Setup (iOS & Android)](docs/download-ma
 | Live capture API | ✅ **Supported** | [PCM live stream](./docs/pcm-live-stream.md) | Native microphone capture with resampling for live transcription (use with streaming STT). |
 | Text-to-Speech | ✅ **Supported** | [TTS](./docs/tts.md) | Multiple model types (VITS, Matcha, Kokoro, etc.). See [Supported Model Types](#supported-model-types). |
 | Streaming Text-to-Speech | ✅ **Supported** | [Streaming TTS](./docs/tts-streaming.md) | Incremental speech generation for low time-to-first-byte and playback while generating. |
+| TTS Alignment / Timestamps | ✅ **Supported** | [TTS Alignment](./docs/tts-alignment.md) | Full implementation: **`fast`** (native chunk-based, estimated timing) and **`accurate`** (wav2vec2 CTC forced alignment, `timingMode: 'aligned'`). Optional alignment ONNX via `react-native-sherpa-onnx/alignment` (see [TTS Alignment](./docs/tts-alignment.md)). Standalone API: `generateSubtitlesFromAudio()`. |
 | Execution providers (CPU, NNAPI, XNNPACK, Core ML, QNN) | ✅ **Supported** | [Execution providers](./docs/execution-providers.md) | CPU default; optional accelerators per platform. |
 | Play Asset Delivery (PAD) | ✅ **Supported** | [Model setup](./docs/model-setup.md) | Android only. Archives: [Extraction API](./docs/extraction.md). |
 | Automatic Model type detection | ✅ **Supported** | [Model detection](./docs/model-setup.md#model-detection) | `detectSttModel()` and `detectTtsModel()` for a path. |
@@ -204,6 +205,7 @@ APIs, batch vs online processing, and initialization are covered in [Speech Enha
 - [Streaming (Online) Speech-to-Text](./docs/stt-streaming.md) – Real-time recognition, partial results, endpoint detection
 - [PCM Live Stream](./docs/pcm-live-stream.md) – Native microphone capture with resampling for live transcription (use with streaming STT)
 - [Text-to-Speech (TTS)](./docs/tts.md) – Offline and streaming generation
+- [TTS Alignment / Timestamps](./docs/tts-alignment.md) – `fast` and `accurate` modes, sentence/word/character granularity, alignment model download, `generateSpeechWithTimestamps()` and `generateSubtitlesFromAudio()`
 - [Streaming Text-to-Speech](./docs/tts-streaming.md) – Incremental TTS (createStreamingTTS)
 - [Execution provider support (QNN, NNAPI, XNNPACK, Core ML)](./docs/execution-providers.md) – Checking and using acceleration backends
 - [Voice Activity Detection (VAD)](./docs/vad.md)
@@ -237,6 +239,7 @@ The example app included in this repository demonstrates audio-to-text transcrip
 - **Offline** audio file transcription
 - **Online (streaming) STT** – live transcription from the microphone with partial results
 - **Streaming TTS** – incremental speech generation and playback
+- **Generate timestamp** – subtitle/timestamp generation from audio (`fast` / `accurate` with optional alignment model download)
 - Test audio files for different languages
 
 **Getting started:**

package/android/src/main/assets/model_licenses/alignment-models-license-status.csv

@@ -0,0 +1,5 @@
+asset_name,license_type,commercial_use,confidence,detection_source,license_file
+wav2vec2-base-960h-fp16.tar.bz2,apache-2.0,yes,high,manual,https://huggingface.co/datasets/choosealicense/licenses/resolve/main/markdown/apache-2.0.md
+wav2vec2-base-960h-int8.tar.bz2,apache-2.0,yes,high,manual,https://huggingface.co/datasets/choosealicense/licenses/resolve/main/markdown/apache-2.0.md
+wav2vec2-base-960h-q4f16.tar.bz2,apache-2.0,yes,high,manual,https://huggingface.co/datasets/choosealicense/licenses/resolve/main/markdown/apache-2.0.md
+wav2vec2-base-960h.tar.bz2,apache-2.0,yes,high,manual,https://huggingface.co/datasets/choosealicense/licenses/resolve/main/markdown/apache-2.0.md

package/android/src/main/cpp/CMakeLists.txt

@@ -84,13 +84,16 @@ set(SOURCES
     jni/model_detect/sherpa-onnx-model-detect-stt.cpp
     jni/model_detect/sherpa-onnx-model-detect-tts.cpp
     jni/model_detect/sherpa-onnx-model-detect-enhancement.cpp
+    jni/model_detect/sherpa-onnx-model-detect-alignment.cpp
     jni/model_detect/sherpa-onnx-validate-stt.cpp
     jni/model_detect/sherpa-onnx-validate-tts.cpp
     jni/model_detect/sherpa-onnx-validate-enhancement.cpp
+    jni/model_detect/sherpa-onnx-validate-alignment.cpp
     jni/model_detect/sherpa-onnx-detect-jni-common.cpp
     jni/model_detect/sherpa-onnx-stt-wrapper.cpp
     jni/model_detect/sherpa-onnx-tts-wrapper.cpp
     jni/model_detect/sherpa-onnx-enhancement-wrapper.cpp
+    jni/model_detect/sherpa-onnx-alignment-wrapper.cpp
     jni/audio/sherpa-onnx-audio-convert-jni.cpp
     crypto/sha256.cpp
 )

package/android/src/main/cpp/jni/model_detect/sherpa-onnx-alignment-wrapper.cpp

@@ -0,0 +1,66 @@
+#include "sherpa-onnx-alignment-wrapper.h"
+
+#include "sherpa-onnx-detect-jni-common.h"
+
+namespace sherpaonnx {
+namespace {
+
+const char* AlignmentModelKindToString(AlignmentModelKind k) {
+  switch (k) {
+    case AlignmentModelKind::kWav2Vec2:
+      return "wav2vec2";
+    default:
+      return "unknown";
+  }
+}
+
+} // namespace
+
+jobject AlignmentDetectResultToJava(
+    JNIEnv* env,
+    const AlignmentDetectResult& result
+) {
+  jclass mapClass = env->FindClass("java/util/HashMap");
+  if (!mapClass) return nullptr;
+  jmethodID mapInit = env->GetMethodID(mapClass, "<init>", "()V");
+  jmethodID mapPut =
+      env->GetMethodID(mapClass, "put",
+                       "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;");
+  if (!mapInit || !mapPut) {
+    env->DeleteLocalRef(mapClass);
+    return nullptr;
+  }
+  jobject map = env->NewObject(mapClass, mapInit);
+  env->DeleteLocalRef(mapClass);
+  if (!map) return nullptr;
+
+  PutBoolean(env, map, mapPut, "success", result.ok);
+  PutString(env, map, mapPut, "error", result.error);
+  PutString(env, map, mapPut, "modelType",
+            AlignmentModelKindToString(result.selectedKind));
+
+  jobject detectedList = BuildDetectedModelsList(env, result.detectedModels);
+  if (detectedList) {
+    jstring keyDetected = env->NewStringUTF("detectedModels");
+    env->CallObjectMethod(map, mapPut, keyDetected, detectedList);
+    env->DeleteLocalRef(keyDetected);
+    env->DeleteLocalRef(detectedList);
+  }
+
+  jclass hashMapClass = env->FindClass("java/util/HashMap");
+  if (hashMapClass) {
+    jobject pathsMap = env->NewObject(hashMapClass, mapInit);
+    env->DeleteLocalRef(hashMapClass);
+    if (pathsMap) {
+      PutString(env, pathsMap, mapPut, "model", result.paths.model);
+      jstring keyPaths = env->NewStringUTF("paths");
+      env->CallObjectMethod(map, mapPut, keyPaths, pathsMap);
+      env->DeleteLocalRef(keyPaths);
+      env->DeleteLocalRef(pathsMap);
+    }
+  }
+
+  return map;
+}
+
+} // namespace sherpaonnx

package/android/src/main/cpp/jni/model_detect/sherpa-onnx-alignment-wrapper.h

@@ -0,0 +1,17 @@
+#ifndef SHERPA_ONNX_ALIGNMENT_WRAPPER_H
+#define SHERPA_ONNX_ALIGNMENT_WRAPPER_H
+
+#include <jni.h>
+
+#include "sherpa-onnx-model-detect.h"
+
+namespace sherpaonnx {
+
+jobject AlignmentDetectResultToJava(
+    JNIEnv* env,
+    const AlignmentDetectResult& result
+);
+
+} // namespace sherpaonnx
+
+#endif // SHERPA_ONNX_ALIGNMENT_WRAPPER_H

package/android/src/main/cpp/jni/model_detect/sherpa-onnx-model-detect-alignment.cpp

@@ -0,0 +1,108 @@
+#include "sherpa-onnx-model-detect.h"
+#include "sherpa-onnx-model-detect-helper.h"
+#include "sherpa-onnx-validate-alignment.h"
+
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace {
+
+using namespace sherpaonnx::model_detect;
+
+sherpaonnx::AlignmentModelKind ParseAlignmentModelType(const std::string& modelType) {
+    if (modelType == "wav2vec2") return sherpaonnx::AlignmentModelKind::kWav2Vec2;
+    return sherpaonnx::AlignmentModelKind::kUnknown;
+}
+
+sherpaonnx::AlignmentDetectResult DetectAlignmentModelFromFiles(
+    const std::vector<FileEntry>& files,
+    const std::string& modelDir,
+    const std::string& modelType
+) {
+    sherpaonnx::AlignmentDetectResult result;
+
+    const std::string wav2vec2Model =
+        FindOnnxByAnyToken(files, {"wav2vec2", "model"}, std::nullopt);
+
+    if (!wav2vec2Model.empty()) {
+        result.detectedModels.push_back({"wav2vec2", modelDir});
+    }
+
+    sherpaonnx::AlignmentModelKind selected = sherpaonnx::AlignmentModelKind::kUnknown;
+    if (modelType == "auto" || modelType.empty()) {
+        if (!wav2vec2Model.empty()) {
+            selected = sherpaonnx::AlignmentModelKind::kWav2Vec2;
+        }
+    } else {
+        selected = ParseAlignmentModelType(modelType);
+        if (selected == sherpaonnx::AlignmentModelKind::kUnknown) {
+            result.error = "Alignment: unknown model type: " + modelType;
+            return result;
+        }
+    }
+
+    switch (selected) {
+        case sherpaonnx::AlignmentModelKind::kWav2Vec2:
+            result.paths.model = wav2vec2Model;
+            break;
+        default:
+            result.error = "Alignment: no compatible model type detected in " +
+                           modelDir;
+            return result;
+    }
+
+    auto validation =
+        sherpaonnx::ValidateAlignmentPaths(selected, result.paths, modelDir);
+    if (!validation.ok) {
+        result.error = validation.error;
+        return result;
+    }
+
+    result.selectedKind = selected;
+    result.ok = true;
+    return result;
+}
+
+} // namespace
+
+namespace sherpaonnx {
+
+using namespace model_detect;
+
+AlignmentDetectResult DetectAlignmentModel(
+    const std::string& modelDir,
+    const std::string& modelType
+) {
+    AlignmentDetectResult result;
+
+    if (modelDir.empty()) {
+        result.error = "Alignment: model directory is empty";
+        return result;
+    }
+    if (!FileExists(modelDir) || !IsDirectory(modelDir)) {
+        result.error =
+            "Alignment: model directory does not exist or is not a directory: " +
+            modelDir;
+        return result;
+    }
+
+    const std::vector<model_detect::FileEntry> files = ListFilesRecursive(modelDir, 4);
+    return DetectAlignmentModelFromFiles(files, modelDir, modelType);
+}
+
+// Test-only: used by host-side model_detect_test; not used in production.
+AlignmentDetectResult DetectAlignmentModelFromFileList(
+    const std::vector<model_detect::FileEntry>& files,
+    const std::string& modelDir,
+    const std::string& modelType
+) {
+    AlignmentDetectResult result;
+    if (modelDir.empty()) {
+        result.error = "Alignment: model directory is empty";
+        return result;
+    }
+    return DetectAlignmentModelFromFiles(files, modelDir, modelType);
+}
+
+} // namespace sherpaonnx

package/android/src/main/cpp/jni/model_detect/sherpa-onnx-model-detect.h

@@ -49,6 +49,11 @@ enum class EnhancementModelKind {
     kDpdfNet
 };
 
+enum class AlignmentModelKind {
+    kUnknown,
+    kWav2Vec2
+};
+
 struct SttModelPaths {
     std::string encoder;
     std::string decoder;
@@ -184,6 +189,10 @@ struct EnhancementModelPaths {
     std::string model;
 };
 
+struct AlignmentModelPaths {
+    std::string model;
+};
+
 struct SttDetectResult {
     bool ok = false;
     std::string error;
@@ -213,6 +222,14 @@ struct EnhancementDetectResult {
     EnhancementModelPaths paths;
 };
 
+struct AlignmentDetectResult {
+    bool ok = false;
+    std::string error;
+    std::vector<DetectedModel> detectedModels;
+    AlignmentModelKind selectedKind = AlignmentModelKind::kUnknown;
+    AlignmentModelPaths paths;
+};
+
 SttDetectResult DetectSttModel(
     const std::string& modelDir,
     const std::optional<bool>& preferInt8,
@@ -251,6 +268,11 @@ EnhancementDetectResult DetectEnhancementModel(
     const std::string& modelType
 );
 
+AlignmentDetectResult DetectAlignmentModel(
+    const std::string& modelDir,
+    const std::string& modelType
+);
+
 /** Test-only: Like DetectEnhancementModel but takes a pre-built file list; no filesystem access.
  *  Only used by the host-side C++ test suite (test/cpp/model_detect_test.cpp). */
 EnhancementDetectResult DetectEnhancementModelFromFileList(
@@ -259,6 +281,14 @@ EnhancementDetectResult DetectEnhancementModelFromFileList(
     const std::string& modelType = "auto"
 );
 
+/** Test-only: Like DetectAlignmentModel but takes a pre-built file list; no filesystem access.
+ *  Only used by the host-side C++ test suite (test/cpp/model_detect_test.cpp). */
+AlignmentDetectResult DetectAlignmentModelFromFileList(
+    const std::vector<model_detect::FileEntry>& files,
+    const std::string& modelDir,
+    const std::string& modelType = "auto"
+);
+
 } // namespace sherpaonnx
 
 #endif // SHERPA_ONNX_MODEL_DETECT_H

package/android/src/main/cpp/jni/model_detect/sherpa-onnx-validate-alignment.cpp

@@ -0,0 +1,66 @@
+#include "sherpa-onnx-validate-alignment.h"
+
+#include <cstddef>
+
+namespace sherpaonnx {
+namespace {
+
+static const AlignmentFieldRequirement kWav2Vec2Reqs[] = {
+    {"model", &AlignmentModelPaths::model, true},
+};
+
+static const AlignmentFieldRequirement* GetRequirements(
+    AlignmentModelKind kind,
+    size_t& count
+) {
+    switch (kind) {
+        case AlignmentModelKind::kWav2Vec2:
+            count = std::size(kWav2Vec2Reqs);
+            return kWav2Vec2Reqs;
+        default:
+            count = 0;
+            return nullptr;
+    }
+}
+
+static const char* AlignmentKindToName(AlignmentModelKind kind) {
+    switch (kind) {
+        case AlignmentModelKind::kWav2Vec2:
+            return "wav2vec2";
+        default:
+            return "unknown";
+    }
+}
+
+} // namespace
+
+AlignmentValidationResult ValidateAlignmentPaths(
+    AlignmentModelKind kind,
+    const AlignmentModelPaths& paths,
+    const std::string& modelDir
+) {
+    AlignmentValidationResult result;
+    size_t count = 0;
+    const auto* reqs = GetRequirements(kind, count);
+    if (!reqs) return result;
+
+    for (size_t i = 0; i < count; ++i) {
+        if (reqs[i].required && (paths.*(reqs[i].field)).empty()) {
+            result.missingRequired.push_back(reqs[i].fieldName);
+        }
+    }
+
+    if (!result.missingRequired.empty()) {
+        result.ok = false;
+        result.error = std::string("Alignment ") + AlignmentKindToName(kind) +
+                       ": missing required files in " + modelDir + ": ";
+        for (size_t i = 0; i < result.missingRequired.size(); ++i) {
+            if (i > 0) result.error += ", ";
+            result.error += result.missingRequired[i];
+        }
+    }
+
+    return result;
+}
+
+} // namespace sherpaonnx

package/android/src/main/cpp/jni/model_detect/sherpa-onnx-validate-alignment.h

@@ -0,0 +1,30 @@
+#ifndef SHERPA_ONNX_VALIDATE_ALIGNMENT_H
+#define SHERPA_ONNX_VALIDATE_ALIGNMENT_H
+
+#include "sherpa-onnx-model-detect.h"
+#include <string>
+#include <vector>
+
+namespace sherpaonnx {
+
+struct AlignmentFieldRequirement {
+    const char* fieldName;
+    std::string AlignmentModelPaths::* field;
+    bool required;
+};
+
+struct AlignmentValidationResult {
+    bool ok = true;
+    std::vector<std::string> missingRequired;
+    std::string error;
+};
+
+AlignmentValidationResult ValidateAlignmentPaths(
+    AlignmentModelKind kind,
+    const AlignmentModelPaths& paths,
+    const std::string& modelDir
+);
+
+} // namespace sherpaonnx
+
+#endif // SHERPA_ONNX_VALIDATE_ALIGNMENT_H

package/android/src/main/cpp/jni/module/sherpa-onnx-module-jni.cpp

@@ -21,6 +21,7 @@
 #include "sherpa-onnx-stt-wrapper.h"
 #include "sherpa-onnx-tts-wrapper.h"
 #include "sherpa-onnx-enhancement-wrapper.h"
+#include "sherpa-onnx-alignment-wrapper.h"
 
 extern "C" {
 
@@ -208,4 +209,24 @@ Java_com_sherpaonnx_SherpaOnnxModule_nativeDetectEnhancementModel(
   return sherpaonnx::EnhancementDetectResultToJava(env, result);
 }
 
+// Detect alignment model in directory. Returns HashMap with success, error, detectedModels, modelType, paths.
+JNIEXPORT jobject JNICALL
+Java_com_sherpaonnx_SherpaOnnxModule_nativeDetectAlignmentModel(
+    JNIEnv* env,
+    jobject /* this */,
+    jstring j_model_dir,
+    jstring j_model_type) {
+  const char* model_dir_c = env->GetStringUTFChars(j_model_dir, nullptr);
+  const char* model_type_c =
+      j_model_type ? env->GetStringUTFChars(j_model_type, nullptr) : nullptr;
+  std::string model_dir(model_dir_c ? model_dir_c : "");
+  std::string model_type(model_type_c ? model_type_c : "auto");
+  env->ReleaseStringUTFChars(j_model_dir, model_dir_c);
+  if (model_type_c) env->ReleaseStringUTFChars(j_model_type, model_type_c);
+
+  sherpaonnx::AlignmentDetectResult result =
+      sherpaonnx::DetectAlignmentModel(model_dir, model_type);
+  return sherpaonnx::AlignmentDetectResultToJava(env, result);
+}
+
 }  // extern "C"