whisper.rn 0.2.4 → 0.3.0-rc.0

package/README.md

@@ -20,6 +20,8 @@ npm install whisper.rn
 
 Then re-run `npx pod-install` again for iOS.
 
+For Expo, you will need to prebuild the project before using it. See [Expo guide](https://docs.expo.io/guides/using-libraries/#using-a-library-in-a-expo-project) for more details.
+
 ## Add Microphone Permissions (Optional)
 
 If you want to use realtime transcribe, you need to add the microphone permission to your app.
@@ -31,6 +33,8 @@ Add these lines to ```ios/[YOU_APP_NAME]/info.plist```
 <string>This app requires microphone access in order to transcribe speech</string>
 ```
 
+For tvOS, please note that the microphone is not supported.
+
 ### Android
 Add the following line to ```android/app/src/main/AndroidManifest.xml```
 ```xml
@@ -42,11 +46,12 @@ Add the following line to ```android/app/src/main/AndroidManifest.xml```
 ```js
 import { initWhisper } from 'whisper.rn'
 
-const filePath = 'file://.../ggml.base.en.bin'
-const sampleFilePath = 'file://.../sample.wav'
-
-const whisperContext = await initWhisper({ filePath })
+const whisperContext = await initWhisper({
+  filePath: 'file://.../ggml-base.en.bin',
+  isBundleAsset: false, // Set to true if you want to load the model from bundle resources, the filePath will be like `ggml-base.en.bin`
+})
 
+const sampleFilePath = 'file://.../sample.wav'
 const options = { language: 'en' }
 const { stop, promise } = whisperContext.transcribe(sampleFilePath, options)
 
@@ -74,7 +79,31 @@ subscribe(evt => {
 
 In Android, you may need to request the microphone permission by [`PermissionAndroid`](https://reactnative.dev/docs/permissionsandroid).
 
-The documentation is not ready yet, please see the comments of [index](./src/index.tsx) file for more details at the moment.
+The documentation is not ready yet, please see the comments of [index](./src/index.ts) file for more details at the moment.
+
+## Core ML support
+
+__*Platform: iOS 15.0+, tvOS 15.0+*__
+
+To use Core ML on iOS, you will need to have the Core ML model files.
+
+The `.mlmodelc` model files is load depend on the ggml model file path. For example, if your ggml model path is `ggml-base.en.bin`, the Core ML model path will be `ggml-base.en-encoder.mlmodelc`. Please note that the ggml model is still needed as decoder or encoder fallback.
+
+Currently there is no official way to get the Core ML models by URL, you will need to convert the ggml model to Core ML model folder by yourself. Please see [Core ML Support](https://github.com/ggerganov/whisper.cpp#core-ml-support) of whisper.cpp for more details.
+
+During the `.mlmodelc` is a directory, you will need to download 5 files:
+
+```json5
+[
+  'model.mil',
+  'metadata.json',
+  'coremldata.bin',
+  'weights/weights.bin',
+  'analysis/coremldata.bin',
+]
+```
+
+Or just add them to your app's bundle resourcesas, like the example app does, but this would increase the app size significantly.
 
 ## Run with example
 

package/android/src/main/java/com/rnwhisper/RNWhisperModule.java

@@ -40,14 +40,19 @@ public class RNWhisperModule extends ReactContextBaseJavaModule implements Lifec
   private HashMap<Integer, WhisperContext> contexts = new HashMap<>();
 
   @ReactMethod
-  public void initContext(final String modelPath, final Promise promise) {
+  public void initContext(final String modelPath, final boolean isBundleAsset, final Promise promise) {
     new AsyncTask<Void, Void, Integer>() {
       private Exception exception;
 
       @Override
       protected Integer doInBackground(Void... voids) {
         try {
-          long context = WhisperContext.initContext(modelPath);
+          long context;
+          if (isBundleAsset) {
+            context = WhisperContext.initContextWithAsset(reactContext.getAssets(), modelPath);
+          } else {
+            context = WhisperContext.initContext(modelPath);
+          }
           if (context == 0) {
             throw new Exception("Failed to initialize context");
           }

package/android/src/main/java/com/rnwhisper/WhisperContext.java

@@ -186,8 +186,6 @@ public class WhisperContext {
           if (fullHandler != null) {
             fullHandler.join(); // Wait for full transcribe to finish
           }
-          // Cleanup
-          resetRealtimeTranscribe();
           recorder.stop();
         } catch (Exception e) {
           e.printStackTrace();
@@ -237,12 +235,11 @@ public class WhisperContext {
     }
 
     nSamplesOfIndex = sliceNSamples.get(transcribeSliceIndex);
-    if (
-      isStoppedByAction ||
+    boolean isStopped = isStoppedByAction ||
       !isCapturing &&
       nSamplesTranscribing == nSamplesOfIndex &&
-      sliceIndex == transcribeSliceIndex
-    ) {
+      sliceIndex == transcribeSliceIndex;
+    if (isStopped) {
       payload.putBoolean("isCapturing", false);
       payload.putBoolean("isStoppedByAction", isStoppedByAction);
       emitTranscribeEvent("@RNWhisper_onRealtimeTranscribeEnd", payload);
@@ -266,6 +263,9 @@ public class WhisperContext {
     if (!isCapturing && nSamplesTranscribing != nSamplesOfIndex) {
       // If no more capturing, continue transcribing until all slices are transcribed
       fullTranscribeSamples(options, true);
+    } else if (isStopped) {
+      // No next, cleanup
+      resetRealtimeTranscribe();
     }
     isTranscribing = false;
   }
@@ -469,6 +469,7 @@ public class WhisperContext {
   }
 
   protected static native long initContext(String modelPath);
+  protected static native long initContextWithAsset(AssetManager assetManager, String modelPath);
   protected static native int fullTranscribe(
     int job_id,
     long context,

package/android/src/main/jni/whisper/jni.cpp

@@ -20,6 +20,39 @@ static inline int min(int a, int b) {
     return (a < b) ? a : b;
 }
 
+static size_t asset_read(void *ctx, void *output, size_t read_size) {
+    return AAsset_read((AAsset *) ctx, output, read_size);
+}
+
+static bool asset_is_eof(void *ctx) {
+    return AAsset_getRemainingLength64((AAsset *) ctx) <= 0;
+}
+
+static void asset_close(void *ctx) {
+    AAsset_close((AAsset *) ctx);
+}
+
+static struct whisper_context *whisper_init_from_asset(
+    JNIEnv *env,
+    jobject assetManager,
+    const char *asset_path
+) {
+    LOGI("Loading model from asset '%s'\n", asset_path);
+    AAssetManager *asset_manager = AAssetManager_fromJava(env, assetManager);
+    AAsset *asset = AAssetManager_open(asset_manager, asset_path, AASSET_MODE_STREAMING);
+    if (!asset) {
+        LOGW("Failed to open '%s'\n", asset_path);
+        return NULL;
+    }
+    whisper_model_loader loader = {
+        .context = asset,
+        .read = &asset_read,
+        .eof = &asset_is_eof,
+        .close = &asset_close
+    };
+    return whisper_init(&loader);
+}
+
 extern "C" {
 
 JNIEXPORT jlong JNICALL
@@ -33,6 +66,22 @@ Java_com_rnwhisper_WhisperContext_initContext(
     return reinterpret_cast<jlong>(context);
 }
 
+JNIEXPORT jlong JNICALL
+Java_com_rnwhisper_WhisperContext_initContextWithAsset(
+    JNIEnv *env,
+    jobject thiz,
+    jobject asset_manager,
+    jstring model_path_str
+) {
+    UNUSED(thiz);
+    struct whisper_context *context = nullptr;
+    const char *model_path_chars = env->GetStringUTFChars(model_path_str, nullptr);
+    context = whisper_init_from_asset(env, asset_manager, model_path_chars);
+    env->ReleaseStringUTFChars(model_path_str, model_path_chars);
+    return reinterpret_cast<jlong>(context);
+}
+
+
 JNIEXPORT jint JNICALL
 Java_com_rnwhisper_WhisperContext_fullTranscribe(
     JNIEnv *env,
@@ -61,7 +110,9 @@ Java_com_rnwhisper_WhisperContext_fullTranscribe(
     struct whisper_context *context = reinterpret_cast<struct whisper_context *>(context_ptr);
     jfloat *audio_data_arr = env->GetFloatArrayElements(audio_data, nullptr);
 
-    int max_threads = min(4, std::thread::hardware_concurrency());
+    int max_threads = std::thread::hardware_concurrency();
+    // Use 2 threads by default on 4-core devices, 4 threads on more cores
+    int default_n_threads = max_threads == 4 ? 2 : min(4, max_threads);
 
     LOGI("About to create params");
 
@@ -79,7 +130,7 @@ Java_com_rnwhisper_WhisperContext_fullTranscribe(
     params.translate = translate;
     const char *language_chars = env->GetStringUTFChars(language, nullptr);
     params.language = language_chars;
-    params.n_threads = n_threads > 0 ? n_threads : max_threads;
+    params.n_threads = n_threads > 0 ? n_threads : default_n_threads;
     params.speed_up = speed_up;
     params.offset_ms = 0;
     params.no_context = true;
@@ -112,11 +163,7 @@ Java_com_rnwhisper_WhisperContext_fullTranscribe(
         params.temperature_inc = temperature_inc;
     }
     if (prompt != nullptr) {
-        rn_whisper_convert_prompt(
-            context,
-            params,
-            new std::string(env->GetStringUTFChars(prompt, nullptr))
-        );
+        params.initial_prompt = env->GetStringUTFChars(prompt, nullptr);
     }
 
     params.encoder_begin_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, void * user_data) {