whisper.rn 0.3.0 → 0.3.2

package/README.md

@@ -23,6 +23,8 @@ npm install whisper.rn
 
 For iOS, please re-run `npx pod-install` again.
 
+If you want to use `medium` or `large` model, the [Extended Virtual Addressing](https://developer.apple.com/documentation/bundleresources/entitlements/com_apple_developer_kernel_extended-virtual-addressing) capability is recommended to enable on iOS project.
+
 For Android, it's recommended to use `ndkVersion = "24.0.8215888"` (or above) in your root project build configuration for Apple Silicon Macs. Otherwise please follow this trobleshooting [issue](./TROUBLESHOOTING.md#android-got-build-error-unknown-host-cpu-architecture-arm64-on-apple-silicon-macs).
 
 For Expo, you will need to prebuild the project before using it. See [Expo guide](https://docs.expo.io/guides/using-libraries/#using-a-library-in-a-expo-project) for more details.
@@ -45,6 +47,9 @@ Add the following line to ```android/app/src/main/AndroidManifest.xml```
 ```xml
 <uses-permission android:name="android.permission.RECORD_AUDIO" />
 ```
+## Tips & Tricks
+
+The [Tips & Tricks](docs/TIPS.md) document is a collection of tips and tricks for using `whisper.rn`.
 
 ## Usage
 
@@ -121,7 +126,9 @@ module.exports = {
 }
 ```
 
-Please note that it will significantly increase the size of the app in release mode.
+Please note that:
+- It will significantly increase the size of the app in release mode.
+- The RN packager is not allowed file size larger than 2GB, so it not able to use original f16 `large` model (2.9GB), you can use quantized models instead.
 
 ## Core ML support
 
@@ -191,7 +198,7 @@ See the [contributing guide](CONTRIBUTING.md) to learn how to contribute to the
 
 ## Troubleshooting
 
-See the [troubleshooting](TROUBLESHOOTING.md) if you encounter any problem while using `whisper.rn`.
+See the [troubleshooting](docs/TROUBLESHOOTING.md) if you encounter any problem while using `whisper.rn`.
 
 ## License
 

package/android/src/main/java/com/rnwhisper/WhisperContext.java

@@ -71,7 +71,7 @@ public class WhisperContext {
     bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT);
   }
 
-  private void resetRealtimeTranscribe() {
+  private void rewind() {
     shortBufferSlices = null;
     sliceNSamples = null;
     sliceIndex = 0;
@@ -97,7 +97,7 @@ public class WhisperContext {
       return state;
     }
 
-    resetRealtimeTranscribe();
+    rewind();
 
     this.jobId = jobId;
 
@@ -269,7 +269,7 @@ public class WhisperContext {
       fullTranscribeSamples(options, true);
     } else if (isStopped) {
       // No next, cleanup
-      resetRealtimeTranscribe();
+      rewind();
     }
     isTranscribing = false;
   }
@@ -282,7 +282,32 @@ public class WhisperContext {
     eventEmitter.emit(eventName, event);
   }
 
+  private void emitProgress(int progress) {
+    WritableMap event = Arguments.createMap();
+    event.putInt("contextId", WhisperContext.this.id);
+    event.putInt("jobId", jobId);
+    event.putInt("progress", progress);
+    eventEmitter.emit("@RNWhisper_onTranscribeProgress", event);
+  }
+
+  private static class ProgressCallback {
+    WhisperContext context;
+
+    public ProgressCallback(WhisperContext context) {
+      this.context = context;
+    }
+
+    void onProgress(int progress) {
+      context.emitProgress(progress);
+    }
+  }
+
   public WritableMap transcribeInputStream(int jobId, InputStream inputStream, ReadableMap options) throws IOException, Exception {
+    if (isCapturing || isTranscribing) {
+      throw new Exception("Context is already in capturing or transcribing");
+    }
+    rewind();
+
     this.jobId = jobId;
     isTranscribing = true;
     float[] audioData = decodeWaveFile(inputStream);
@@ -292,7 +317,9 @@ public class WhisperContext {
     if (code != 0) {
       throw new Exception("Failed to transcribe the file. Code: " + code);
     }
-    return getTextSegments();
+    WritableMap result = getTextSegments();
+    result.putBoolean("isAborted", isStoppedByAction);
+    return result;
   }
 
   private int full(int jobId, ReadableMap options, float[] audioData, int audioDataLen) {
@@ -334,7 +361,9 @@ public class WhisperContext {
       // jstring language,
       options.hasKey("language") ? options.getString("language") : "auto",
       // jstring prompt
-      options.hasKey("prompt") ? options.getString("prompt") : null
+      options.hasKey("prompt") ? options.getString("prompt") : null,
+      // ProgressCallback progressCallback
+      options.hasKey("onProgress") && options.getBoolean("onProgress") ? new ProgressCallback(this) : null
     );
   }
 
@@ -469,6 +498,7 @@ public class WhisperContext {
     }
   }
 
+
   protected static native long initContext(String modelPath);
   protected static native long initContextWithAsset(AssetManager assetManager, String modelPath);
   protected static native long initContextWithInputStream(PushbackInputStream inputStream);
@@ -491,7 +521,8 @@ public class WhisperContext {
     boolean speed_up,
     boolean translate,
     String language,
-    String prompt
+    String prompt,
+    ProgressCallback progressCallback
   );
   protected static native void abortTranscribe(int jobId);
   protected static native void abortAllTranscribe();

package/android/src/main/jni/whisper/jni.cpp

@@ -184,6 +184,11 @@ Java_com_rnwhisper_WhisperContext_initContextWithInputStream(
     return reinterpret_cast<jlong>(context);
 }
 
+struct progress_callback_context {
+    JNIEnv *env;
+    jobject progress_callback_instance;
+};
+
 JNIEXPORT jint JNICALL
 Java_com_rnwhisper_WhisperContext_fullTranscribe(
     JNIEnv *env,
@@ -206,7 +211,8 @@ Java_com_rnwhisper_WhisperContext_fullTranscribe(
     jboolean speed_up,
     jboolean translate,
     jstring language,
-    jstring prompt
+    jstring prompt,
+    jobject progress_callback_instance
 ) {
     UNUSED(thiz);
     struct whisper_context *context = reinterpret_cast<struct whisper_context *>(context_ptr);
@@ -274,6 +280,21 @@ Java_com_rnwhisper_WhisperContext_fullTranscribe(
     };
     params.encoder_begin_callback_user_data = rn_whisper_assign_abort_map(job_id);
 
+    if (progress_callback_instance != nullptr) {
+        params.progress_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, int progress, void * user_data) {
+            progress_callback_context *cb_ctx = (progress_callback_context *)user_data;
+            JNIEnv *env = cb_ctx->env;
+            jobject progress_callback_instance = cb_ctx->progress_callback_instance;
+            jclass progress_callback_class = env->GetObjectClass(progress_callback_instance);
+            jmethodID onProgress = env->GetMethodID(progress_callback_class, "onProgress", "(I)V");
+            env->CallVoidMethod(progress_callback_instance, onProgress, progress);
+        };
+        progress_callback_context *cb_ctx = new progress_callback_context;
+        cb_ctx->env = env;
+        cb_ctx->progress_callback_instance = env->NewGlobalRef(progress_callback_instance);
+        params.progress_callback_user_data = cb_ctx;
+    }
+
     LOGI("About to reset timings");
     whisper_reset_timings(context);
 

package/cpp/rn-whisper.cpp

@@ -25,6 +25,13 @@ void rn_whisper_abort_transcribe(int job_id) {
   }
 }
 
+bool rn_whisper_transcribe_is_aborted(int job_id) {
+  if (abort_map.find(job_id) != abort_map.end()) {
+    return abort_map[job_id];
+  }
+  return false;
+}
+
 void rn_whisper_abort_all_transcribe() {
   for (auto it = abort_map.begin(); it != abort_map.end(); ++it) {
     it->second = true;

package/cpp/rn-whisper.h

@@ -8,6 +8,7 @@ extern "C" {
 bool* rn_whisper_assign_abort_map(int job_id);
 void rn_whisper_remove_abort_map(int job_id);
 void rn_whisper_abort_transcribe(int job_id);
+bool rn_whisper_transcribe_is_aborted(int job_id);
 void rn_whisper_abort_all_transcribe();
 
 #ifdef __cplusplus