whisper.rn 0.4.0-rc.2 → 0.4.0-rc.4

package/android/src/main/CMakeLists.txt

@@ -9,6 +9,8 @@ set(
     SOURCE_FILES
     ${RNWHISPER_LIB_DIR}/ggml.c
     ${RNWHISPER_LIB_DIR}/ggml-alloc.c
+    ${RNWHISPER_LIB_DIR}/ggml-backend.c
+    ${RNWHISPER_LIB_DIR}/ggml-quants.c
     ${RNWHISPER_LIB_DIR}/whisper.cpp
     ${RNWHISPER_LIB_DIR}/rn-whisper.cpp
     ${CMAKE_SOURCE_DIR}/jni.cpp

package/android/src/main/java/com/rnwhisper/RNWhisper.java

@@ -13,6 +13,7 @@ import com.facebook.react.bridge.ReactMethod;
 import com.facebook.react.bridge.LifecycleEventListener;
 import com.facebook.react.bridge.ReadableMap;
 import com.facebook.react.bridge.WritableMap;
+import com.facebook.react.bridge.Arguments;
 
 import java.util.HashMap;
 import java.util.Random;
@@ -107,7 +108,11 @@ public class RNWhisper implements LifecycleEventListener {
           promise.reject(exception);
           return;
         }
-        promise.resolve(id);
+        WritableMap result = Arguments.createMap();
+        result.putInt("contextId", id);
+        result.putBoolean("gpu", false);
+        result.putString("reasonNoGPU", "Currently not supported");
+        promise.resolve(result);
         tasks.remove(this);
       }
     }.execute();

package/android/src/main/java/com/rnwhisper/WhisperContext.java

@@ -82,8 +82,9 @@ public class WhisperContext {
   private boolean vad(ReadableMap options, short[] shortBuffer, int nSamples, int n) {
     boolean isSpeech = true;
     if (!isTranscribing && options.hasKey("useVad") && options.getBoolean("useVad")) {
-      int vadSec = options.hasKey("vadMs") ? options.getInt("vadMs") / 1000 : 2;
-      int sampleSize = vadSec * SAMPLE_RATE;
+      int vadMs = options.hasKey("vadMs") ? options.getInt("vadMs") : 2000;
+      if (vadMs < 2000) vadMs = 2000;
+      int sampleSize = (int) (SAMPLE_RATE * vadMs / 1000);
       if (nSamples + n > sampleSize) {
         int start = nSamples + n - sampleSize;
         float[] audioData = new float[sampleSize];
@@ -100,6 +101,21 @@ public class WhisperContext {
     return isSpeech;
   }
 
+  private void finishRealtimeTranscribe(ReadableMap options, WritableMap result) {
+    String audioOutputPath = options.hasKey("audioOutputPath") ? options.getString("audioOutputPath") : null;
+    if (audioOutputPath != null) {
+       // TODO: Append in real time so we don't need to keep all slices & also reduce memory usage
+      Log.d(NAME, "Begin saving wav file to " + audioOutputPath);
+      try {
+        AudioUtils.saveWavFile(AudioUtils.concatShortBuffers(shortBufferSlices), audioOutputPath);
+      } catch (IOException e) {
+        Log.e(NAME, "Error saving wav file: " + e.getMessage());
+      }
+    }
+
+    emitTranscribeEvent("@RNWhisper_onRealtimeTranscribeEnd", Arguments.createMap());
+  }
+
   public int startRealtimeTranscribe(int jobId, ReadableMap options) {
     if (isCapturing || isTranscribing) {
       return -100;
@@ -131,7 +147,7 @@ public class WhisperContext {
     shortBufferSlices.add(new short[audioSliceSec * SAMPLE_RATE]);
     sliceNSamples = new ArrayList<Integer>();
     sliceNSamples.add(0);
-  
+
     isCapturing = true;
     recorder.startRecording();
 
@@ -159,12 +175,12 @@ public class WhisperContext {
                   nSamples == nSamplesTranscribing &&
                   sliceIndex == transcribeSliceIndex
                 ) {
-                  emitTranscribeEvent("@RNWhisper_onRealtimeTranscribeEnd", Arguments.createMap());
+                  finishRealtimeTranscribe(options, Arguments.createMap());
                 } else if (!isTranscribing) {
                   short[] shortBuffer = shortBufferSlices.get(sliceIndex);
                   boolean isSpeech = vad(options, shortBuffer, nSamples, 0);
                   if (!isSpeech) {
-                    emitTranscribeEvent("@RNWhisper_onRealtimeTranscribeEnd", Arguments.createMap());
+                    finishRealtimeTranscribe(options, Arguments.createMap());
                     break;
                   }
                   isTranscribing = true;
@@ -210,11 +226,9 @@ public class WhisperContext {
               Log.e(NAME, "Error transcribing realtime: " + e.getMessage());
             }
           }
-          // TODO: Append in real time so we don't need to keep all slices & also reduce memory usage
-          Log.d(NAME, "Begin saving wav file to " + audioOutputPath);
-          AudioUtils.saveWavFile(AudioUtils.concatShortBuffers(shortBufferSlices), audioOutputPath);
+
           if (!isTranscribing) {
-            emitTranscribeEvent("@RNWhisper_onRealtimeTranscribeEnd", Arguments.createMap());
+            finishRealtimeTranscribe(options, Arguments.createMap());
           }
           if (fullHandler != null) {
             fullHandler.join(); // Wait for full transcribe to finish
@@ -264,7 +278,7 @@ public class WhisperContext {
 
     if (code == 0) {
       payload.putMap("data", getTextSegments(0, getTextSegmentCount(context)));
-    } else {
+    } else if (code != -999) { // Not aborted
       payload.putString("error", "Transcribe failed with code " + code);
     }
 
@@ -283,12 +297,12 @@ public class WhisperContext {
       nSamplesTranscribing = 0;
     }
 
-    boolean continueNeeded = !isCapturing && nSamplesTranscribing != nSamplesOfIndex;
+    boolean continueNeeded = !isCapturing && nSamplesTranscribing != nSamplesOfIndex && code != -999;
 
     if (isStopped && !continueNeeded) {
       payload.putBoolean("isCapturing", false);
       payload.putBoolean("isStoppedByAction", isStoppedByAction);
-      emitTranscribeEvent("@RNWhisper_onRealtimeTranscribeEnd", payload);
+      finishRealtimeTranscribe(options, payload);
     } else if (code == 0) {
       payload.putBoolean("isCapturing", true);
       emitTranscribeEvent("@RNWhisper_onRealtimeTranscribe", payload);
@@ -372,7 +386,7 @@ public class WhisperContext {
     int code = full(jobId, options, audioData, audioData.length);
     isTranscribing = false;
     this.jobId = -1;
-    if (code != 0) {
+    if (code != 0 && code != 999) {
       throw new Exception("Failed to transcribe the file. Code: " + code);
     }
     WritableMap result = getTextSegments(0, getTextSegmentCount(context));
@@ -401,7 +415,7 @@ public class WhisperContext {
       options.hasKey("maxLen") ? options.getInt("maxLen") : -1,
       // jboolean token_timestamps,
       options.hasKey("tokenTimestamps") ? options.getBoolean("tokenTimestamps") : false,
-  
+
       // jint offset,
       options.hasKey("offset") ? options.getInt("offset") : -1,
       // jint duration,
@@ -577,4 +591,4 @@ public class WhisperContext {
   protected static native int getTextSegmentT0(long context, int index);
   protected static native int getTextSegmentT1(long context, int index);
   protected static native void freeContext(long contextPtr);
-}
+}

package/android/src/main/jni.cpp

@@ -297,16 +297,17 @@ Java_com_rnwhisper_WhisperContext_fullTranscribe(
     }
 
     // abort handlers
+    bool* abort_ptr = rn_whisper_assign_abort_map(job_id);
     params.encoder_begin_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, void * user_data) {
         bool is_aborted = *(bool*)user_data;
         return !is_aborted;
     };
-    params.encoder_begin_callback_user_data = rn_whisper_assign_abort_map(job_id);
+    params.encoder_begin_callback_user_data = abort_ptr;
     params.abort_callback = [](void * user_data) {
         bool is_aborted = *(bool*)user_data;
         return is_aborted;
     };
-    params.abort_callback_user_data = rn_whisper_assign_abort_map(job_id);
+    params.abort_callback_user_data = abort_ptr;
 
     if (callback_instance != nullptr) {
         callback_context *cb_ctx = new callback_context;
@@ -344,6 +345,9 @@ Java_com_rnwhisper_WhisperContext_fullTranscribe(
     }
     env->ReleaseFloatArrayElements(audio_data, audio_data_arr, JNI_ABORT);
     env->ReleaseStringUTFChars(language, language_chars);
+    if (rn_whisper_transcribe_is_aborted(job_id)) {
+        code = -999;
+    }
     rn_whisper_remove_abort_map(job_id);
     return code;
 }