whisper.rn 0.3.4 → 0.3.6

package/README.md

@@ -21,13 +21,26 @@ React Native binding of [whisper.cpp](https://github.com/ggerganov/whisper.cpp).
 npm install whisper.rn
 ```
 
-For iOS, please re-run `npx pod-install` again.
+#### iOS
+
+Please re-run `npx pod-install` again.
+
+#### Android
 
 If you want to use `medium` or `large` model, the [Extended Virtual Addressing](https://developer.apple.com/documentation/bundleresources/entitlements/com_apple_developer_kernel_extended-virtual-addressing) capability is recommended to enable on iOS project.
 
 For Android, it's recommended to use `ndkVersion = "24.0.8215888"` (or above) in your root project build configuration for Apple Silicon Macs. Otherwise please follow this trobleshooting [issue](./TROUBLESHOOTING.md#android-got-build-error-unknown-host-cpu-architecture-arm64-on-apple-silicon-macs).
 
-For Expo, you will need to prebuild the project before using it. See [Expo guide](https://docs.expo.io/guides/using-libraries/#using-a-library-in-a-expo-project) for more details.
+Don't forget to add proguard rule if it's enabled in project (android/app/proguard-rules.pro):
+
+```proguard
+# whisper.rn
+-keep class com.rnwhisper.** { *; }
+```
+
+#### Expo
+
+You will need to prebuild the project before using it. See [Expo guide](https://docs.expo.io/guides/using-libraries/#using-a-library-in-a-expo-project) for more details.
 
 ## Add Microphone Permissions (Optional)
 

package/android/build.gradle

@@ -30,6 +30,11 @@ def getExtOrIntegerDefault(name) {
   return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["RNWhisper_" + name]).toInteger()
 }
 
+def reactNativeArchitectures() {
+  def value = project.getProperties().get("reactNativeArchitectures")
+  return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
+}
+
 android {
   ndkVersion getExtOrDefault("ndkVersion")
   compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
@@ -38,10 +43,15 @@ android {
     minSdkVersion getExtOrIntegerDefault("minSdkVersion")
     targetSdkVersion getExtOrIntegerDefault("targetSdkVersion")
     buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", isNewArchitectureEnabled().toString()
+    externalNativeBuild {
+      cmake {
+        abiFilters (*reactNativeArchitectures())
+      }
+    }
   }
   externalNativeBuild {
-    ndkBuild {
-      path 'src/main/jni/whisper/Android.mk'
+    cmake {
+      path = file('src/main/CMakeLists.txt')
     }
   }
   buildTypes {

package/android/src/main/CMakeLists.txt

@@ -0,0 +1,55 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(whisper.rn)
+
+set(CMAKE_CXX_STANDARD 11)
+set(RNWHISPER_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../cpp)
+
+set(
+    SOURCE_FILES
+    ${RNWHISPER_LIB_DIR}/ggml.c
+    ${RNWHISPER_LIB_DIR}/whisper.cpp
+    ${RNWHISPER_LIB_DIR}/rn-whisper.cpp
+    ${CMAKE_SOURCE_DIR}/jni.cpp
+)
+
+find_library(LOG_LIB log)
+
+function(build_library target_name)
+    add_library(
+        ${target_name}
+        SHARED
+        ${SOURCE_FILES}
+    )
+    
+    target_link_libraries(${target_name} ${LOG_LIB} android)
+
+    if (${target_name} STREQUAL "whisper_v8fp16_va")
+        target_compile_options(${target_name} PRIVATE -march=armv8.2-a+fp16)
+    elseif (${target_name} STREQUAL "whisper_vfpv4")
+        target_compile_options(${target_name} PRIVATE -mfpu=neon-vfpv4)
+    endif ()
+
+    # NOTE: If you want to debug the native code, you can uncomment if and endif
+    # if (NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+
+        target_compile_options(${target_name} PRIVATE -O3 -DNDEBUG -pthread)
+        target_compile_options(${target_name} PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden)
+        target_compile_options(${target_name} PRIVATE -ffunction-sections -fdata-sections)
+
+        target_link_options(${target_name} PRIVATE -Wl,--gc-sections)
+        target_link_options(${target_name} PRIVATE -Wl,--exclude-libs,ALL)
+        target_link_options(${target_name} PRIVATE -flto)
+
+    # endif ()
+endfunction()
+
+build_library("whisper") # Default target
+
+if (${ANDROID_ABI} STREQUAL "arm64-v8a")
+    build_library("whisper_v8fp16_va")
+elseif (${ANDROID_ABI} STREQUAL "armeabi-v7a")
+    build_library("whisper_vfpv4")
+endif ()
+
+include_directories(${RNWHISPER_LIB_DIR})

package/android/src/main/java/com/rnwhisper/RNWhisper.java

@@ -0,0 +1,328 @@
+package com.rnwhisper;
+
+import androidx.annotation.NonNull;
+import android.util.Log;
+import android.os.Build;
+import android.os.Handler;
+import android.os.AsyncTask;
+import android.media.AudioRecord;
+
+import com.facebook.react.bridge.Promise;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.bridge.ReactMethod;
+import com.facebook.react.bridge.LifecycleEventListener;
+import com.facebook.react.bridge.ReadableMap;
+import com.facebook.react.bridge.WritableMap;
+
+import java.util.HashMap;
+import java.util.Random;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.PushbackInputStream;
+
+public class RNWhisper implements LifecycleEventListener {
+  public static final String NAME = "RNWhisper";
+
+  private ReactApplicationContext reactContext;
+  private Downloader downloader;
+
+  public RNWhisper(ReactApplicationContext reactContext) {
+    reactContext.addLifecycleEventListener(this);
+    this.reactContext = reactContext;
+    this.downloader = new Downloader(reactContext);
+  }
+
+  public HashMap<String, Object> getTypedExportedConstants() {
+    HashMap<String, Object> constants = new HashMap<>();
+
+    // iOS only constants, put for passing type checks
+    constants.put("useCoreML", false);
+    constants.put("coreMLAllowFallback", false);
+
+    return constants;
+  }
+
+  private HashMap<AsyncTask, String> tasks = new HashMap<>();
+
+  private HashMap<Integer, WhisperContext> contexts = new HashMap<>();
+
+  private int getResourceIdentifier(String filePath) {
+    int identifier = reactContext.getResources().getIdentifier(
+      filePath,
+      "drawable",
+      reactContext.getPackageName()
+    );
+    if (identifier == 0) {
+      identifier = reactContext.getResources().getIdentifier(
+        filePath,
+        "raw",
+        reactContext.getPackageName()
+      );
+    }
+    return identifier;
+  }
+
+  public void initContext(final ReadableMap options, final Promise promise) {
+    AsyncTask task = new AsyncTask<Void, Void, Integer>() {
+      private Exception exception;
+
+      @Override
+      protected Integer doInBackground(Void... voids) {
+        try {
+          String modelPath = options.getString("filePath");
+          boolean isBundleAsset = options.getBoolean("isBundleAsset");
+
+          String modelFilePath = modelPath;
+          if (!isBundleAsset && (modelPath.startsWith("http://") || modelPath.startsWith("https://"))) {
+            modelFilePath = downloader.downloadFile(modelPath);
+          }
+
+          long context;
+          int resId = getResourceIdentifier(modelFilePath);
+          if (resId > 0) {
+            context = WhisperContext.initContextWithInputStream(
+              new PushbackInputStream(reactContext.getResources().openRawResource(resId))
+            );
+          } else if (isBundleAsset) {
+            context = WhisperContext.initContextWithAsset(reactContext.getAssets(), modelFilePath);
+          } else {
+            context = WhisperContext.initContext(modelFilePath);
+          }
+          if (context == 0) {
+            throw new Exception("Failed to initialize context");
+          }
+          int id = Math.abs(new Random().nextInt());
+          WhisperContext whisperContext = new WhisperContext(id, reactContext, context);
+          contexts.put(id, whisperContext);
+          return id;
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
+
+      @Override
+      protected void onPostExecute(Integer id) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(id);
+        tasks.remove(this);
+      }
+    }.execute();
+    tasks.put(task, "initContext");
+  }
+
+  public void transcribeFile(double id, double jobId, String filePath, ReadableMap options, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    if (context.isCapturing()) {
+      promise.reject("The context is in realtime transcribe mode");
+      return;
+    }
+    if (context.isTranscribing()) {
+      promise.reject("Context is already transcribing");
+      return;
+    }
+    AsyncTask task = new AsyncTask<Void, Void, WritableMap>() {
+      private Exception exception;
+
+      @Override
+      protected WritableMap doInBackground(Void... voids) {
+        try {
+          String waveFilePath = filePath;
+
+          if (filePath.startsWith("http://") || filePath.startsWith("https://")) {
+            waveFilePath = downloader.downloadFile(filePath);
+          }
+
+          int resId = getResourceIdentifier(waveFilePath);
+          if (resId > 0) {
+            return context.transcribeInputStream(
+              (int) jobId,
+              reactContext.getResources().openRawResource(resId),
+              options
+            );
+          }
+
+          return context.transcribeInputStream(
+            (int) jobId,
+            new FileInputStream(new File(waveFilePath)),
+            options
+          );
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
+
+      @Override
+      protected void onPostExecute(WritableMap data) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(data);
+        tasks.remove(this);
+      }
+    }.execute();
+    tasks.put(task, "transcribeFile-" + id);
+  }
+
+  public void startRealtimeTranscribe(double id, double jobId, ReadableMap options, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    if (context.isCapturing()) {
+      promise.reject("Context is already in capturing");
+      return;
+    }
+    int state = context.startRealtimeTranscribe((int) jobId, options);
+    if (state == AudioRecord.STATE_INITIALIZED) {
+      promise.resolve(null);
+      return;
+    }
+    promise.reject("Failed to start realtime transcribe. State: " + state);
+  }
+
+  public void abortTranscribe(double id, double jobId, Promise promise) {
+    WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    AsyncTask task = new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          context.stopTranscribe((int) jobId);
+          AsyncTask completionTask = null;
+          for (AsyncTask task : tasks.keySet()) {
+            if (tasks.get(task).equals("transcribeFile-" + id)) {
+              task.get();
+              break;
+            }
+          }
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+        tasks.remove(this);
+      }
+    }.execute();
+    tasks.put(task, "abortTranscribe-" + id);
+  }
+
+  public void releaseContext(double id, Promise promise) {
+    final int contextId = (int) id;
+    AsyncTask task = new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          WhisperContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context " + id + " not found");
+          }
+          context.stopCurrentTranscribe();
+          AsyncTask completionTask = null;
+          for (AsyncTask task : tasks.keySet()) {
+            if (tasks.get(task).equals("transcribeFile-" + contextId)) {
+              task.get();
+              break;
+            }
+          }
+          context.release();
+          contexts.remove(contextId);
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+        tasks.remove(this);
+      }
+    }.execute();
+    tasks.put(task, "releaseContext-" + id);
+  }
+
+  public void releaseAllContexts(Promise promise) {
+    AsyncTask task = new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          onHostDestroy();
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+        tasks.remove(this);
+      }
+    }.execute();
+    tasks.put(task, "releaseAllContexts");
+  }
+
+  @Override
+  public void onHostResume() {
+  }
+
+  @Override
+  public void onHostPause() {
+  }
+
+  @Override
+  public void onHostDestroy() {
+    for (WhisperContext context : contexts.values()) {
+      context.stopCurrentTranscribe();
+    }
+    for (AsyncTask task : tasks.keySet()) {
+      try {
+        task.get();
+      } catch (Exception e) {
+        Log.e(NAME, "Failed to wait for task", e);
+      }
+    }
+    for (WhisperContext context : contexts.values()) {
+      context.release();
+    }
+    WhisperContext.abortAllTranscribe(); // graceful abort
+    contexts.clear();
+    downloader.clearCache();
+  }
+}

package/android/src/main/java/com/rnwhisper/WhisperContext.java

@@ -61,6 +61,7 @@ public class WhisperContext {
   private boolean isCapturing = false;
   private boolean isStoppedByAction = false;
   private boolean isTranscribing = false;
+  private Thread rootFullHandler = null;
   private Thread fullHandler = null;
 
   public WhisperContext(int id, ReactApplicationContext reactContext, long context) {
@@ -81,6 +82,7 @@ public class WhisperContext {
     isCapturing = false;
     isStoppedByAction = false;
     isTranscribing = false;
+    rootFullHandler = null;
     fullHandler = null;
   }
 
@@ -117,7 +119,7 @@ public class WhisperContext {
     isCapturing = true;
     recorder.startRecording();
 
-    new Thread(new Runnable() {
+    rootFullHandler = new Thread(new Runnable() {
       @Override
       public void run() {
         try {
@@ -195,7 +197,8 @@ public class WhisperContext {
           recorder = null;
         }
       }
-    }).start();
+    });
+    rootFullHandler.start();
     return state;
   }
 
@@ -402,6 +405,14 @@ public class WhisperContext {
     abortTranscribe(jobId);
     isCapturing = false;
     isStoppedByAction = true;
+    if (rootFullHandler != null) {
+      try {
+        rootFullHandler.join();
+      } catch (Exception e) {
+        Log.e(NAME, "Error joining rootFullHandler: " + e.getMessage());
+      }
+      rootFullHandler = null;
+    }
   }
 
   public void stopCurrentTranscribe() {