whisper.rn 0.3.3 → 0.3.5

package/README.md

@@ -21,13 +21,26 @@ React Native binding of [whisper.cpp](https://github.com/ggerganov/whisper.cpp).
 npm install whisper.rn
 ```
 
-For iOS, please re-run `npx pod-install` again.
+#### iOS
+
+Please re-run `npx pod-install` again.
+
+#### Android
 
 If you want to use `medium` or `large` model, the [Extended Virtual Addressing](https://developer.apple.com/documentation/bundleresources/entitlements/com_apple_developer_kernel_extended-virtual-addressing) capability is recommended to enable on iOS project.
 
 For Android, it's recommended to use `ndkVersion = "24.0.8215888"` (or above) in your root project build configuration for Apple Silicon Macs. Otherwise please follow this trobleshooting [issue](./TROUBLESHOOTING.md#android-got-build-error-unknown-host-cpu-architecture-arm64-on-apple-silicon-macs).
 
-For Expo, you will need to prebuild the project before using it. See [Expo guide](https://docs.expo.io/guides/using-libraries/#using-a-library-in-a-expo-project) for more details.
+Don't forget to add proguard rule if it's enabled in project (android/app/proguard-rules.pro):
+
+```proguard
+# whisper.rn
+-keep class com.rnwhisper.** { *; }
+```
+
+#### Expo
+
+You will need to prebuild the project before using it. See [Expo guide](https://docs.expo.io/guides/using-libraries/#using-a-library-in-a-expo-project) for more details.
 
 ## Add Microphone Permissions (Optional)
 

package/android/build.gradle

@@ -30,6 +30,11 @@ def getExtOrIntegerDefault(name) {
   return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["RNWhisper_" + name]).toInteger()
 }
 
+def reactNativeArchitectures() {
+  def value = project.getProperties().get("reactNativeArchitectures")
+  return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
+}
+
 android {
   ndkVersion getExtOrDefault("ndkVersion")
   compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
@@ -38,10 +43,15 @@ android {
     minSdkVersion getExtOrIntegerDefault("minSdkVersion")
     targetSdkVersion getExtOrIntegerDefault("targetSdkVersion")
     buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", isNewArchitectureEnabled().toString()
+    externalNativeBuild {
+      cmake {
+        abiFilters (*reactNativeArchitectures())
+      }
+    }
   }
   externalNativeBuild {
-    ndkBuild {
-      path 'src/main/jni/whisper/Android.mk'
+    cmake {
+      path = file('src/main/CMakeLists.txt')
     }
   }
   buildTypes {

package/android/src/main/CMakeLists.txt

@@ -0,0 +1,55 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(whisper.rn)
+
+set(CMAKE_CXX_STANDARD 11)
+set(RNWHISPER_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../cpp)
+
+set(
+    SOURCE_FILES
+    ${RNWHISPER_LIB_DIR}/ggml.c
+    ${RNWHISPER_LIB_DIR}/whisper.cpp
+    ${RNWHISPER_LIB_DIR}/rn-whisper.cpp
+    ${CMAKE_SOURCE_DIR}/jni.cpp
+)
+
+find_library(LOG_LIB log)
+
+function(build_library target_name)
+    add_library(
+        ${target_name}
+        SHARED
+        ${SOURCE_FILES}
+    )
+    
+    target_link_libraries(${target_name} ${LOG_LIB} android)
+
+    if (${target_name} STREQUAL "whisper_v8fp16_va")
+        target_compile_options(${target_name} PRIVATE -march=armv8.2-a+fp16)
+    elseif (${target_name} STREQUAL "whisper_vfpv4")
+        target_compile_options(${target_name} PRIVATE -mfpu=neon-vfpv4)
+    endif ()
+
+    # NOTE: If you want to debug the native code, you can uncomment if and endif
+    # if (NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+
+        target_compile_options(${target_name} PRIVATE -O3 -DNDEBUG -pthread)
+        target_compile_options(${target_name} PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden)
+        target_compile_options(${target_name} PRIVATE -ffunction-sections -fdata-sections)
+
+        target_link_options(${target_name} PRIVATE -Wl,--gc-sections)
+        target_link_options(${target_name} PRIVATE -Wl,--exclude-libs,ALL)
+        target_link_options(${target_name} PRIVATE -flto)
+
+    # endif ()
+endfunction()
+
+build_library("whisper") # Default target
+
+if (${ANDROID_ABI} STREQUAL "arm64-v8a")
+    build_library("whisper_v8fp16_va")
+elseif (${ANDROID_ABI} STREQUAL "armeabi-v7a")
+    build_library("whisper_vfpv4")
+endif ()
+
+include_directories(${RNWHISPER_LIB_DIR})

package/android/src/main/java/com/rnwhisper/RNWhisper.java

@@ -0,0 +1,268 @@
+package com.rnwhisper;
+
+import androidx.annotation.NonNull;
+import android.util.Log;
+import android.os.Build;
+import android.os.Handler;
+import android.os.AsyncTask;
+import android.media.AudioRecord;
+
+import com.facebook.react.bridge.Promise;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.bridge.ReactMethod;
+import com.facebook.react.bridge.LifecycleEventListener;
+import com.facebook.react.bridge.ReadableMap;
+import com.facebook.react.bridge.WritableMap;
+
+import java.util.HashMap;
+import java.util.Random;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.PushbackInputStream;
+
+public class RNWhisper implements LifecycleEventListener {
+  private ReactApplicationContext reactContext;
+  private Downloader downloader;
+
+  public RNWhisper(ReactApplicationContext reactContext) {
+    reactContext.addLifecycleEventListener(this);
+    this.reactContext = reactContext;
+    this.downloader = new Downloader(reactContext);
+  }
+
+  public HashMap<String, Object> getTypedExportedConstants() {
+    HashMap<String, Object> constants = new HashMap<>();
+
+    // iOS only constants, put for passing type checks
+    constants.put("useCoreML", false);
+    constants.put("coreMLAllowFallback", false);
+
+    return constants;
+  }
+
+  private HashMap<Integer, WhisperContext> contexts = new HashMap<>();
+
+  private int getResourceIdentifier(String filePath) {
+    int identifier = reactContext.getResources().getIdentifier(
+      filePath,
+      "drawable",
+      reactContext.getPackageName()
+    );
+    if (identifier == 0) {
+      identifier = reactContext.getResources().getIdentifier(
+        filePath,
+        "raw",
+        reactContext.getPackageName()
+      );
+    }
+    return identifier;
+  }
+
+  public void initContext(final ReadableMap options, final Promise promise) {
+    new AsyncTask<Void, Void, Integer>() {
+      private Exception exception;
+
+      @Override
+      protected Integer doInBackground(Void... voids) {
+        try {
+          String modelPath = options.getString("filePath");
+          boolean isBundleAsset = options.getBoolean("isBundleAsset");
+
+          String modelFilePath = modelPath;
+          if (!isBundleAsset && (modelPath.startsWith("http://") || modelPath.startsWith("https://"))) {
+            modelFilePath = downloader.downloadFile(modelPath);
+          }
+
+          long context;
+          int resId = getResourceIdentifier(modelFilePath);
+          if (resId > 0) {
+            context = WhisperContext.initContextWithInputStream(
+              new PushbackInputStream(reactContext.getResources().openRawResource(resId))
+            );
+          } else if (isBundleAsset) {
+            context = WhisperContext.initContextWithAsset(reactContext.getAssets(), modelFilePath);
+          } else {
+            context = WhisperContext.initContext(modelFilePath);
+          }
+          if (context == 0) {
+            throw new Exception("Failed to initialize context");
+          }
+          int id = Math.abs(new Random().nextInt());
+          WhisperContext whisperContext = new WhisperContext(id, reactContext, context);
+          contexts.put(id, whisperContext);
+          return id;
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
+
+      @Override
+      protected void onPostExecute(Integer id) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(id);
+      }
+    }.execute();
+  }
+
+  public void transcribeFile(double id, double jobId, String filePath, ReadableMap options, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    if (context.isCapturing()) {
+      promise.reject("The context is in realtime transcribe mode");
+      return;
+    }
+    if (context.isTranscribing()) {
+      promise.reject("Context is already transcribing");
+      return;
+    }
+    new AsyncTask<Void, Void, WritableMap>() {
+      private Exception exception;
+
+      @Override
+      protected WritableMap doInBackground(Void... voids) {
+        try {
+          String waveFilePath = filePath;
+
+          if (filePath.startsWith("http://") || filePath.startsWith("https://")) {
+            waveFilePath = downloader.downloadFile(filePath);
+          }
+
+          int resId = getResourceIdentifier(waveFilePath);
+          if (resId > 0) {
+            return context.transcribeInputStream(
+              (int) jobId,
+              reactContext.getResources().openRawResource(resId),
+              options
+            );
+          }
+
+          return context.transcribeInputStream(
+            (int) jobId,
+            new FileInputStream(new File(waveFilePath)),
+            options
+          );
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
+
+      @Override
+      protected void onPostExecute(WritableMap data) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(data);
+      }
+    }.execute();
+  }
+
+  public void startRealtimeTranscribe(double id, double jobId, ReadableMap options, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    if (context.isCapturing()) {
+      promise.reject("Context is already in capturing");
+      return;
+    }
+    int state = context.startRealtimeTranscribe((int) jobId, options);
+    if (state == AudioRecord.STATE_INITIALIZED) {
+      promise.resolve(null);
+      return;
+    }
+    promise.reject("Failed to start realtime transcribe. State: " + state);
+  }
+
+  public void abortTranscribe(double contextId, double jobId, Promise promise) {
+    WhisperContext context = contexts.get((int) contextId);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    context.stopTranscribe((int) jobId);
+  }
+
+  public void releaseContext(double id, Promise promise) {
+    final int contextId = (int) id;
+    new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          WhisperContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context " + id + " not found");
+          }
+          context.release();
+          contexts.remove(contextId);
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+      }
+    }.execute();
+  }
+
+  public void releaseAllContexts(Promise promise) {
+    new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          onHostDestroy();
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+      }
+    }.execute();
+  }
+
+  @Override
+  public void onHostResume() {
+  }
+
+  @Override
+  public void onHostPause() {
+  }
+
+  @Override
+  public void onHostDestroy() {
+    WhisperContext.abortAllTranscribe();
+    for (WhisperContext context : contexts.values()) {
+      context.release();
+    }
+    contexts.clear();
+    downloader.clearCache();
+  }
+}

package/android/src/newarch/java/com/rnwhisper/RNWhisperModule.java

@@ -1,18 +1,11 @@
 package com.rnwhisper;
 
 import androidx.annotation.NonNull;
-import android.util.Log;
-import android.os.Build;
-import android.os.Handler;
-import android.os.AsyncTask;
-import android.media.AudioRecord;
 
 import com.facebook.react.bridge.Promise;
 import com.facebook.react.bridge.ReactApplicationContext;
 import com.facebook.react.bridge.ReactMethod;
-import com.facebook.react.bridge.LifecycleEventListener;
 import com.facebook.react.bridge.ReadableMap;
-import com.facebook.react.bridge.WritableMap;
 import com.facebook.react.module.annotations.ReactModule;
 
 import java.util.HashMap;
@@ -22,17 +15,14 @@ import java.io.FileInputStream;
 import java.io.PushbackInputStream;
 
 @ReactModule(name = RNWhisperModule.NAME)
-public class RNWhisperModule extends NativeRNWhisperSpec implements LifecycleEventListener {
+public class RNWhisperModule extends NativeRNWhisperSpec {
   public static final String NAME = "RNWhisper";
 
-  private ReactApplicationContext reactContext;
-  private Downloader downloader;
+  private RNWhisper rnwhisper;
 
   public RNWhisperModule(ReactApplicationContext reactContext) {
     super(reactContext);
-    reactContext.addLifecycleEventListener(this);
-    this.reactContext = reactContext;
-    this.downloader = new Downloader(reactContext);
+    rnwhisper = new RNWhisper(reactContext);
   }
 
   @Override
@@ -43,244 +33,36 @@ public class RNWhisperModule extends NativeRNWhisperSpec implements LifecycleEve
 
   @Override
   public HashMap<String, Object> getTypedExportedConstants() {
-    HashMap<String, Object> constants = new HashMap<>();
-
-    // iOS only constants, put for passing type checks
-    constants.put("useCoreML", false);
-    constants.put("coreMLAllowFallback", false);
-
-    return constants;
-  }
-
-  private HashMap<Integer, WhisperContext> contexts = new HashMap<>();
-
-  private int getResourceIdentifier(String filePath) {
-    int identifier = reactContext.getResources().getIdentifier(
-      filePath,
-      "drawable",
-      reactContext.getPackageName()
-    );
-    if (identifier == 0) {
-      identifier = reactContext.getResources().getIdentifier(
-        filePath,
-        "raw",
-        reactContext.getPackageName()
-      );
-    }
-    return identifier;
+    return rnwhisper.getTypedExportedConstants();
   }
 
   @ReactMethod
   public void initContext(final ReadableMap options, final Promise promise) {
-    new AsyncTask<Void, Void, Integer>() {
-      private Exception exception;
-
-      @Override
-      protected Integer doInBackground(Void... voids) {
-        try {
-          String modelPath = options.getString("filePath");
-          boolean isBundleAsset = options.getBoolean("isBundleAsset");
-
-          String modelFilePath = modelPath;
-          if (!isBundleAsset && (modelPath.startsWith("http://") || modelPath.startsWith("https://"))) {
-            modelFilePath = downloader.downloadFile(modelPath);
-          }
-
-          long context;
-          int resId = getResourceIdentifier(modelFilePath);
-          if (resId > 0) {
-            context = WhisperContext.initContextWithInputStream(
-              new PushbackInputStream(reactContext.getResources().openRawResource(resId))
-            );
-          } else if (isBundleAsset) {
-            context = WhisperContext.initContextWithAsset(reactContext.getAssets(), modelFilePath);
-          } else {
-            context = WhisperContext.initContext(modelFilePath);
-          }
-          if (context == 0) {
-            throw new Exception("Failed to initialize context");
-          }
-          int id = Math.abs(new Random().nextInt());
-          WhisperContext whisperContext = new WhisperContext(id, reactContext, context);
-          contexts.put(id, whisperContext);
-          return id;
-        } catch (Exception e) {
-          exception = e;
-          return null;
-        }
-      }
-
-      @Override
-      protected void onPostExecute(Integer id) {
-        if (exception != null) {
-          promise.reject(exception);
-          return;
-        }
-        promise.resolve(id);
-      }
-    }.execute();
+    rnwhisper.initContext(options, promise);
   }
 
   @ReactMethod
   public void transcribeFile(double id, double jobId, String filePath, ReadableMap options, Promise promise) {
-    final WhisperContext context = contexts.get((int) id);
-    if (context == null) {
-      promise.reject("Context not found");
-      return;
-    }
-    if (context.isCapturing()) {
-      promise.reject("The context is in realtime transcribe mode");
-      return;
-    }
-    if (context.isTranscribing()) {
-      promise.reject("Context is already transcribing");
-      return;
-    }
-    new AsyncTask<Void, Void, WritableMap>() {
-      private Exception exception;
-
-      @Override
-      protected WritableMap doInBackground(Void... voids) {
-        try {
-          String waveFilePath = filePath;
-
-          if (filePath.startsWith("http://") || filePath.startsWith("https://")) {
-            waveFilePath = downloader.downloadFile(filePath);
-          }
-
-          int resId = getResourceIdentifier(waveFilePath);
-          if (resId > 0) {
-            return context.transcribeInputStream(
-              (int) jobId,
-              reactContext.getResources().openRawResource(resId),
-              options
-            );
-          }
-
-          return context.transcribeInputStream(
-            (int) jobId,
-            new FileInputStream(new File(waveFilePath)),
-            options
-          );
-        } catch (Exception e) {
-          exception = e;
-          return null;
-        }
-      }
-
-      @Override
-      protected void onPostExecute(WritableMap data) {
-        if (exception != null) {
-          promise.reject(exception);
-          return;
-        }
-        promise.resolve(data);
-      }
-    }.execute();
+    rnwhisper.transcribeFile(id, jobId, filePath, options, promise);
   }
 
   @ReactMethod
   public void startRealtimeTranscribe(double id, double jobId, ReadableMap options, Promise promise) {
-    final WhisperContext context = contexts.get((int) id);
-    if (context == null) {
-      promise.reject("Context not found");
-      return;
-    }
-    if (context.isCapturing()) {
-      promise.reject("Context is already in capturing");
-      return;
-    }
-    int state = context.startRealtimeTranscribe((int) jobId, options);
-    if (state == AudioRecord.STATE_INITIALIZED) {
-      promise.resolve(null);
-      return;
-    }
-    promise.reject("Failed to start realtime transcribe. State: " + state);
+    rnwhisper.startRealtimeTranscribe(id, jobId, options, promise);
   }
 
   @ReactMethod
   public void abortTranscribe(double contextId, double jobId, Promise promise) {
-    WhisperContext context = contexts.get((int) contextId);
-    if (context == null) {
-      promise.reject("Context not found");
-      return;
-    }
-    context.stopTranscribe((int) jobId);
+    rnwhisper.abortTranscribe(contextId, jobId, promise);
   }
 
   @ReactMethod
   public void releaseContext(double id, Promise promise) {
-    final int contextId = (int) id;
-    new AsyncTask<Void, Void, Void>() {
-      private Exception exception;
-
-      @Override
-      protected Void doInBackground(Void... voids) {
-        try {
-          WhisperContext context = contexts.get(contextId);
-          if (context == null) {
-            throw new Exception("Context " + id + " not found");
-          }
-          context.release();
-          contexts.remove(contextId);
-        } catch (Exception e) {
-          exception = e;
-        }
-        return null;
-      }
-
-      @Override
-      protected void onPostExecute(Void result) {
-        if (exception != null) {
-          promise.reject(exception);
-          return;
-        }
-        promise.resolve(null);
-      }
-    }.execute();
+    rnwhisper.releaseContext(id, promise);
   }
 
   @ReactMethod
   public void releaseAllContexts(Promise promise) {
-    new AsyncTask<Void, Void, Void>() {
-      private Exception exception;
-
-      @Override
-      protected Void doInBackground(Void... voids) {
-        try {
-          onHostDestroy();
-        } catch (Exception e) {
-          exception = e;
-        }
-        return null;
-      }
-
-      @Override
-      protected void onPostExecute(Void result) {
-        if (exception != null) {
-          promise.reject(exception);
-          return;
-        }
-        promise.resolve(null);
-      }
-    }.execute();
-  }
-
-  @Override
-  public void onHostResume() {
-  }
-
-  @Override
-  public void onHostPause() {
-  }
-
-  @Override
-  public void onHostDestroy() {
-    WhisperContext.abortAllTranscribe();
-    for (WhisperContext context : contexts.values()) {
-      context.release();
-    }
-    contexts.clear();
-    downloader.clearCache();
+    rnwhisper.releaseAllContexts(promise);
   }
 }