whisper.rn 0.4.0-rc.1 → 0.4.0-rc.11

package/README.md

@@ -25,19 +25,19 @@ npm install whisper.rn
 
 Please re-run `npx pod-install` again.
 
-#### Android
-
 If you want to use `medium` or `large` model, the [Extended Virtual Addressing](https://developer.apple.com/documentation/bundleresources/entitlements/com_apple_developer_kernel_extended-virtual-addressing) capability is recommended to enable on iOS project.
 
-For Android, it's recommended to use `ndkVersion = "24.0.8215888"` (or above) in your root project build configuration for Apple Silicon Macs. Otherwise please follow this trobleshooting [issue](./TROUBLESHOOTING.md#android-got-build-error-unknown-host-cpu-architecture-arm64-on-apple-silicon-macs).
+#### Android
 
-Don't forget to add proguard rule if it's enabled in project (android/app/proguard-rules.pro):
+Add proguard rule if it's enabled in project (android/app/proguard-rules.pro):
 
 ```proguard
 # whisper.rn
 -keep class com.rnwhisper.** { *; }
 ```
 
+For build, it's recommended to use `ndkVersion = "24.0.8215888"` (or above) in your root project build configuration for Apple Silicon Macs. Otherwise please follow this trobleshooting [issue](./TROUBLESHOOTING.md#android-got-build-error-unknown-host-cpu-architecture-arm64-on-apple-silicon-macs).
+
 #### Expo
 
 You will need to prebuild the project before using it. See [Expo guide](https://docs.expo.io/guides/using-libraries/#using-a-library-in-a-expo-project) for more details.
@@ -91,7 +91,7 @@ subscribe(evt => {
   console.log(
     `Realtime transcribing: ${isCapturing ? 'ON' : 'OFF'}\n` +
       // The inference text result from audio record:
-      `Result: ${data.result}\n\n` + 
+      `Result: ${data.result}\n\n` +
       `Process time: ${processTime}ms\n` +
       `Recording time: ${recordingTime}ms`,
   )
@@ -220,7 +220,7 @@ In real world, we recommended to split the asset imports into another platform s
 
 The example app provide a simple UI for testing the functions.
 
-Used Whisper model: `tiny.en` in https://huggingface.co/ggerganov/whisper.cpp  
+Used Whisper model: `tiny.en` in https://huggingface.co/ggerganov/whisper.cpp
 Sample file: `jfk.wav` in https://github.com/ggerganov/whisper.cpp/tree/master/samples
 
 Please follow the [Development Workflow section of contributing guide](./CONTRIBUTING.md#development-workflow) to run the example app.

package/android/build.gradle

@@ -36,6 +36,10 @@ def reactNativeArchitectures() {
 }
 
 android {
+  def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION
+  if (agpVersion.tokenize('.')[0].toInteger() >= 7) {
+    namespace "com.rnwhisper"
+  }
   ndkVersion getExtOrDefault("ndkVersion")
   compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
 

package/android/src/main/CMakeLists.txt

@@ -2,14 +2,28 @@ cmake_minimum_required(VERSION 3.10)
 
 project(whisper.rn)
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(RNWHISPER_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../cpp)
 
 set(
     SOURCE_FILES
     ${RNWHISPER_LIB_DIR}/ggml.c
     ${RNWHISPER_LIB_DIR}/ggml-alloc.c
+    ${RNWHISPER_LIB_DIR}/ggml-backend.cpp
+    ${RNWHISPER_LIB_DIR}/ggml-backend-reg.cpp
+    ${RNWHISPER_LIB_DIR}/ggml-cpu.c
+    ${RNWHISPER_LIB_DIR}/ggml-cpu.cpp
+    ${RNWHISPER_LIB_DIR}/ggml-cpu-aarch64.cpp
+    ${RNWHISPER_LIB_DIR}/ggml-cpu-quants.c
+    ${RNWHISPER_LIB_DIR}/ggml-cpu-traits.cpp
+    ${RNWHISPER_LIB_DIR}/ggml-opt.cpp
+    ${RNWHISPER_LIB_DIR}/ggml-threading.cpp
+    ${RNWHISPER_LIB_DIR}/ggml-quants.c
+    ${RNWHISPER_LIB_DIR}/gguf.cpp
+    ${RNWHISPER_LIB_DIR}/amx/amx.cpp
+    ${RNWHISPER_LIB_DIR}/amx/mmq.cpp
     ${RNWHISPER_LIB_DIR}/whisper.cpp
+    ${RNWHISPER_LIB_DIR}/rn-audioutils.cpp
     ${RNWHISPER_LIB_DIR}/rn-whisper.cpp
     ${CMAKE_SOURCE_DIR}/jni.cpp
 )
@@ -25,12 +39,18 @@ function(build_library target_name)
 
     target_link_libraries(${target_name} ${LOG_LIB} android)
 
+    target_compile_options(${target_name} PRIVATE -DWSP_GGML_USE_CPU -DWSP_GGML_USE_CPU_AARCH64)
+
     if (${target_name} STREQUAL "whisper_v8fp16_va")
         target_compile_options(${target_name} PRIVATE -march=armv8.2-a+fp16)
     elseif (${target_name} STREQUAL "whisper_vfpv4")
         target_compile_options(${target_name} PRIVATE -mfpu=neon-vfpv4)
     endif ()
 
+    if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+        target_compile_options(${target_name} PRIVATE -DRNWHISPER_ANDROID_ENABLE_LOGGING)
+    endif ()
+
     # NOTE: If you want to debug the native code, you can uncomment if and endif
     # if (NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
 

package/android/src/main/java/com/rnwhisper/AudioUtils.java

@@ -2,112 +2,29 @@ package com.rnwhisper;
 
 import android.util.Log;
 
-import java.util.ArrayList;
-import java.lang.StringBuilder;
-import java.io.IOException;
-import java.io.FileReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileOutputStream;
-import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.ShortBuffer;
+import java.util.Base64;
+
+import java.util.Arrays;
 
 public class AudioUtils {
   private static final String NAME = "RNWhisperAudioUtils";
 
-  private static final int SAMPLE_RATE = 16000;
-
-  private static byte[] shortToByte(short[] shortInts) {
-    int j = 0;
-    int length = shortInts.length;
-    byte[] byteData = new byte[length * 2];
-    for (int i = 0; i < length; i++) {
-      byteData[j++] = (byte) (shortInts[i] >>> 8);
-      byteData[j++] = (byte) (shortInts[i] >>> 0);
-    }
-    return byteData;
-  }
-
-  public static byte[] concatShortBuffers(ArrayList<short[]> buffers) {
-    int totalLength = 0;
-    for (int i = 0; i < buffers.size(); i++) {
-      totalLength += buffers.get(i).length;
-    }
-    byte[] result = new byte[totalLength * 2];
-    int offset = 0;
-    for (int i = 0; i < buffers.size(); i++) {
-      byte[] bytes = shortToByte(buffers.get(i));
-      System.arraycopy(bytes, 0, result, offset, bytes.length);
-      offset += bytes.length;
-    }
-
-    return result;
-  }
-
-  private static byte[] removeTrailingZeros(byte[] audioData) {
-    int i = audioData.length - 1;
-    while (i >= 0 && audioData[i] == 0) {
-      --i;
-    }
-    byte[] newData = new byte[i + 1];
-    System.arraycopy(audioData, 0, newData, 0, i + 1);
-    return newData;
-  }
-
-  public static void saveWavFile(byte[] rawData, String audioOutputFile) throws IOException {
-    Log.d(NAME, "call saveWavFile");
-    rawData = removeTrailingZeros(rawData);
-    DataOutputStream output = null;
-    try {
-      output = new DataOutputStream(new FileOutputStream(audioOutputFile));
-      // WAVE header
-      // see http://ccrma.stanford.edu/courses/422/projects/WaveFormat/
-      output.writeBytes("RIFF"); // chunk id
-      output.writeInt(Integer.reverseBytes(36 + rawData.length)); // chunk size
-      output.writeBytes("WAVE"); // format
-      output.writeBytes("fmt "); // subchunk 1 id
-      output.writeInt(Integer.reverseBytes(16)); // subchunk 1 size
-      output.writeShort(Short.reverseBytes((short) 1)); // audio format (1 = PCM)
-      output.writeShort(Short.reverseBytes((short) 1)); // number of channels
-      output.writeInt(Integer.reverseBytes(SAMPLE_RATE)); // sample rate
-      output.writeInt(Integer.reverseBytes(SAMPLE_RATE * 2)); // byte rate
-      output.writeShort(Short.reverseBytes((short) 2)); // block align
-      output.writeShort(Short.reverseBytes((short) 16)); // bits per sample
-      output.writeBytes("data"); // subchunk 2 id
-      output.writeInt(Integer.reverseBytes(rawData.length)); // subchunk 2 size
-      // Audio data (conversion big endian -> little endian)
-      short[] shorts = new short[rawData.length / 2];
-      ByteBuffer.wrap(rawData).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(shorts);
-      ByteBuffer bytes = ByteBuffer.allocate(shorts.length * 2);
-      for (short s : shorts) {
-        bytes.putShort(s);
-      }
-      Log.d(NAME, "writing audio file: " + audioOutputFile);
-      output.write(bytes.array());
-    } finally {
-      if (output != null) {
-        output.close();
-      }
-    }
-  }
-
-  public static float[] decodeWaveFile(InputStream inputStream) throws IOException {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    byte[] buffer = new byte[1024];
-    int bytesRead;
-    while ((bytesRead = inputStream.read(buffer)) != -1) {
-      baos.write(buffer, 0, bytesRead);
-    }
-    ByteBuffer byteBuffer = ByteBuffer.wrap(baos.toByteArray());
+  private static float[] bufferToFloatArray(byte[] buffer, Boolean cutHeader) {
+    ByteBuffer byteBuffer = ByteBuffer.wrap(buffer);
     byteBuffer.order(ByteOrder.LITTLE_ENDIAN);
-    byteBuffer.position(44);
     ShortBuffer shortBuffer = byteBuffer.asShortBuffer();
     short[] shortArray = new short[shortBuffer.limit()];
     shortBuffer.get(shortArray);
+    if (cutHeader) {
+      shortArray = Arrays.copyOfRange(shortArray, 44, shortArray.length);
+    }
     float[] floatArray = new float[shortArray.length];
     for (int i = 0; i < shortArray.length; i++) {
       floatArray[i] = ((float) shortArray[i]) / 32767.0f;
@@ -116,4 +33,22 @@ public class AudioUtils {
     }
     return floatArray;
   }
-}
+
+  public static float[] decodeWaveFile(InputStream inputStream) throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    byte[] buffer = new byte[1024];
+    int bytesRead;
+    while ((bytesRead = inputStream.read(buffer)) != -1) {
+      baos.write(buffer, 0, bytesRead);
+    }
+    return bufferToFloatArray(baos.toByteArray(), true);
+  }
+
+  public static float[] decodeWaveData(String dataBase64) throws IOException {
+    return bufferToFloatArray(Base64.getDecoder().decode(dataBase64), true);
+  }
+
+  public static float[] decodePcmData(String dataBase64) {
+    return bufferToFloatArray(Base64.getDecoder().decode(dataBase64), false);
+  }
+}

package/android/src/main/java/com/rnwhisper/RNWhisper.java

@@ -13,11 +13,13 @@ import com.facebook.react.bridge.ReactMethod;
 import com.facebook.react.bridge.LifecycleEventListener;
 import com.facebook.react.bridge.ReadableMap;
 import com.facebook.react.bridge.WritableMap;
+import com.facebook.react.bridge.Arguments;
 
 import java.util.HashMap;
 import java.util.Random;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.InputStream;
 import java.io.PushbackInputStream;
 
 public class RNWhisper implements LifecycleEventListener {
@@ -107,51 +109,27 @@ public class RNWhisper implements LifecycleEventListener {
           promise.reject(exception);
           return;
         }
-        promise.resolve(id);
+        WritableMap result = Arguments.createMap();
+        result.putInt("contextId", id);
+        result.putBoolean("gpu", false);
+        result.putString("reasonNoGPU", "Currently not supported");
+        promise.resolve(result);
         tasks.remove(this);
       }
-    }.execute();
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
     tasks.put(task, "initContext");
   }
 
-  public void transcribeFile(double id, double jobId, String filePath, ReadableMap options, Promise promise) {
-    final WhisperContext context = contexts.get((int) id);
-    if (context == null) {
-      promise.reject("Context not found");
-      return;
-    }
-    if (context.isCapturing()) {
-      promise.reject("The context is in realtime transcribe mode");
-      return;
-    }
-    if (context.isTranscribing()) {
-      promise.reject("Context is already transcribing");
-      return;
-    }
+  private AsyncTask transcribe(WhisperContext context, double jobId, final float[] audioData, final ReadableMap options, Promise promise) {
     AsyncTask task = new AsyncTask<Void, Void, WritableMap>() {
       private Exception exception;
 
       @Override
       protected WritableMap doInBackground(Void... voids) {
         try {
-          String waveFilePath = filePath;
-
-          if (filePath.startsWith("http://") || filePath.startsWith("https://")) {
-            waveFilePath = downloader.downloadFile(filePath);
-          }
-
-          int resId = getResourceIdentifier(waveFilePath);
-          if (resId > 0) {
-            return context.transcribeInputStream(
-              (int) jobId,
-              reactContext.getResources().openRawResource(resId),
-              options
-            );
-          }
-
-          return context.transcribeInputStream(
+          return context.transcribe(
             (int) jobId,
-            new FileInputStream(new File(waveFilePath)),
+            audioData,
             options
           );
         } catch (Exception e) {
@@ -169,8 +147,67 @@ public class RNWhisper implements LifecycleEventListener {
         promise.resolve(data);
         tasks.remove(this);
       }
-    }.execute();
-    tasks.put(task, "transcribeFile-" + id);
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    return task;
+  }
+
+  public void transcribeFile(double id, double jobId, String filePathOrBase64, ReadableMap options, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    if (context.isCapturing()) {
+      promise.reject("The context is in realtime transcribe mode");
+      return;
+    }
+    if (context.isTranscribing()) {
+      promise.reject("Context is already transcribing");
+      return;
+    }
+
+    String waveFilePath = filePathOrBase64;
+    try {
+      if (filePathOrBase64.startsWith("http://") || filePathOrBase64.startsWith("https://")) {
+        waveFilePath = downloader.downloadFile(filePathOrBase64);
+      }
+
+      float[] audioData;
+      int resId = getResourceIdentifier(waveFilePath);
+      if (resId > 0) {
+        audioData = AudioUtils.decodeWaveFile(reactContext.getResources().openRawResource(resId));
+      } else if (filePathOrBase64.startsWith("data:audio/wav;base64,")) {
+        audioData = AudioUtils.decodeWaveData(filePathOrBase64);
+      } else {
+        audioData = AudioUtils.decodeWaveFile(new FileInputStream(new File(waveFilePath)));
+      }
+
+      AsyncTask task = transcribe(context, jobId, audioData, options, promise);
+      tasks.put(task, "transcribeFile-" + id);
+    } catch (Exception e) {
+      promise.reject(e);
+    }
+  }
+
+  public void transcribeData(double id, double jobId, String dataBase64, ReadableMap options, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    if (context.isCapturing()) {
+      promise.reject("The context is in realtime transcribe mode");
+      return;
+    }
+    if (context.isTranscribing()) {
+      promise.reject("Context is already transcribing");
+      return;
+    }
+
+    float[] audioData = AudioUtils.decodePcmData(dataBase64);
+    AsyncTask task = transcribe(context, jobId, audioData, options, promise);
+
+    tasks.put(task, "transcribeData-" + id);
   }
 
   public void startRealtimeTranscribe(double id, double jobId, ReadableMap options, Promise promise) {
@@ -206,7 +243,7 @@ public class RNWhisper implements LifecycleEventListener {
           context.stopTranscribe((int) jobId);
           AsyncTask completionTask = null;
           for (AsyncTask task : tasks.keySet()) {
-            if (tasks.get(task).equals("transcribeFile-" + id)) {
+            if (tasks.get(task).equals("transcribeFile-" + id) || tasks.get(task).equals("transcribeData-" + id)) {
               task.get();
               break;
             }
@@ -226,10 +263,19 @@ public class RNWhisper implements LifecycleEventListener {
         promise.resolve(null);
         tasks.remove(this);
       }
-    }.execute();
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
     tasks.put(task, "abortTranscribe-" + id);
   }
 
+  public void bench(double id, double nThreads, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    promise.resolve(context.bench((int) nThreads));
+  }
+
   public void releaseContext(double id, Promise promise) {
     final int contextId = (int) id;
     AsyncTask task = new AsyncTask<Void, Void, Void>() {
@@ -245,7 +291,7 @@ public class RNWhisper implements LifecycleEventListener {
           context.stopCurrentTranscribe();
           AsyncTask completionTask = null;
           for (AsyncTask task : tasks.keySet()) {
-            if (tasks.get(task).equals("transcribeFile-" + contextId)) {
+            if (tasks.get(task).equals("transcribeFile-" + contextId) || tasks.get(task).equals("transcribeData-" + contextId)) {
               task.get();
               break;
             }
@@ -267,7 +313,7 @@ public class RNWhisper implements LifecycleEventListener {
         promise.resolve(null);
         tasks.remove(this);
       }
-    }.execute();
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
     tasks.put(task, "releaseContext-" + id);
   }
 
@@ -294,7 +340,7 @@ public class RNWhisper implements LifecycleEventListener {
         promise.resolve(null);
         tasks.remove(this);
       }
-    }.execute();
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
     tasks.put(task, "releaseAllContexts");
   }