whisper.rn 0.1.0

package/LICENSE

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) 2023 Jhen <developer@jhen.me>
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,82 @@
+# whisper.rn
+
+[![Actions Status](https://github.com/mybigday/whisper.rn/workflows/CI/badge.svg)](https://github.com/mybigday/whisper.rn/actions)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![npm](https://img.shields.io/npm/v/whisper.rn.svg)](https://www.npmjs.com/package/whisper.rn/)
+
+React Native binding of [whisper.cpp](https://github.com/ggerganov/whisper.cpp).
+
+[whisper.cpp](https://github.com/ggerganov/whisper.cpp): High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisper) automatic speech recognition (ASR) model
+
+<img src="https://user-images.githubusercontent.com/3001525/225511664-8b2ba3ec-864d-4f55-bcb0-447aef168a32.jpeg" width="500" />
+
+> Run example with release mode on iPhone 13 Pro Max
+
+## Installation
+
+```sh
+npm install whisper.rn
+```
+
+Then re-run `npx pod-install` again for iOS.
+
+## Usage
+
+```js
+import { initWhisper } from 'whisper.rn'
+
+const filePath = 'file://.../ggml.base.en.bin'
+const sampleFilePath = 'file://.../sample.wav'
+
+const whisperContext = await initWhisper({ filePath })
+
+const { result } = await whisperContext.transcribe(sampleFilePath, {
+  language: 'en',
+  // More options
+})
+// result: (The inference text result from audio file)
+```
+
+## Run with example
+
+The example app is using [react-native-fs](https://github.com/itinance/react-native-fs) to download the model file and audio file.
+
+Model: `base.en` in https://huggingface.co/datasets/ggerganov/whisper.cpp
+Sample file: `jfk.wav` in https://github.com/ggerganov/whisper.cpp/tree/master/samples
+
+For test better performance on transcribe, you can run the app in Release mode.
+  - iOS: `yarn example ios --configuration Release`
+  - Android: `yarn example android --mode release`
+
+Please follow [CONTIBUTING.md](./CONTRIBUTING.md#development-workflow) to run the example app.
+
+## Mock `whisper.rn`
+
+We have provided a mock version of `whisper.rn` for testing purpose you can use on Jest:
+
+```js
+jest.mock('whisper.rn', () => require('whisper.rn/jest/mock'))
+```
+
+## Contributing
+
+See the [contributing guide](CONTRIBUTING.md) to learn how to contribute to the repository and the development workflow.
+
+## License
+
+MIT
+
+---
+
+Made with [create-react-native-library](https://github.com/callstack/react-native-builder-bob)
+
+---
+
+<p align="center">
+  <a href="https://bricks.tools">
+    <img width="90px" src="https://avatars.githubusercontent.com/u/17320237?s=200&v=4">
+  </a>
+  <p align="center">
+    Built and maintained by <a href="https://bricks.tools">BRICKS</a>.
+  </p>
+</p>

package/android/build.gradle

@@ -0,0 +1,85 @@
+buildscript {
+  repositories {
+    google()
+    mavenCentral()
+  }
+
+  dependencies {
+    classpath "com.android.tools.build:gradle:7.2.1"
+  }
+}
+
+def isNewArchitectureEnabled() {
+  return rootProject.hasProperty("newArchEnabled") && rootProject.getProperty("newArchEnabled") == "true"
+}
+
+apply plugin: "com.android.library"
+
+
+def appProject = rootProject.allprojects.find { it.plugins.hasPlugin('com.android.application') }
+
+if (isNewArchitectureEnabled()) {
+  apply plugin: "com.facebook.react"
+}
+
+def getExtOrDefault(name) {
+  return rootProject.ext.has(name) ? rootProject.ext.get(name) : project.properties["RNWhisper_" + name]
+}
+
+def getExtOrIntegerDefault(name) {
+  return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["RNWhisper_" + name]).toInteger()
+}
+
+android {
+  ndkVersion getExtOrDefault("ndkVersion")
+  compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
+
+  defaultConfig {
+    minSdkVersion getExtOrIntegerDefault("minSdkVersion")
+    targetSdkVersion getExtOrIntegerDefault("targetSdkVersion")
+    buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", isNewArchitectureEnabled().toString()
+  }
+  externalNativeBuild {
+    externalNativeBuild {
+      ndkBuild {
+        path 'src/main/jni/whisper/Android.mk'
+      }
+    }
+  }
+  buildTypes {
+    release {
+      minifyEnabled false
+    }
+  }
+
+  lintOptions {
+    disable "GradleCompatible"
+  }
+
+  compileOptions {
+    sourceCompatibility JavaVersion.VERSION_1_8
+    targetCompatibility JavaVersion.VERSION_1_8
+  }
+
+}
+
+repositories {
+  mavenCentral()
+  google()
+}
+
+
+dependencies {
+  // For < 0.71, this will be from the local maven repo
+  // For > 0.71, this will be replaced by `com.facebook.react:react-android:$version` by react gradle plugin
+  //noinspection GradleDynamicVersion
+  implementation "com.facebook.react:react-native:+"
+}
+
+if (isNewArchitectureEnabled()) {
+  react {
+    jsRootDir = file("../src/")
+    libraryName = "RNWhisper"
+    codegenJavaPackageName = "com.rnwhisper"
+  }
+}

package/android/gradle.properties

@@ -0,0 +1,5 @@
+RNWhisper_kotlinVersion=1.7.0
+RNWhisper_minSdkVersion=23
+RNWhisper_targetSdkVersion=31
+RNWhisper_compileSdkVersion=31
+RNWhisper_ndkversion=25.1.8937393

package/android/src/main/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+          package="com.rnwhisper">
+
+</manifest>

package/android/src/main/java/com/rnwhisper/RNWhisperModule.java

@@ -0,0 +1,175 @@
+package com.rnwhisper;
+
+import androidx.annotation.NonNull;
+import android.util.Log;
+import android.os.Build;
+import android.os.Handler;
+import android.os.AsyncTask;
+
+import com.facebook.react.bridge.Promise;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.bridge.ReactContextBaseJavaModule;
+import com.facebook.react.bridge.ReactMethod;
+import com.facebook.react.bridge.LifecycleEventListener;
+import com.facebook.react.bridge.ReadableMap;
+import com.facebook.react.module.annotations.ReactModule;
+
+import java.util.HashMap;
+import java.util.Random;
+
+@ReactModule(name = RNWhisperModule.NAME)
+public class RNWhisperModule extends ReactContextBaseJavaModule implements LifecycleEventListener {
+  public static final String NAME = "RNWhisper";
+
+  private ReactApplicationContext reactContext;
+
+  public RNWhisperModule(ReactApplicationContext reactContext) {
+    super(reactContext);
+    reactContext.addLifecycleEventListener(this);
+    this.reactContext = reactContext;
+  }
+
+  @Override
+  @NonNull
+  public String getName() {
+    return NAME;
+  }
+
+  private HashMap<Integer, WhisperContext> contexts = new HashMap<>();
+
+  @ReactMethod
+  public void initContext(final String modelPath, final Promise promise) {
+    new AsyncTask<Void, Void, Integer>() {
+      private Exception exception;
+
+      @Override
+      protected Integer doInBackground(Void... voids) {
+        try {
+          long context = WhisperContext.initContext(modelPath);
+          if (context == 0) {
+            throw new Exception("Failed to initialize context");
+          }
+          int id = Math.abs(new Random().nextInt());
+          WhisperContext whisperContext = new WhisperContext(context);
+          contexts.put(id, whisperContext);
+          return id;
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
+
+      @Override
+      protected void onPostExecute(Integer id) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(id);
+      }
+    }.execute();
+  }
+
+  @ReactMethod
+  public void transcribe(int id, String filePath, ReadableMap options, Promise promise) {
+    new AsyncTask<Void, Void, String>() {
+      private Exception exception;
+
+      @Override
+      protected String doInBackground(Void... voids) {
+        try {
+          WhisperContext context = contexts.get(id);
+          if (context == null) {
+            throw new Exception("Context " + id + " not found");
+          }
+          return context.transcribe(filePath, options);
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
+
+      @Override
+      protected void onPostExecute(String result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(result);
+      }
+    }.execute();
+  }
+
+  @ReactMethod
+  public void releaseContext(int id, Promise promise) {
+    new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          WhisperContext context = contexts.get(id);
+          if (context == null) {
+            throw new Exception("Context " + id + " not found");
+          }
+          context.release();
+          contexts.remove(id);
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+      }
+    }.execute();
+  }
+
+  @ReactMethod
+  public void releaseAllContexts(Promise promise) {
+    new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          onHostDestroy();
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+      }
+    }.execute();
+  }
+
+  @Override
+  public void onHostResume() {
+  }
+
+  @Override
+  public void onHostPause() {
+  }
+
+  @Override
+  public void onHostDestroy() {
+    for (WhisperContext context : contexts.values()) {
+      context.release();
+    }
+    contexts.clear();
+  }
+}

package/android/src/main/java/com/rnwhisper/RNWhisperPackage.java

@@ -0,0 +1,28 @@
+package com.rnwhisper;
+
+import androidx.annotation.NonNull;
+
+import com.facebook.react.ReactPackage;
+import com.facebook.react.bridge.NativeModule;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.uimanager.ViewManager;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class RNWhisperPackage implements ReactPackage {
+  @NonNull
+  @Override
+  public List<NativeModule> createNativeModules(@NonNull ReactApplicationContext reactContext) {
+    List<NativeModule> modules = new ArrayList<>();
+    modules.add(new RNWhisperModule(reactContext));
+    return modules;
+  }
+
+  @NonNull
+  @Override
+  public List<ViewManager> createViewManagers(@NonNull ReactApplicationContext reactContext) {
+    return Collections.emptyList();
+  }
+}

package/android/src/main/java/com/rnwhisper/WhisperContext.java

@@ -0,0 +1,186 @@
+package com.rnwhisper;
+
+import com.facebook.react.bridge.ReadableMap;
+
+import android.util.Log;
+import android.os.Build;
+import android.content.res.AssetManager;
+
+import java.util.Random;
+import java.lang.StringBuilder;
+import java.io.File;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.FileReader;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.ShortBuffer;
+
+public class WhisperContext {
+  public static final String NAME = "RNWhisperContext";
+  private long context;
+
+  public WhisperContext(long context) {
+    this.context = context;
+  }
+
+  public String transcribe(final String filePath, final ReadableMap options) throws IOException, Exception {
+    int code = fullTranscribe(
+      context,
+      decodeWaveFile(new File(filePath)),
+      // jint n_threads,
+      options.hasKey("maxThreads") ? options.getInt("maxThreads") : -1,
+      // jint max_context,
+      options.hasKey("maxContext") ? options.getInt("maxContext") : -1,
+      // jint max_len,
+      options.hasKey("maxLen") ? options.getInt("maxLen") : -1,
+      // jint offset,
+      options.hasKey("offset") ? options.getInt("offset") : -1,
+      // jint duration,
+      options.hasKey("duration") ? options.getInt("duration") : -1,
+      // jint word_thold,
+      options.hasKey("wordThold") ? options.getInt("wordThold") : -1,
+      // jfloat temperature,
+      options.hasKey("temperature") ? (float) options.getDouble("temperature") : -1.0f,
+      // jfloat temperature_inc,
+      options.hasKey("temperatureInc") ? (float) options.getDouble("temperatureInc") : -1.0f,
+      // jint beam_size,
+      options.hasKey("beamSize") ? options.getInt("beamSize") : -1,
+      // jint best_of,
+      options.hasKey("bestOf") ? options.getInt("bestOf") : -1,
+      // jboolean speed_up,
+      options.hasKey("speedUp") ? options.getBoolean("speedUp") : false,
+      // jboolean translate,
+      options.hasKey("translate") ? options.getBoolean("translate") : false,
+      // jstring language,
+      options.hasKey("language") ? options.getString("language") : "auto"
+    );
+    if (code != 0) {
+      throw new Exception("Transcription failed with code " + code);
+    }
+    Integer count = getTextSegmentCount(context);
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < count; i++) {
+      builder.append(getTextSegment(context, i));
+    }
+    return builder.toString();
+  }
+
+  public void release() {
+    freeContext(context);
+  }
+
+  public static float[] decodeWaveFile(File file) throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    try (InputStream inputStream = new FileInputStream(file)) {
+      byte[] buffer = new byte[1024];
+      int bytesRead;
+      while ((bytesRead = inputStream.read(buffer)) != -1) {
+        baos.write(buffer, 0, bytesRead);
+      }
+    }
+    ByteBuffer byteBuffer = ByteBuffer.wrap(baos.toByteArray());
+    byteBuffer.order(ByteOrder.LITTLE_ENDIAN);
+    byteBuffer.position(44);
+    ShortBuffer shortBuffer = byteBuffer.asShortBuffer();
+    short[] shortArray = new short[shortBuffer.limit()];
+    shortBuffer.get(shortArray);
+    float[] floatArray = new float[shortArray.length];
+    for (int i = 0; i < shortArray.length; i++) {
+      floatArray[i] = ((float) shortArray[i]) / 32767.0f;
+      floatArray[i] = Math.max(floatArray[i], -1f);
+      floatArray[i] = Math.min(floatArray[i], 1f);
+    }
+    return floatArray;
+  }
+
+  static {
+    Log.d(NAME, "Primary ABI: " + Build.SUPPORTED_ABIS[0]);
+    boolean loadVfpv4 = false;
+    boolean loadV8fp16 = false;
+    if (isArmeabiV7a()) {
+      // armeabi-v7a needs runtime detection support
+      String cpuInfo = cpuInfo();
+      if (cpuInfo != null) {
+        Log.d(NAME, "CPU info: " + cpuInfo);
+        if (cpuInfo.contains("vfpv4")) {
+          Log.d(NAME, "CPU supports vfpv4");
+          loadVfpv4 = true;
+        }
+      }
+    } else if (isArmeabiV8a()) {
+      // ARMv8.2a needs runtime detection support
+      String cpuInfo = cpuInfo();
+      if (cpuInfo != null) {
+        Log.d(NAME, "CPU info: " + cpuInfo);
+        if (cpuInfo.contains("fphp")) {
+          Log.d(NAME, "CPU supports fp16 arithmetic");
+          loadV8fp16 = true;
+        }
+      }
+    }
+
+    if (loadVfpv4) {
+      Log.d(NAME, "Loading libwhisper_vfpv4.so");
+      System.loadLibrary("whisper_vfpv4");
+    } else if (loadV8fp16) {
+      Log.d(NAME, "Loading libwhisper_v8fp16_va.so");
+      System.loadLibrary("whisper_v8fp16_va");
+    } else {
+      Log.d(NAME, "Loading libwhisper.so");
+      System.loadLibrary("whisper");
+    }
+  }
+
+  private static boolean isArmeabiV7a() {
+    return Build.SUPPORTED_ABIS[0].equals("armeabi-v7a");
+  }
+
+  private static boolean isArmeabiV8a() {
+    return Build.SUPPORTED_ABIS[0].equals("arm64-v8a");
+  }
+
+  private static String cpuInfo() {
+    File file = new File("/proc/cpuinfo");
+    StringBuilder stringBuilder = new StringBuilder();
+    try {
+      BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
+      String line;
+      while ((line = bufferedReader.readLine()) != null) {
+          stringBuilder.append(line);
+      }
+      bufferedReader.close();
+      return stringBuilder.toString();
+    } catch (IOException e) {
+      Log.w(NAME, "Couldn't read /proc/cpuinfo", e);
+      return null;
+    }
+  }
+
+  protected static native long initContext(String modelPath);
+  protected static native int fullTranscribe(
+    long context,
+    float[] audio_data,
+    int n_threads,
+    int max_context,
+    int max_len,
+    int offset,
+    int duration,
+    int word_thold,
+    float temperature,
+    float temperature_inc,
+    int beam_size,
+    int best_of,
+    boolean speed_up,
+    boolean translate,
+    String language
+  );
+  protected static native int getTextSegmentCount(long context);
+  protected static native String getTextSegment(long context, int index);
+  protected static native void freeContext(long contextPtr);
+}

package/android/src/main/jni/whisper/Android.mk

@@ -0,0 +1,26 @@
+LOCAL_PATH := $(call my-dir)
+include $(CLEAR_VARS)
+LOCAL_MODULE    := libwhisper
+include $(LOCAL_PATH)/Whisper.mk
+include $(BUILD_SHARED_LIBRARY)
+
+ifeq ($(TARGET_ARCH_ABI),armeabi-v7a)
+	include $(CLEAR_VARS)
+	LOCAL_MODULE    := libwhisper_vfpv4
+	include $(LOCAL_PATH)/Whisper.mk
+	# Allow building NEON FMA code.
+	# https://android.googlesource.com/platform/ndk/+/master/sources/android/cpufeatures/cpu-features.h
+	LOCAL_CFLAGS += -mfpu=neon-vfpv4
+	include $(BUILD_SHARED_LIBRARY)
+endif
+
+ifeq ($(TARGET_ARCH_ABI),arm64-v8a)
+	include $(CLEAR_VARS)
+	LOCAL_MODULE    := libwhisper_v8fp16_va
+	include $(LOCAL_PATH)/Whisper.mk
+	# Allow building NEON FMA code.
+	# https://android.googlesource.com/platform/ndk/+/master/sources/android/cpufeatures/cpu-features.h
+	LOCAL_CFLAGS += -march=armv8.2-a+fp16
+	include $(BUILD_SHARED_LIBRARY)
+endif
+

package/android/src/main/jni/whisper/Application.mk

@@ -0,0 +1 @@
+APP_STL := c++_static

package/android/src/main/jni/whisper/Whisper.mk

@@ -0,0 +1,18 @@
+WHISPER_LIB_DIR := $(LOCAL_PATH)/../../../../../cpp
+LOCAL_LDLIBS    := -landroid -llog
+
+# Make the final output library smaller by only keeping the symbols referenced from the app.
+ifneq ($(APP_OPTIM),debug)
+    LOCAL_CFLAGS += -O3
+    LOCAL_CFLAGS += -fvisibility=hidden -fvisibility-inlines-hidden
+    LOCAL_CFLAGS += -ffunction-sections -fdata-sections
+    LOCAL_LDFLAGS += -Wl,--gc-sections
+    LOCAL_LDFLAGS += -Wl,--exclude-libs,ALL
+    LOCAL_LDFLAGS += -flto
+endif
+
+LOCAL_CFLAGS    += -DSTDC_HEADERS -std=c11 -I $(WHISPER_LIB_DIR)
+LOCAL_CPPFLAGS  += -std=c++11
+LOCAL_SRC_FILES := $(WHISPER_LIB_DIR)/ggml.c \
+                   $(WHISPER_LIB_DIR)/whisper.cpp \
+                   $(LOCAL_PATH)/jni.c