whisper.rn 0.3.0-rc.4 → 0.3.0-rc.6

package/android/src/newarch/java/com/rnwhisper/RNWhisperModule.java

@@ -0,0 +1,286 @@
+package com.rnwhisper;
+
+import androidx.annotation.NonNull;
+import android.util.Log;
+import android.os.Build;
+import android.os.Handler;
+import android.os.AsyncTask;
+import android.media.AudioRecord;
+
+import com.facebook.react.bridge.Promise;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.bridge.ReactMethod;
+import com.facebook.react.bridge.LifecycleEventListener;
+import com.facebook.react.bridge.ReadableMap;
+import com.facebook.react.bridge.WritableMap;
+import com.facebook.react.module.annotations.ReactModule;
+
+import java.util.HashMap;
+import java.util.Random;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.PushbackInputStream;
+
+@ReactModule(name = RNWhisperModule.NAME)
+public class RNWhisperModule extends NativeRNWhisperSpec implements LifecycleEventListener {
+  public static final String NAME = "RNWhisper";
+
+  private ReactApplicationContext reactContext;
+  private Downloader downloader;
+
+  public RNWhisperModule(ReactApplicationContext reactContext) {
+    super(reactContext);
+    reactContext.addLifecycleEventListener(this);
+    this.reactContext = reactContext;
+    this.downloader = new Downloader(reactContext);
+  }
+
+  @Override
+  @NonNull
+  public String getName() {
+    return NAME;
+  }
+
+  @Override
+  public HashMap<String, Object> getTypedExportedConstants() {
+    HashMap<String, Object> constants = new HashMap<>();
+
+    // iOS only constants, put for passing type checks
+    constants.put("useCoreML", false);
+    constants.put("coreMLAllowFallback", false);
+
+    return constants;
+  }
+
+  private HashMap<Integer, WhisperContext> contexts = new HashMap<>();
+
+  private int getResourceIdentifier(String filePath) {
+    int identifier = reactContext.getResources().getIdentifier(
+      filePath,
+      "drawable",
+      reactContext.getPackageName()
+    );
+    if (identifier == 0) {
+      identifier = reactContext.getResources().getIdentifier(
+        filePath,
+        "raw",
+        reactContext.getPackageName()
+      );
+    }
+    return identifier;
+  }
+
+  @ReactMethod
+  public void initContext(final ReadableMap options, final Promise promise) {
+    new AsyncTask<Void, Void, Integer>() {
+      private Exception exception;
+
+      @Override
+      protected Integer doInBackground(Void... voids) {
+        try {
+          String modelPath = options.getString("filePath");
+          boolean isBundleAsset = options.getBoolean("isBundleAsset");
+
+          String modelFilePath = modelPath;
+          if (!isBundleAsset && (modelPath.startsWith("http://") || modelPath.startsWith("https://"))) {
+            modelFilePath = downloader.downloadFile(modelPath);
+          }
+
+          long context;
+          int resId = getResourceIdentifier(modelFilePath);
+          if (resId > 0) {
+            context = WhisperContext.initContextWithInputStream(
+              new PushbackInputStream(reactContext.getResources().openRawResource(resId))
+            );
+          } else if (isBundleAsset) {
+            context = WhisperContext.initContextWithAsset(reactContext.getAssets(), modelFilePath);
+          } else {
+            context = WhisperContext.initContext(modelFilePath);
+          }
+          if (context == 0) {
+            throw new Exception("Failed to initialize context");
+          }
+          int id = Math.abs(new Random().nextInt());
+          WhisperContext whisperContext = new WhisperContext(id, reactContext, context);
+          contexts.put(id, whisperContext);
+          return id;
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
+
+      @Override
+      protected void onPostExecute(Integer id) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(id);
+      }
+    }.execute();
+  }
+
+  @ReactMethod
+  public void transcribeFile(double id, double jobId, String filePath, ReadableMap options, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    if (context.isCapturing()) {
+      promise.reject("The context is in realtime transcribe mode");
+      return;
+    }
+    if (context.isTranscribing()) {
+      promise.reject("Context is already transcribing");
+      return;
+    }
+    new AsyncTask<Void, Void, WritableMap>() {
+      private Exception exception;
+
+      @Override
+      protected WritableMap doInBackground(Void... voids) {
+        try {
+          String waveFilePath = filePath;
+
+          if (filePath.startsWith("http://") || filePath.startsWith("https://")) {
+            waveFilePath = downloader.downloadFile(filePath);
+          }
+
+          int resId = getResourceIdentifier(waveFilePath);
+          if (resId > 0) {
+            return context.transcribeInputStream(
+              (int) jobId,
+              reactContext.getResources().openRawResource(resId),
+              options
+            );
+          }
+
+          return context.transcribeInputStream(
+            (int) jobId,
+            new FileInputStream(new File(waveFilePath)),
+            options
+          );
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
+
+      @Override
+      protected void onPostExecute(WritableMap data) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(data);
+      }
+    }.execute();
+  }
+
+  @ReactMethod
+  public void startRealtimeTranscribe(double id, double jobId, ReadableMap options, Promise promise) {
+    final WhisperContext context = contexts.get((int) id);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    if (context.isCapturing()) {
+      promise.reject("Context is already in capturing");
+      return;
+    }
+    int state = context.startRealtimeTranscribe((int) jobId, options);
+    if (state == AudioRecord.STATE_INITIALIZED) {
+      promise.resolve(null);
+      return;
+    }
+    promise.reject("Failed to start realtime transcribe. State: " + state);
+  }
+
+  @ReactMethod
+  public void abortTranscribe(double contextId, double jobId, Promise promise) {
+    WhisperContext context = contexts.get((int) contextId);
+    if (context == null) {
+      promise.reject("Context not found");
+      return;
+    }
+    context.stopTranscribe((int) jobId);
+  }
+
+  @ReactMethod
+  public void releaseContext(double id, Promise promise) {
+    final int contextId = (int) id;
+    new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          WhisperContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context " + id + " not found");
+          }
+          context.release();
+          contexts.remove(contextId);
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+      }
+    }.execute();
+  }
+
+  @ReactMethod
+  public void releaseAllContexts(Promise promise) {
+    new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          onHostDestroy();
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+      }
+    }.execute();
+  }
+
+  @Override
+  public void onHostResume() {
+  }
+
+  @Override
+  public void onHostPause() {
+  }
+
+  @Override
+  public void onHostDestroy() {
+    WhisperContext.abortAllTranscribe();
+    for (WhisperContext context : contexts.values()) {
+      context.release();
+    }
+    contexts.clear();
+    downloader.clearCache();
+  }
+}

package/android/src/{main → oldarch}/java/com/rnwhisper/RNWhisperModule.java

@@ -18,17 +18,22 @@ import com.facebook.react.module.annotations.ReactModule;
 
 import java.util.HashMap;
 import java.util.Random;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.PushbackInputStream;
 
 @ReactModule(name = RNWhisperModule.NAME)
 public class RNWhisperModule extends ReactContextBaseJavaModule implements LifecycleEventListener {
   public static final String NAME = "RNWhisper";
 
   private ReactApplicationContext reactContext;
+  private Downloader downloader;
 
   public RNWhisperModule(ReactApplicationContext reactContext) {
     super(reactContext);
     reactContext.addLifecycleEventListener(this);
     this.reactContext = reactContext;
+    this.downloader = new Downloader(reactContext);
   }
 
   @Override
@@ -39,19 +44,48 @@ public class RNWhisperModule extends ReactContextBaseJavaModule implements Lifec
 
   private HashMap<Integer, WhisperContext> contexts = new HashMap<>();
 
+  private int getResourceIdentifier(String filePath) {
+    int identifier = reactContext.getResources().getIdentifier(
+      filePath,
+      "drawable",
+      reactContext.getPackageName()
+    );
+    if (identifier == 0) {
+      identifier = reactContext.getResources().getIdentifier(
+        filePath,
+        "raw",
+        reactContext.getPackageName()
+      );
+    }
+    return identifier;
+  }
+
   @ReactMethod
-  public void initContext(final String modelPath, final boolean isBundleAsset, final Promise promise) {
+  public void initContext(final ReadableMap options, final Promise promise) {
     new AsyncTask<Void, Void, Integer>() {
       private Exception exception;
 
       @Override
       protected Integer doInBackground(Void... voids) {
         try {
+          String modelPath = options.getString("filePath");
+          boolean isBundleAsset = options.getBoolean("isBundleAsset");
+
+          String modelFilePath = modelPath;
+          if (!isBundleAsset && (modelPath.startsWith("http://") || modelPath.startsWith("https://"))) {
+            modelFilePath = downloader.downloadFile(modelPath);
+          }
+
           long context;
-          if (isBundleAsset) {
-            context = WhisperContext.initContextWithAsset(reactContext.getAssets(), modelPath);
+          int resId = getResourceIdentifier(modelFilePath);
+          if (resId > 0) {
+            context = WhisperContext.initContextWithInputStream(
+              new PushbackInputStream(reactContext.getResources().openRawResource(resId))
+            );
+          } else if (isBundleAsset) {
+            context = WhisperContext.initContextWithAsset(reactContext.getAssets(), modelFilePath);
           } else {
-            context = WhisperContext.initContext(modelPath);
+            context = WhisperContext.initContext(modelFilePath);
           }
           if (context == 0) {
             throw new Exception("Failed to initialize context");
@@ -98,7 +132,26 @@ public class RNWhisperModule extends ReactContextBaseJavaModule implements Lifec
       @Override
       protected WritableMap doInBackground(Void... voids) {
         try {
-          return context.transcribeFile(jobId, filePath, options);
+          String waveFilePath = filePath;
+
+          if (filePath.startsWith("http://") || filePath.startsWith("https://")) {
+            waveFilePath = downloader.downloadFile(filePath);
+          }
+
+          int resId = getResourceIdentifier(waveFilePath);
+          if (resId > 0) {
+            return context.transcribeInputStream(
+              (int) jobId,
+              reactContext.getResources().openRawResource(resId),
+              options
+            );
+          }
+
+          return context.transcribeInputStream(
+            (int) jobId,
+            new FileInputStream(new File(waveFilePath)),
+            options
+          );
         } catch (Exception e) {
           exception = e;
           return null;
@@ -217,5 +270,6 @@ public class RNWhisperModule extends ReactContextBaseJavaModule implements Lifec
       context.release();
     }
     contexts.clear();
+    downloader.clearCache();
   }
 }

package/ios/RNWhisper.mm

@@ -1,29 +1,42 @@
 #import "RNWhisper.h"
 #import "RNWhisperContext.h"
+#import "RNWhisperDownloader.h"
 #include <stdlib.h>
 #include <string>
 
+#ifdef RCT_NEW_ARCH_ENABLED
+#import <RNWhisperSpec/RNWhisperSpec.h>
+#endif
+
 @implementation RNWhisper
 
 NSMutableDictionary *contexts;
 
 RCT_EXPORT_MODULE()
 
++ (BOOL)requiresMainQueueSetup
+{
+  return NO;
+}
+
 - (NSDictionary *)constantsToExport
 {
   return @{
 #if WHISPER_USE_COREML
-    @"WHISPER_USE_COREML": @YES,
+    @"useCoreML": @YES,
+#else
+    @"useCoreML": @NO,
 #endif
 #if WHISPER_COREML_ALLOW_FALLBACK
-    @"WHISPER_COREML_ALLOW_FALLBACK": @YES,
+    @"coreMLAllowFallback": @YES,
+#else
+    @"coreMLAllowFallback": @NO,
 #endif
   };
 }
 
 RCT_REMAP_METHOD(initContext,
-                 withPath:(NSString *)modelPath
-                 withBundleResource:(BOOL)isBundleAsset
+                 withOptions:(NSDictionary *)modelOptions
                  withResolver:(RCTPromiseResolveBlock)resolve
                  withRejecter:(RCTPromiseRejectBlock)reject)
 {
@@ -31,7 +44,26 @@ RCT_REMAP_METHOD(initContext,
         contexts = [[NSMutableDictionary alloc] init];
     }
 
+    NSString *modelPath = [modelOptions objectForKey:@"filePath"];
+    BOOL isBundleAsset = [[modelOptions objectForKey:@"isBundleAsset"] boolValue];
+
+    // For support debug assets in development mode
+    BOOL downloadCoreMLAssets = [[modelOptions objectForKey:@"downloadCoreMLAssets"] boolValue];
+    if (downloadCoreMLAssets) {
+        NSArray *coreMLAssets = [modelOptions objectForKey:@"coreMLAssets"];
+        // Download coreMLAssets ([{ uri, filepath }])
+        for (NSDictionary *coreMLAsset in coreMLAssets) {
+            NSString *path = coreMLAsset[@"uri"];
+            if ([path hasPrefix:@"http://"] || [path hasPrefix:@"https://"]) {
+                [RNWhisperDownloader downloadFile:path toFile:coreMLAsset[@"filepath"]];
+            }
+        }
+    }
+
     NSString *path = modelPath;
+    if ([path hasPrefix:@"http://"] || [path hasPrefix:@"https://"]) {
+        path = [RNWhisperDownloader downloadFile:path toFile:nil];
+    }
     if (isBundleAsset) {
         path = [[NSBundle mainBundle] pathForResource:modelPath ofType:nil];
     }
@@ -71,10 +103,13 @@ RCT_REMAP_METHOD(transcribeFile,
         return;
     }
 
-    NSURL *url = [NSURL fileURLWithPath:waveFilePath];
+    NSString *path = waveFilePath;
+    if ([path hasPrefix:@"http://"] || [path hasPrefix:@"https://"]) {
+        path = [RNWhisperDownloader downloadFile:path toFile:nil];
+    }
 
     int count = 0;
-    float *waveFile = [self decodeWaveFile:url count:&count];
+    float *waveFile = [self decodeWaveFile:path count:&count];
     if (waveFile == nil) {
         reject(@"whisper_error", @"Invalid file", nil);
         return;
@@ -172,8 +207,9 @@ RCT_REMAP_METHOD(releaseAllContexts,
     resolve(nil);
 }
 
-- (float *)decodeWaveFile:(NSURL*)fileURL count:(int *)count {
-    NSData *fileData = [NSData dataWithContentsOfURL:fileURL];
+- (float *)decodeWaveFile:(NSString*)filePath count:(int *)count {
+    NSURL *url = [NSURL fileURLWithPath:filePath];
+    NSData *fileData = [NSData dataWithContentsOfURL:url];
     if (fileData == nil) {
         return nil;
     }
@@ -206,6 +242,16 @@ RCT_REMAP_METHOD(releaseAllContexts,
 
     [contexts removeAllObjects];
     contexts = nil;
+
+    [RNWhisperDownloader clearCache];
 }
 
+#ifdef RCT_NEW_ARCH_ENABLED
+- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
+    (const facebook::react::ObjCTurboModule::InitParams &)params
+{
+    return std::make_shared<facebook::react::NativeRNWhisperSpecJSI>(params);
+}
+#endif
+
 @end

package/ios/RNWhisperDownloader.h

@@ -0,0 +1,8 @@
+
+
+@interface RNWhisperDownloader : NSObject
+
++ (NSString *)downloadFile:(NSString *)urlString toFile:(NSString *)path;
++ (void)clearCache;
+
+@end

package/ios/RNWhisperDownloader.m

@@ -0,0 +1,39 @@
+#import "RNWhisperDownloader.h"
+
+/**
+ * NOTE: This is simple downloader,
+ * the main purpose is supported load assets on RN Debug mode,
+ * so it's a very crude implementation.
+ * 
+ * If you want to use file download in production to load model / audio files,
+ * I would recommend using react-native-fs or expo-file-system to manage the files.
+ */
+@implementation RNWhisperDownloader
+
++ (NSString *)downloadFile:(NSString *)urlString toFile:(NSString *)path {
+  NSURL *url = [NSURL URLWithString:urlString];
+  NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:@"rnwhisper_debug_assets/"];
+  if (path) {
+    filePath = [filePath stringByAppendingPathComponent:path];
+  } else {
+    filePath = [filePath stringByAppendingPathComponent:[url lastPathComponent]];
+  }
+  
+  NSString *folderPath = [filePath stringByDeletingLastPathComponent];
+  if (![[NSFileManager defaultManager] fileExistsAtPath:folderPath]) {
+    [[NSFileManager defaultManager] createDirectoryAtPath:folderPath withIntermediateDirectories:YES attributes:nil error:nil];
+  }
+  if ([[NSFileManager defaultManager] fileExistsAtPath:filePath]) {
+    return filePath;
+  }
+  NSData *urlData = [NSData dataWithContentsOfURL:url];
+  [urlData writeToFile:filePath atomically:YES];
+  return filePath;
+}
+
++ (void)clearCache {
+  NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:@"rnwhisper_debug_assets/"];
+  [[NSFileManager defaultManager] removeItemAtPath:filePath error:nil];
+}
+
+@end

package/lib/commonjs/NativeRNWhisper.js

@@ -0,0 +1,10 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _reactNative = require("react-native");
+var _default = _reactNative.TurboModuleRegistry.get('RNWhisper');
+exports.default = _default;
+//# sourceMappingURL=NativeRNWhisper.js.map

package/lib/commonjs/NativeRNWhisper.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_reactNative","require","_default","TurboModuleRegistry","get","exports","default"],"sourceRoot":"../../src","sources":["NativeRNWhisper.ts"],"mappings":";;;;;;AACA,IAAAA,YAAA,GAAAC,OAAA;AAAkD,IAAAC,QAAA,GA6EnCC,gCAAmB,CAACC,GAAG,CAAO,WAAW,CAAC;AAAAC,OAAA,CAAAC,OAAA,GAAAJ,QAAA"}