react-native-executorch 0.4.8 → 0.5.0

package/android/src/main/java/com/swmansion/rnexecutorch/utils/TensorUtils.kt

@@ -1,103 +0,0 @@
-package com.swmansion.rnexecutorch.utils
-
-import android.graphics.Bitmap
-import android.graphics.Color
-import com.facebook.react.bridge.ReadableArray
-import org.pytorch.executorch.EValue
-import org.pytorch.executorch.Tensor
-import java.nio.FloatBuffer
-
-class TensorUtils {
-  companion object {
-    fun getExecutorchInput(
-      input: ReadableArray,
-      shape: LongArray,
-      type: Int,
-    ): EValue {
-      try {
-        when (type) {
-          1 -> {
-            val inputTensor = Tensor.fromBlob(ArrayUtils.createByteArray(input), shape)
-            return EValue.from(inputTensor)
-          }
-          3 -> {
-            val inputTensor = Tensor.fromBlob(ArrayUtils.createIntArray(input), shape)
-            return EValue.from(inputTensor)
-          }
-          4 -> {
-            val inputTensor = Tensor.fromBlob(ArrayUtils.createLongArray(input), shape)
-            return EValue.from(inputTensor)
-          }
-          6 -> {
-            val inputTensor = Tensor.fromBlob(ArrayUtils.createFloatArray(input), shape)
-            return EValue.from(inputTensor)
-          }
-          7 -> {
-            val inputTensor = Tensor.fromBlob(ArrayUtils.createDoubleArray(input), shape)
-            return EValue.from(inputTensor)
-          }
-
-          else -> {
-            throw IllegalArgumentException("Invalid input type: $type")
-          }
-        }
-      } catch (e: IllegalArgumentException) {
-        throw e
-      }
-    }
-
-    fun float32TensorToBitmap(tensor: Tensor): Bitmap {
-      val shape = tensor.shape() // Assuming the tensor shape is [1, 3, H, W]
-      val height = shape[2].toInt()
-      val width = shape[3].toInt()
-
-      val floatArray = tensor.dataAsFloatArray
-
-      val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
-      val pixels = IntArray(width * height)
-
-      val offsetG = height * width
-      val offsetB = 2 * height * width
-
-      for (y in 0 until height) {
-        for (x in 0 until width) {
-          val r = Math.round(floatArray[y * width + x] * 255.0f)
-          val g = Math.round(floatArray[offsetG + y * width + x] * 255.0f)
-          val b = Math.round(floatArray[offsetB + y * width + x] * 255.0f)
-          pixels[y * width + x] = (0xFF shl 24) or (r shl 16) or (g shl 8) or b
-        }
-      }
-
-      bitmap.setPixels(pixels, 0, width, 0, 0, width, height)
-      return bitmap
-    }
-
-    fun bitmapToFloat32Tensor(bitmap: Bitmap): Tensor {
-      val height = bitmap.height
-      val width = bitmap.width
-      val floatBuffer = Tensor.allocateFloatBuffer(3 * width * height)
-      bitmapToFloatBuffer(bitmap, floatBuffer)
-      return Tensor.fromBlob(floatBuffer, longArrayOf(1, 3, height.toLong(), width.toLong()))
-    }
-
-    private fun bitmapToFloatBuffer(
-      bitmap: Bitmap,
-      outBuffer: FloatBuffer,
-    ) {
-      val pixelsCount = bitmap.height * bitmap.width
-      val pixels = IntArray(pixelsCount)
-      bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height)
-      val offsetG = pixelsCount
-      val offsetB = 2 * pixelsCount
-      for (i in 0 until pixelsCount) {
-        val c = pixels[i]
-        val r = Color.red(c) / 255.0f
-        val g = Color.green(c) / 255.0f
-        val b = Color.blue(c) / 255.0f
-        outBuffer.put(i, r)
-        outBuffer.put(offsetG + i, g)
-        outBuffer.put(offsetB + i, b)
-      }
-    }
-  }
-}

package/ios/RnExecutorch/Classification.h

@@ -1,5 +0,0 @@
-#import <RnExecutorchSpec/RnExecutorchSpec.h>
-
-@interface Classification : NSObject <NativeClassificationSpec>
-
-@end

package/ios/RnExecutorch/Classification.mm

@@ -1,54 +0,0 @@
-#import "Classification.h"
-#import "ImageProcessor.h"
-#import "models/classification/ClassificationModel.h"
-
-@implementation Classification {
-  ClassificationModel *model;
-}
-
-RCT_EXPORT_MODULE()
-
-- (void)releaseResources {
-  model = nil;
-}
-
-- (void)loadModule:(NSString *)modelSource
-           resolve:(RCTPromiseResolveBlock)resolve
-            reject:(RCTPromiseRejectBlock)reject {
-  model = [[ClassificationModel alloc] init];
-
-  NSNumber *errorCode = [model loadModel:modelSource];
-  if ([errorCode intValue] != 0) {
-    [self releaseResources];
-    reject(@"init_module_error",
-           [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
-           nil);
-    return;
-  }
-
-  resolve(@0);
-}
-
-- (void)forward:(NSString *)input
-        resolve:(RCTPromiseResolveBlock)resolve
-         reject:(RCTPromiseRejectBlock)reject {
-  @try {
-    cv::Mat image = [ImageProcessor readImage:input];
-    NSDictionary *result = [model runModel:image];
-
-    resolve(result);
-    return;
-  } @catch (NSException *exception) {
-    NSLog(@"An exception occurred: %@, %@", exception.name, exception.reason);
-    reject(@"forward_error",
-           [NSString stringWithFormat:@"%@", exception.reason], nil);
-    return;
-  }
-}
-
-- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
-    (const facebook::react::ObjCTurboModule::InitParams &)params {
-  return std::make_shared<facebook::react::NativeClassificationSpecJSI>(params);
-}
-
-@end

package/ios/RnExecutorch/ETModule.h

@@ -1,5 +0,0 @@
-#import <RnExecutorchSpec/RnExecutorchSpec.h>
-
-@interface ETModule : NSObject <NativeETModuleSpec>
-
-@end

package/ios/RnExecutorch/ETModule.mm

@@ -1,75 +0,0 @@
-#import "ETModule.h"
-#import <ExecutorchLib/ETModel.h>
-
-@implementation ETModule {
-  ETModel *module;
-}
-
-RCT_EXPORT_MODULE()
-
-- (void)releaseResources {
-  module = nil;
-}
-
-- (void)loadModule:(NSString *)modelSource
-           resolve:(RCTPromiseResolveBlock)resolve
-            reject:(RCTPromiseRejectBlock)reject {
-  if (!module) {
-    module = [[ETModel alloc] init];
-  }
-
-  NSNumber *result = [self->module loadModel:modelSource];
-  if ([result intValue] != 0) {
-    [self releaseResources];
-    NSError *error =
-        [NSError errorWithDomain:@"ETModuleErrorDomain"
-                            code:[result intValue]
-                        userInfo:@{
-                          NSLocalizedDescriptionKey : [NSString
-                              stringWithFormat:@"%ld", (long)[result longValue]]
-                        }];
-
-    reject(@"init_module_error", error.localizedDescription, error);
-    return;
-  }
-
-  resolve(result);
-}
-
-- (void)forward:(NSArray *)inputs
-         shapes:(NSArray *)shapes
-     inputTypes:(NSArray *)inputTypes
-        resolve:(RCTPromiseResolveBlock)resolve
-         reject:(RCTPromiseRejectBlock)reject {
-  @try {
-    NSArray *result = [module forward:inputs
-                               shapes:shapes
-                           inputTypes:inputTypes];
-    resolve(result);
-  } @catch (NSException *exception) {
-    NSLog(@"An exception occurred in forward: %@, %@", exception.name,
-          exception.reason);
-    reject(
-        @"forward_error",
-        [NSString stringWithFormat:@"An error occurred: %@", exception.reason],
-        nil);
-  }
-}
-
-- (void)loadMethod:(NSString *)methodName
-           resolve:(RCTPromiseResolveBlock)resolve
-            reject:(RCTPromiseRejectBlock)reject {
-  NSNumber *result = [module loadMethod:methodName];
-  if ([result intValue] == 0) {
-    resolve(result);
-    return;
-  }
-  reject(@"load_method_error", [result stringValue], nil);
-}
-
-- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
-    (const facebook::react::ObjCTurboModule::InitParams &)params {
-  return std::make_shared<facebook::react::NativeETModuleSpecJSI>(params);
-}
-
-@end

package/ios/RnExecutorch/ImageSegmentation.h

@@ -1,5 +0,0 @@
-#import <RnExecutorchSpec/RnExecutorchSpec.h>
-
-@interface ImageSegmentation : NSObject <NativeImageSegmentationSpec>
-
-@end

package/ios/RnExecutorch/ImageSegmentation.mm

@@ -1,60 +0,0 @@
-#import "ImageSegmentation.h"
-#import "ImageProcessor.h"
-#import "models/image_segmentation/ImageSegmentationModel.h"
-
-@implementation ImageSegmentation {
-  ImageSegmentationModel *model;
-}
-
-RCT_EXPORT_MODULE()
-
-- (void)releaseResources {
-  model = nil;
-}
-
-- (void)loadModule:(NSString *)modelSource
-           resolve:(RCTPromiseResolveBlock)resolve
-            reject:(RCTPromiseRejectBlock)reject {
-  model = [[ImageSegmentationModel alloc] init];
-
-  NSNumber *errorCode = [model loadModel:modelSource];
-  if ([errorCode intValue] != 0) {
-    [self releaseResources];
-    reject(@"init_module_error",
-           [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
-           nil);
-    return;
-  }
-
-  resolve(@0);
-}
-
-- (void)forward:(NSString *)input
-    classesOfInterest:(NSArray *)classesOfInterest
-               resize:(BOOL)resize
-              resolve:(RCTPromiseResolveBlock)resolve
-               reject:(RCTPromiseRejectBlock)reject {
-
-  @try {
-    cv::Mat image = [ImageProcessor readImage:input];
-    NSDictionary *result = [model runModel:image
-                             returnClasses:classesOfInterest
-                                    resize:resize];
-
-    resolve(result);
-    return;
-  } @catch (NSException *exception) {
-    NSLog(@"An exception occurred: %@, %@", exception.name, exception.reason);
-    reject(@"forward_error",
-           [NSString stringWithFormat:@"%@", exception.reason], nil);
-    return;
-  }
-}
-
-- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
-    (const facebook::react::ObjCTurboModule::InitParams &)params {
-  return std::make_shared<facebook::react::NativeImageSegmentationSpecJSI>(
-      params);
-}
-
-@end

package/ios/RnExecutorch/OCR.h

@@ -1,5 +0,0 @@
-#import <RnExecutorchSpec/RnExecutorchSpec.h>
-
-@interface OCR : NSObject <NativeOCRSpec>
-
-@end

package/ios/RnExecutorch/OCR.mm

@@ -1,96 +0,0 @@
-#import "OCR.h"
-#import "models/ocr/Detector.h"
-#import "models/ocr/RecognitionHandler.h"
-#import "models/ocr/utils/Constants.h"
-#import "utils/ImageProcessor.h"
-
-@implementation OCR {
-  Detector *detector;
-  RecognitionHandler *recognitionHandler;
-}
-
-RCT_EXPORT_MODULE()
-
-- (void)releaseResources {
-  detector = nil;
-  recognitionHandler = nil;
-}
-
-- (void)loadModule:(NSString *)detectorSource
-     recognizerSourceLarge:(NSString *)recognizerSourceLarge
-    recognizerSourceMedium:(NSString *)recognizerSourceMedium
-     recognizerSourceSmall:(NSString *)recognizerSourceSmall
-                   symbols:(NSString *)symbols
-                   resolve:(RCTPromiseResolveBlock)resolve
-                    reject:(RCTPromiseRejectBlock)reject {
-  detector = [[Detector alloc] init];
-  NSNumber *errorCode = [detector loadModel:detectorSource];
-  if ([errorCode intValue] != 0) {
-    [self releaseResources];
-    NSError *error = [NSError
-        errorWithDomain:@"OCRErrorDomain"
-                   code:[errorCode intValue]
-               userInfo:@{
-                 NSLocalizedDescriptionKey : [NSString
-                     stringWithFormat:@"%ld", (long)[errorCode longValue]]
-               }];
-    reject(@"init_module_error", @"Failed to initialize detector module",
-           error);
-    return;
-  }
-
-  recognitionHandler = [[RecognitionHandler alloc] initWithSymbols:symbols];
-  errorCode = [recognitionHandler loadRecognizers:recognizerSourceLarge
-                             mediumRecognizerPath:recognizerSourceMedium
-                              smallRecognizerPath:recognizerSourceSmall];
-  if ([errorCode intValue] != 0) {
-    [self releaseResources];
-    NSError *error = [NSError
-        errorWithDomain:@"OCRErrorDomain"
-                   code:[errorCode intValue]
-               userInfo:@{
-                 NSLocalizedDescriptionKey : [NSString
-                     stringWithFormat:@"%ld", (long)[errorCode longValue]]
-               }];
-    reject(@"init_recognizer_error",
-           @"Failed to initialize one or more recognizer models", error);
-    return;
-  }
-
-  resolve(@0);
-}
-
-- (void)forward:(NSString *)input
-        resolve:(RCTPromiseResolveBlock)resolve
-         reject:(RCTPromiseRejectBlock)reject {
-  /*
-   The OCR consists of two phases:
-   1. Detection - detecting text regions in the image, the result of this phase
-   is a list of bounding boxes.
-   2. Recognition - recognizing the text in the bounding boxes, the result is a
-   list of strings and corresponding confidence scores.
-
-   Recognition uses three models, each model is resposible for recognizing text
-   of different sizes (e.g. large - 512x64, medium - 256x64, small - 128x64).
-   */
-  @try {
-    cv::Mat image = [ImageProcessor readImage:input];
-    NSArray *result = [detector runModel:image];
-    cv::cvtColor(image, image, cv::COLOR_BGR2GRAY);
-    result = [self->recognitionHandler recognize:result
-                                         imgGray:image
-                                    desiredWidth:recognizerImageSize
-                                   desiredHeight:recognizerImageSize];
-    resolve(result);
-  } @catch (NSException *exception) {
-    reject(@"forward_error",
-           [NSString stringWithFormat:@"%@", exception.reason], nil);
-  }
-}
-
-- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
-    (const facebook::react::ObjCTurboModule::InitParams &)params {
-  return std::make_shared<facebook::react::NativeOCRSpecJSI>(params);
-}
-
-@end

package/ios/RnExecutorch/ObjectDetection.h

@@ -1,5 +0,0 @@
-#import <RnExecutorchSpec/RnExecutorchSpec.h>
-
-@interface ObjectDetection : NSObject <NativeObjectDetectionSpec>
-
-@end

package/ios/RnExecutorch/ObjectDetection.mm

@@ -1,56 +0,0 @@
-#import "ObjectDetection.h"
-#import "models/object_detection/SSDLiteLargeModel.hpp"
-#import "utils/ImageProcessor.h"
-
-@implementation ObjectDetection {
-  SSDLiteLargeModel *model;
-}
-
-RCT_EXPORT_MODULE()
-
-- (void)releaseResources {
-  model = nil;
-}
-
-- (void)loadModule:(NSString *)modelSource
-           resolve:(RCTPromiseResolveBlock)resolve
-            reject:(RCTPromiseRejectBlock)reject {
-  model = [[SSDLiteLargeModel alloc] init];
-
-  NSNumber *errorCode = [model loadModel:modelSource];
-  if ([errorCode intValue] != 0) {
-    [self releaseResources];
-    NSError *error = [NSError
-        errorWithDomain:@"StyleTransferErrorDomain"
-                   code:[errorCode intValue]
-               userInfo:@{
-                 NSLocalizedDescriptionKey : [NSString
-                     stringWithFormat:@"%ld", (long)[errorCode longValue]]
-               }];
-    reject(@"init_module_error", error.localizedDescription, error);
-    return;
-  }
-
-  resolve(@0);
-}
-
-- (void)forward:(NSString *)input
-        resolve:(RCTPromiseResolveBlock)resolve
-         reject:(RCTPromiseRejectBlock)reject {
-  @try {
-    cv::Mat image = [ImageProcessor readImage:input];
-    NSArray *result = [model runModel:image];
-    resolve(result);
-  } @catch (NSException *exception) {
-    reject(@"forward_error",
-           [NSString stringWithFormat:@"%@", exception.reason], nil);
-  }
-}
-
-- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
-    (const facebook::react::ObjCTurboModule::InitParams &)params {
-  return std::make_shared<facebook::react::NativeObjectDetectionSpecJSI>(
-      params);
-}
-
-@end

package/ios/RnExecutorch/SpeechToText.h

@@ -1,5 +0,0 @@
-#import <RnExecutorchSpec/RnExecutorchSpec.h>
-
-@interface SpeechToText : NativeSpeechToTextSpecBase <NativeSpeechToTextSpec>
-
-@end

package/ios/RnExecutorch/SpeechToText.mm

@@ -1,125 +0,0 @@
-#import "SpeechToText.h"
-#import "models/stt/Moonshine.hpp"
-#import "models/stt/Whisper.hpp"
-
-@implementation SpeechToText {
-  Whisper *whisper;
-  Moonshine *moonshine;
-}
-
-RCT_EXPORT_MODULE()
-
-- (void)releaseResources {
-  whisper = nil;
-  moonshine = nil;
-}
-
-- (void)generate:(NSArray *)waveform
-         resolve:(RCTPromiseResolveBlock)resolve
-          reject:(RCTPromiseRejectBlock)reject {
-  @try {
-    SpeechToTextBaseModel *model =
-        self->whisper ? self->whisper : self->moonshine;
-
-    dispatch_async(
-        dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
-          NSArray *encodingResult = [model encode:waveform];
-
-          NSMutableArray *mutablePrevTokens =
-              [NSMutableArray arrayWithObject:model->START_TOKEN];
-          NSNumber *currentSeqLen = @0;
-          while ([currentSeqLen unsignedIntegerValue] < model -> maxSeqLen) {
-            NSArray *result = [model decode:mutablePrevTokens
-                     encoderLastHiddenState:encodingResult];
-            if (!result || result.count == 0) {
-              reject(@"forward_error", @"Decoder returned an empty result.",
-                     nil);
-              return;
-            }
-            NSNumber *predictedToken = result[0];
-            [mutablePrevTokens addObject:predictedToken];
-            [self emitOnToken:predictedToken];
-            if ([predictedToken isEqualToNumber:model->EOS_TOKEN]) {
-              break;
-            }
-            currentSeqLen = @([currentSeqLen unsignedIntegerValue] + 1);
-          }
-          resolve(mutablePrevTokens);
-        });
-  } @catch (NSException *exception) {
-    NSLog(@"Exception caught before dispatch: %@, Reason: %@", exception.name,
-          exception.reason);
-    reject(@"exception_before_dispatch", exception.reason, nil);
-  }
-}
-
-- (void)loadModule:(NSString *)modelName
-      modelSources:(NSArray *)modelSources
-           resolve:(RCTPromiseResolveBlock)resolve
-            reject:(RCTPromiseRejectBlock)reject {
-
-  if ([modelSources count] != 2) {
-    reject(@"corrupted model sources", nil, nil);
-    return;
-  }
-  if (![@[ @"moonshine", @"whisper" ] containsObject:modelName]) {
-    reject(@"invalid_model_identifier", nil, nil);
-    return;
-  }
-
-  SpeechToTextBaseModel *model;
-  if ([modelName isEqualToString:@"moonshine"]) {
-    moonshine = [[Moonshine alloc] init];
-    model = moonshine;
-  }
-  if ([modelName isEqualToString:@"whisper"]) {
-    whisper = [[Whisper alloc] init];
-    model = whisper;
-  }
-
-  @try {
-    [model loadModules:modelSources];
-    resolve(@(0));
-  } @catch (NSException *exception) {
-    [self releaseResources];
-    reject(@"init_decoder_error",
-           [NSString stringWithFormat:@"%@", exception.reason], nil);
-  }
-}
-
-- (void)encode:(NSArray *)input
-       resolve:(RCTPromiseResolveBlock)resolve
-        reject:(RCTPromiseRejectBlock)reject {
-  SpeechToTextBaseModel *model =
-      self->whisper ? self->whisper : self->moonshine;
-  @try {
-    NSArray *encodingResult = [model encode:input];
-    resolve(encodingResult);
-  } @catch (NSException *exception) {
-    reject(@"forward_error",
-           [NSString stringWithFormat:@"%@", exception.reason], nil);
-  }
-}
-
-- (void)decode:(NSArray *)prevTokens
-    encoderOutput:(NSArray *)encoderOutput
-          resolve:(RCTPromiseResolveBlock)resolve
-           reject:(RCTPromiseRejectBlock)reject {
-  SpeechToTextBaseModel *model =
-      self->whisper ? self->whisper : self->moonshine;
-  @try {
-    NSArray *token = [model decode:prevTokens
-            encoderLastHiddenState:encoderOutput];
-    resolve(token);
-  } @catch (NSException *exception) {
-    reject(@"forward_error",
-           [NSString stringWithFormat:@"%@", exception.reason], nil);
-  }
-}
-
-- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
-    (const facebook::react::ObjCTurboModule::InitParams &)params {
-  return std::make_shared<facebook::react::NativeSpeechToTextSpecJSI>(params);
-}
-
-@end

package/ios/RnExecutorch/StyleTransfer.h

@@ -1,5 +0,0 @@
-#import <RnExecutorchSpec/RnExecutorchSpec.h>
-
-@interface StyleTransfer : NSObject <NativeStyleTransferSpec>
-
-@end

package/ios/RnExecutorch/StyleTransfer.mm

@@ -1,55 +0,0 @@
-#import "StyleTransfer.h"
-#import "ImageProcessor.h"
-#import "models/style_transfer/StyleTransferModel.h"
-
-@implementation StyleTransfer {
-  StyleTransferModel *model;
-}
-
-RCT_EXPORT_MODULE()
-
-- (void)releaseResources {
-  model = nil;
-}
-
-- (void)loadModule:(NSString *)modelSource
-           resolve:(RCTPromiseResolveBlock)resolve
-            reject:(RCTPromiseRejectBlock)reject {
-  model = [[StyleTransferModel alloc] init];
-
-  NSNumber *errorCode = [model loadModel:modelSource];
-  if ([errorCode intValue] != 0) {
-    [self releaseResources];
-    reject(@"init_module_error",
-           [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
-           nil);
-    return;
-  }
-
-  resolve(@0);
-}
-
-- (void)forward:(NSString *)input
-        resolve:(RCTPromiseResolveBlock)resolve
-         reject:(RCTPromiseRejectBlock)reject {
-  @try {
-    cv::Mat image = [ImageProcessor readImage:input];
-    cv::Mat resultImage = [model runModel:image];
-
-    NSString *tempFilePath = [ImageProcessor saveToTempFile:resultImage];
-    resolve(tempFilePath);
-    return;
-  } @catch (NSException *exception) {
-    NSLog(@"An exception occurred: %@, %@", exception.name, exception.reason);
-    reject(@"forward_error",
-           [NSString stringWithFormat:@"%@", exception.reason], nil);
-    return;
-  }
-}
-
-- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
-    (const facebook::react::ObjCTurboModule::InitParams &)params {
-  return std::make_shared<facebook::react::NativeStyleTransferSpecJSI>(params);
-}
-
-@end

package/ios/RnExecutorch/TextEmbeddings.h

@@ -1,5 +0,0 @@
-#import <RnExecutorchSpec/RnExecutorchSpec.h>
-
-@interface TextEmbeddings : NSObject <NativeTextEmbeddingsSpec>
-
-@end