react-native-executorch 0.3.3 → 0.4.1

package/android/src/main/java/com/swmansion/rnexecutorch/models/imageSegmentation/ImageSegmentationModel.kt

@@ -0,0 +1,142 @@
+package com.swmansion.rnexecutorch.models.imagesegmentation
+
+import com.facebook.react.bridge.Arguments
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReadableArray
+import com.facebook.react.bridge.WritableMap
+import com.swmansion.rnexecutorch.models.BaseModel
+import com.swmansion.rnexecutorch.utils.ArrayUtils
+import com.swmansion.rnexecutorch.utils.ImageProcessor
+import com.swmansion.rnexecutorch.utils.softmax
+import org.opencv.core.CvType
+import org.opencv.core.Mat
+import org.opencv.core.Size
+import org.opencv.imgproc.Imgproc
+import org.pytorch.executorch.EValue
+
+class ImageSegmentationModel(
+  reactApplicationContext: ReactApplicationContext,
+) : BaseModel<Triple<Mat, ReadableArray, Boolean>, WritableMap>(reactApplicationContext) {
+  private lateinit var originalSize: Size
+
+  private fun getModelImageSize(): Size {
+    val inputShape = module.getInputShape(0)
+    val width = inputShape[inputShape.lastIndex]
+    val height = inputShape[inputShape.lastIndex - 1]
+
+    return Size(height.toDouble(), width.toDouble())
+  }
+
+  fun preprocess(input: Mat): EValue {
+    originalSize = input.size()
+    Imgproc.resize(input, input, getModelImageSize())
+    return ImageProcessor.matToEValue(input, module.getInputShape(0))
+  }
+
+  private fun extractResults(
+    result: FloatArray,
+    numLabels: Int,
+    resize: Boolean,
+  ): List<FloatArray> {
+    val modelSize = getModelImageSize()
+    val numModelPixels = (modelSize.height * modelSize.width).toInt()
+
+    val extractedLabelScores = mutableListOf<FloatArray>()
+
+    for (label in 0..<numLabels) {
+      // Calls to OpenCV via JNI are very slow so we do as much as we can
+      // with pure Kotlin
+      val range = IntRange(label * numModelPixels, (label + 1) * numModelPixels - 1)
+      val pixelBuffer = result.slice(range).toFloatArray()
+
+      if (resize) {
+        // Rescale the image with OpenCV
+        val mat = Mat(modelSize, CvType.CV_32F)
+        mat.put(0, 0, pixelBuffer)
+        val resizedMat = Mat()
+        Imgproc.resize(mat, resizedMat, originalSize)
+        val resizedBuffer = FloatArray((originalSize.height * originalSize.width).toInt())
+        resizedMat.get(0, 0, resizedBuffer)
+        extractedLabelScores.add(resizedBuffer)
+      } else {
+        extractedLabelScores.add(pixelBuffer)
+      }
+    }
+    return extractedLabelScores
+  }
+
+  private fun adjustScoresPerPixel(
+    labelScores: List<FloatArray>,
+    numLabels: Int,
+    outputSize: Size,
+  ): IntArray {
+    val numPixels = (outputSize.height * outputSize.width).toInt()
+    val argMax = IntArray(numPixels)
+    for (pixel in 0..<numPixels) {
+      val scores = mutableListOf<Float>()
+      for (buffer in labelScores) {
+        scores.add(buffer[pixel])
+      }
+      val adjustedScores = softmax(scores.toTypedArray())
+      for (label in 0..<numLabels) {
+        labelScores[label][pixel] = adjustedScores[label]
+      }
+
+      val maxIndex = scores.withIndex().maxBy { it.value }.index
+      argMax[pixel] = maxIndex
+    }
+
+    return argMax
+  }
+
+  fun postprocess(
+    output: Array<EValue>,
+    classesOfInterest: ReadableArray,
+    resize: Boolean,
+  ): WritableMap {
+    val outputData = output[0].toTensor().dataAsFloatArray
+    val modelSize = getModelImageSize()
+    val numLabels = deeplabv3_resnet50_labels.size
+
+    require(outputData.count() == (numLabels * modelSize.height * modelSize.width).toInt()) { "Model generated unexpected output size." }
+
+    val outputSize = if (resize) originalSize else modelSize
+
+    val extractedResults = extractResults(outputData, numLabels, resize)
+
+    val argMax = adjustScoresPerPixel(extractedResults, numLabels, outputSize)
+
+    val labelSet = mutableSetOf<String>()
+    // Filter by the label set when base class changed
+    for (i in 0..<classesOfInterest.size()) {
+      val label = classesOfInterest.getString(i)
+      if (label != null) {
+        labelSet.add(label)
+      }
+    }
+
+    val res = Arguments.createMap()
+
+    for (label in 0..<numLabels) {
+      if (labelSet.contains(deeplabv3_resnet50_labels[label])) {
+        res.putArray(
+          deeplabv3_resnet50_labels[label],
+          ArrayUtils.createReadableArrayFromFloatArray(extractedResults[label]),
+        )
+      }
+    }
+
+    res.putArray(
+      "ARGMAX",
+      ArrayUtils.createReadableArrayFromIntArray(argMax),
+    )
+
+    return res
+  }
+
+  override fun runModel(input: Triple<Mat, ReadableArray, Boolean>): WritableMap {
+    val modelInput = preprocess(input.first)
+    val modelOutput = forward(modelInput)
+    return postprocess(modelOutput, input.second, input.third)
+  }
+}

package/android/src/main/java/com/swmansion/rnexecutorch/models/ocr/utils/RecognizerUtils.kt

@@ -335,6 +335,9 @@ class RecognizerUtils {
         cords.x *= originalPaddings["resizeRatio"]!! as Float
         cords.y *= originalPaddings["resizeRatio"]!! as Float
 
+        cords.x = cords.x.coerceIn(0.0, (image.cols() - 1).toDouble())
+        cords.y = cords.y.coerceIn(0.0, (image.rows() - 1).toDouble())
+
         points[i] = Point(cords.x, cords.y)
       }
 

package/android/src/main/java/com/swmansion/rnexecutorch/models/{StyleTransferModel.kt → styleTransfer/StyleTransferModel.kt}

@@ -1,4 +1,4 @@
-package com.swmansion.rnexecutorch.models
+package com.swmansion.rnexecutorch.models.styletransfer
 
 import com.facebook.react.bridge.ReactApplicationContext
 import com.swmansion.rnexecutorch.utils.ImageProcessor
@@ -6,6 +6,7 @@ import org.opencv.core.Mat
 import org.opencv.core.Size
 import org.opencv.imgproc.Imgproc
 import org.pytorch.executorch.EValue
+import com.swmansion.rnexecutorch.models.BaseModel
 
 class StyleTransferModel(
   reactApplicationContext: ReactApplicationContext,

package/android/src/main/java/com/swmansion/rnexecutorch/utils/ArrayUtils.kt

@@ -32,14 +32,6 @@ class ArrayUtils {
     fun createDoubleArray(input: ReadableArray): DoubleArray =
       createTypedArrayFromReadableArray(input) { array, index -> array.getDouble(index) }.toDoubleArray()
 
-    fun <V> createMapArray(input: ReadableArray): Array<Map<String, V>> {
-      val mapArray = Array<Map<String, V>>(input.size()) { mapOf() }
-      for (i in 0 until input.size()) {
-        mapArray[i] = input.getMap(i)?.toHashMap() as Map<String, V>
-      }
-      return mapArray
-    }
-
     fun createReadableArrayFromTensor(result: Tensor): ReadableArray {
       val resultArray = Arguments.createArray()
 

package/android/src/main/java/com/swmansion/rnexecutorch/{models/classification/Utils.kt → utils/Numerical.kt}

@@ -1,4 +1,4 @@
-package com.swmansion.rnexecutorch.models.classification
+package com.swmansion.rnexecutorch.utils
 
 fun softmax(x: Array<Float>): Array<Float> {
   val max = x.maxOrNull()!!

package/ios/ExecutorchLib.xcframework/Info.plist

@@ -8,7 +8,7 @@
 			<key>BinaryPath</key>
 			<string>ExecutorchLib.framework/ExecutorchLib</string>
 			<key>LibraryIdentifier</key>
-			<string>ios-arm64</string>
+			<string>ios-arm64-simulator</string>
 			<key>LibraryPath</key>
 			<string>ExecutorchLib.framework</string>
 			<key>SupportedArchitectures</key>
@@ -17,12 +17,14 @@
 			</array>
 			<key>SupportedPlatform</key>
 			<string>ios</string>
+			<key>SupportedPlatformVariant</key>
+			<string>simulator</string>
 		</dict>
 		<dict>
 			<key>BinaryPath</key>
 			<string>ExecutorchLib.framework/ExecutorchLib</string>
 			<key>LibraryIdentifier</key>
-			<string>ios-arm64-simulator</string>
+			<string>ios-arm64</string>
 			<key>LibraryPath</key>
 			<string>ExecutorchLib.framework</string>
 			<key>SupportedArchitectures</key>
@@ -31,8 +33,6 @@
 			</array>
 			<key>SupportedPlatform</key>
 			<string>ios</string>
-			<key>SupportedPlatformVariant</key>
-			<string>simulator</string>
 		</dict>
 	</array>
 	<key>CFBundlePackageType</key>

package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h

@@ -0,0 +1,14 @@
+#import <Foundation/Foundation.h>
+
+@interface HuggingFaceTokenizer : NSObject
+
+- (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
+- (NSArray<NSNumber *> *)encode:(NSString *)text;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds
+    skipSpecialTokens:(BOOL)skipSpecialTokens;
+- (NSUInteger)getVocabSize;
+- (NSString *)idToToken:(NSInteger)tokenId;
+- (NSInteger)tokenToId:(NSString *)token;
+
+@end

package/ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/LLaMARunner.h

@@ -11,14 +11,13 @@
 NS_ASSUME_NONNULL_BEGIN
 
 FOUNDATION_EXPORT NSErrorDomain const LLaMARunnerErrorDomain;
-FOUNDATION_EXPORT NSErrorDomain const LLaVARunnerErrorDomain;
 
 NS_SWIFT_NAME(Runner)
 @interface LLaMARunner : NSObject
 
 - (instancetype)initWithModelPath:(NSString *)filePath
                     tokenizerPath:(NSString *)tokenizerPath;
-- (BOOL)isloaded;
+- (BOOL)isLoaded;
 - (BOOL)loadWithError:(NSError **)error;
 - (BOOL)generate:(NSString *)prompt
     withTokenCallback:(nullable void (^)(NSString *))callback
@@ -30,25 +29,4 @@ NS_SWIFT_NAME(Runner)
 
 @end
 
-NS_SWIFT_NAME(LLaVARunner)
-@interface LLaVARunner : NSObject
-
-- (instancetype)initWithModelPath:(NSString *)filePath
-                    tokenizerPath:(NSString *)tokenizerPath;
-- (BOOL)isloaded;
-- (BOOL)loadWithError:(NSError **)error;
-- (BOOL)generate:(void *)imageBuffer
-                width:(CGFloat)width
-               height:(CGFloat)height
-               prompt:(NSString *)prompt
-       sequenceLength:(NSInteger)seq_len
-    withTokenCallback:(nullable void (^)(NSString *))callback
-                error:(NSError **)error;
-- (void)stop;
-
-+ (instancetype)new NS_UNAVAILABLE;
-- (instancetype)init NS_UNAVAILABLE;
-
-@end
-
 NS_ASSUME_NONNULL_END

package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h

@@ -0,0 +1,14 @@
+#import <Foundation/Foundation.h>
+
+@interface HuggingFaceTokenizer : NSObject
+
+- (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
+- (NSArray<NSNumber *> *)encode:(NSString *)text;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds
+    skipSpecialTokens:(BOOL)skipSpecialTokens;
+- (NSUInteger)getVocabSize;
+- (NSString *)idToToken:(NSInteger)tokenId;
+- (NSInteger)tokenToId:(NSString *)token;
+
+@end

package/ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/LLaMARunner.h

@@ -11,14 +11,13 @@
 NS_ASSUME_NONNULL_BEGIN
 
 FOUNDATION_EXPORT NSErrorDomain const LLaMARunnerErrorDomain;
-FOUNDATION_EXPORT NSErrorDomain const LLaVARunnerErrorDomain;
 
 NS_SWIFT_NAME(Runner)
 @interface LLaMARunner : NSObject
 
 - (instancetype)initWithModelPath:(NSString *)filePath
                     tokenizerPath:(NSString *)tokenizerPath;
-- (BOOL)isloaded;
+- (BOOL)isLoaded;
 - (BOOL)loadWithError:(NSError **)error;
 - (BOOL)generate:(NSString *)prompt
     withTokenCallback:(nullable void (^)(NSString *))callback
@@ -30,25 +29,4 @@ NS_SWIFT_NAME(Runner)
 
 @end
 
-NS_SWIFT_NAME(LLaVARunner)
-@interface LLaVARunner : NSObject
-
-- (instancetype)initWithModelPath:(NSString *)filePath
-                    tokenizerPath:(NSString *)tokenizerPath;
-- (BOOL)isloaded;
-- (BOOL)loadWithError:(NSError **)error;
-- (BOOL)generate:(void *)imageBuffer
-                width:(CGFloat)width
-               height:(CGFloat)height
-               prompt:(NSString *)prompt
-       sequenceLength:(NSInteger)seq_len
-    withTokenCallback:(nullable void (^)(NSString *))callback
-                error:(NSError **)error;
-- (void)stop;
-
-+ (instancetype)new NS_UNAVAILABLE;
-- (instancetype)init NS_UNAVAILABLE;
-
-@end
-
 NS_ASSUME_NONNULL_END

package/ios/RnExecutorch/Classification.mm

@@ -1,11 +1,6 @@
 #import "Classification.h"
 #import "ImageProcessor.h"
-#import "models/BaseModel.h"
 #import "models/classification/ClassificationModel.h"
-#import "opencv2/opencv.hpp"
-#import "utils/ETError.h"
-#import <ExecutorchLib/ETModel.h>
-#import <React/RCTBridgeModule.h>
 
 @implementation Classification {
   ClassificationModel *model;
@@ -13,23 +8,25 @@
 
 RCT_EXPORT_MODULE()
 
+- (void)releaseResources {
+  model = nil;
+}
+
 - (void)loadModule:(NSString *)modelSource
            resolve:(RCTPromiseResolveBlock)resolve
             reject:(RCTPromiseRejectBlock)reject {
   model = [[ClassificationModel alloc] init];
-  [model
-       loadModel:[NSURL URLWithString:modelSource]
-      completion:^(BOOL success, NSNumber *errorCode) {
-        if (success) {
-          resolve(errorCode);
-          return;
-        }
-
-        reject(@"init_module_error",
-               [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
-               nil);
-        return;
-      }];
+
+  NSNumber *errorCode = [model loadModel:modelSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    reject(@"init_module_error",
+           [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
+           nil);
+    return;
+  }
+
+  resolve(@0);
 }
 
 - (void)forward:(NSString *)input

package/ios/RnExecutorch/ETModule.mm

@@ -1,8 +1,5 @@
 #import "ETModule.h"
 #import <ExecutorchLib/ETModel.h>
-#include <Foundation/Foundation.h>
-#import <React/RCTBridgeModule.h>
-#include <string>
 
 @implementation ETModule {
   ETModel *module;
@@ -10,16 +7,20 @@
 
 RCT_EXPORT_MODULE()
 
+- (void)releaseResources {
+  module = nil;
+}
+
 - (void)loadModule:(NSString *)modelSource
            resolve:(RCTPromiseResolveBlock)resolve
             reject:(RCTPromiseRejectBlock)reject {
   if (!module) {
     module = [[ETModel alloc] init];
   }
-  NSURL *modelURL = [NSURL URLWithString:modelSource];
-  NSNumber *result = [self->module loadModel:modelURL.path];
 
+  NSNumber *result = [self->module loadModel:modelSource];
   if ([result intValue] != 0) {
+    [self releaseResources];
     NSError *error =
         [NSError errorWithDomain:@"ETModuleErrorDomain"
                             code:[result intValue]

package/ios/RnExecutorch/ImageSegmentation.h

@@ -0,0 +1,5 @@
+#import <RnExecutorchSpec/RnExecutorchSpec.h>
+
+@interface ImageSegmentation : NSObject <NativeImageSegmentationSpec>
+
+@end

package/ios/RnExecutorch/ImageSegmentation.mm

@@ -0,0 +1,60 @@
+#import "ImageSegmentation.h"
+#import "ImageProcessor.h"
+#import "models/image_segmentation/ImageSegmentationModel.h"
+
+@implementation ImageSegmentation {
+  ImageSegmentationModel *model;
+}
+
+RCT_EXPORT_MODULE()
+
+- (void)releaseResources {
+  model = nil;
+}
+
+- (void)loadModule:(NSString *)modelSource
+           resolve:(RCTPromiseResolveBlock)resolve
+            reject:(RCTPromiseRejectBlock)reject {
+  model = [[ImageSegmentationModel alloc] init];
+
+  NSNumber *errorCode = [model loadModel:modelSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    reject(@"init_module_error",
+           [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
+           nil);
+    return;
+  }
+
+  resolve(@0);
+}
+
+- (void)forward:(NSString *)input
+    classesOfInterest:(NSArray *)classesOfInterest
+               resize:(BOOL)resize
+              resolve:(RCTPromiseResolveBlock)resolve
+               reject:(RCTPromiseRejectBlock)reject {
+
+  @try {
+    cv::Mat image = [ImageProcessor readImage:input];
+    NSDictionary *result = [model runModel:image
+                             returnClasses:classesOfInterest
+                                    resize:resize];
+
+    resolve(result);
+    return;
+  } @catch (NSException *exception) {
+    NSLog(@"An exception occurred: %@, %@", exception.name, exception.reason);
+    reject(@"forward_error",
+           [NSString stringWithFormat:@"%@", exception.reason], nil);
+    return;
+  }
+}
+
+- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
+    (const facebook::react::ObjCTurboModule::InitParams &)params {
+  return std::make_shared<facebook::react::NativeImageSegmentationSpecJSI>(
+      params);
+}
+
+@end

package/ios/RnExecutorch/LLM.mm

@@ -1,28 +1,12 @@
 #import "LLM.h"
-#import "utils/llms/Constants.h"
-#import "utils/llms/ConversationManager.h"
 #import <ExecutorchLib/LLaMARunner.h>
-#import <React/RCTBridge+Private.h>
-#import <React/RCTBridge.h>
-#import <React/RCTBridgeModule.h>
-#import <React/RCTUtils.h>
-#import <ReactCommon/CallInvoker.h>
-#import <ReactCommon/RCTTurboModule.h>
-#import <UIKit/UIKit.h>
-#import <react/renderer/uimanager/primitives.h>
-#import <string>
 
 @implementation LLM {
   LLaMARunner *runner;
-  ConversationManager *conversationManager;
-  NSMutableString *tempLlamaResponse;
 }
 
 - (instancetype)init {
   self = [super init];
-  if (self) {
-    tempLlamaResponse = [[NSMutableString alloc] init];
-  }
 
   return self;
 }
@@ -36,64 +20,39 @@ RCT_EXPORT_MODULE()
 
   dispatch_async(dispatch_get_main_queue(), ^{
     [self emitOnToken:token];
-    [self->tempLlamaResponse appendString:token];
   });
 }
 
 - (void)loadLLM:(NSString *)modelSource
-        tokenizerSource:(NSString *)tokenizerSource
-           systemPrompt:(NSString *)systemPrompt
-         messageHistory:(NSArray *)messageHistory
-    contextWindowLength:(double)contextWindowLength
-                resolve:(RCTPromiseResolveBlock)resolve
-                 reject:(RCTPromiseRejectBlock)reject {
-  NSURL *modelURL = [NSURL URLWithString:modelSource];
-  NSURL *tokenizerURL = [NSURL URLWithString:tokenizerSource];
+    tokenizerSource:(NSString *)tokenizerSource
+            resolve:(RCTPromiseResolveBlock)resolve
+             reject:(RCTPromiseRejectBlock)reject {
   @try {
-    self->runner = [[LLaMARunner alloc] initWithModelPath:modelURL.path
-                                            tokenizerPath:tokenizerURL.path];
-    NSUInteger contextWindowLengthUInt = (NSUInteger)round(contextWindowLength);
-
-    self->conversationManager = [[ConversationManager alloc]
-        initWithNumMessagesContextWindow:contextWindowLengthUInt
-                            systemPrompt:systemPrompt
-                          messageHistory:messageHistory];
+    self->runner = [[LLaMARunner alloc] initWithModelPath:modelSource
+                                            tokenizerPath:tokenizerSource];
 
-    self->tempLlamaResponse = [NSMutableString string];
     resolve(@"Model and tokenizer loaded successfully");
     return;
   } @catch (NSException *exception) {
+    [self releaseResources];
     reject(@"Model or tokenizer loading failed", exception.reason, nil);
     return;
   }
 }
 
-- (void)runInference:(NSString *)input
-             resolve:(RCTPromiseResolveBlock)resolve
-              reject:(RCTPromiseRejectBlock)reject {
-  [conversationManager addResponse:input senderRole:ChatRole::USER];
-  NSString *prompt = [conversationManager getConversation];
+- (void)forward:(NSString *)input
+        resolve:(RCTPromiseResolveBlock)resolve
+         reject:(RCTPromiseRejectBlock)reject {
 
   dispatch_async(
       dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
         NSError *error = nil;
-        [self->runner generate:prompt
+        [self->runner generate:input
              withTokenCallback:^(NSString *token) {
-               [self onResult:token prompt:prompt];
+               [self onResult:token prompt:input];
              }
                          error:&error];
 
-        // make sure to add eot token once generation is done
-        if (![self->tempLlamaResponse hasSuffix:END_OF_TEXT_TOKEN_NS]) {
-          [self onResult:END_OF_TEXT_TOKEN_NS prompt:prompt];
-        }
-
-        if (self->tempLlamaResponse) {
-          [self->conversationManager addResponse:self->tempLlamaResponse
-                                      senderRole:ChatRole::ASSISTANT];
-          self->tempLlamaResponse = [NSMutableString string];
-        }
-
         if (error) {
           reject(@"error_in_generation", error.localizedDescription, nil);
           return;
@@ -107,7 +66,7 @@ RCT_EXPORT_MODULE()
   [self->runner stop];
 }
 
-- (void)deleteModule {
+- (void)releaseResources {
   self->runner = nil;
 }