react-native-executorch 0.3.3 → 0.4.2

package/ios/RnExecutorch/OCR.mm

@@ -3,8 +3,6 @@
 #import "models/ocr/RecognitionHandler.h"
 #import "models/ocr/utils/Constants.h"
 #import "utils/ImageProcessor.h"
-#import <ExecutorchLib/ETModel.h>
-#import <React/RCTBridgeModule.h>
 
 @implementation OCR {
   Detector *detector;
@@ -13,6 +11,11 @@
 
 RCT_EXPORT_MODULE()
 
+- (void)releaseResources {
+  detector = nil;
+  recognitionHandler = nil;
+}
+
 - (void)loadModule:(NSString *)detectorSource
      recognizerSourceLarge:(NSString *)recognizerSourceLarge
     recognizerSourceMedium:(NSString *)recognizerSourceMedium
@@ -21,47 +24,40 @@ RCT_EXPORT_MODULE()
                    resolve:(RCTPromiseResolveBlock)resolve
                     reject:(RCTPromiseRejectBlock)reject {
   detector = [[Detector alloc] init];
-  [detector
-       loadModel:[NSURL URLWithString:detectorSource]
-      completion:^(BOOL success, NSNumber *errorCode) {
-        if (!success) {
-          NSError *error = [NSError
-              errorWithDomain:@"OCRErrorDomain"
-                         code:[errorCode intValue]
-                     userInfo:@{
-                       NSLocalizedDescriptionKey : [NSString
-                           stringWithFormat:@"%ld", (long)[errorCode longValue]]
-                     }];
-          reject(@"init_module_error", @"Failed to initialize detector module",
-                 error);
-          return;
-        }
-        self->recognitionHandler =
-            [[RecognitionHandler alloc] initWithSymbols:symbols];
-        [self->recognitionHandler
-                 loadRecognizers:recognizerSourceLarge
-            mediumRecognizerPath:recognizerSourceMedium
-             smallRecognizerPath:recognizerSourceSmall
-                      completion:^(BOOL allModelsLoaded, NSNumber *errorCode) {
-                        if (allModelsLoaded) {
-                          resolve(@(YES));
-                        } else {
-                          NSError *error = [NSError
-                              errorWithDomain:@"OCRErrorDomain"
-                                         code:[errorCode intValue]
-                                     userInfo:@{
-                                       NSLocalizedDescriptionKey : [NSString
-                                           stringWithFormat:@"%ld",
-                                                            (long)[errorCode
-                                                                longValue]]
-                                     }];
-                          reject(@"init_recognizer_error",
-                                 @"Failed to initialize one or more "
-                                 @"recognizer models",
-                                 error);
-                        }
-                      }];
-      }];
+  NSNumber *errorCode = [detector loadModel:detectorSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    NSError *error = [NSError
+        errorWithDomain:@"OCRErrorDomain"
+                   code:[errorCode intValue]
+               userInfo:@{
+                 NSLocalizedDescriptionKey : [NSString
+                     stringWithFormat:@"%ld", (long)[errorCode longValue]]
+               }];
+    reject(@"init_module_error", @"Failed to initialize detector module",
+           error);
+    return;
+  }
+
+  recognitionHandler = [[RecognitionHandler alloc] initWithSymbols:symbols];
+  errorCode = [recognitionHandler loadRecognizers:recognizerSourceLarge
+                             mediumRecognizerPath:recognizerSourceMedium
+                              smallRecognizerPath:recognizerSourceSmall];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    NSError *error = [NSError
+        errorWithDomain:@"OCRErrorDomain"
+                   code:[errorCode intValue]
+               userInfo:@{
+                 NSLocalizedDescriptionKey : [NSString
+                     stringWithFormat:@"%ld", (long)[errorCode longValue]]
+               }];
+    reject(@"init_recognizer_error",
+           @"Failed to initialize one or more recognizer models", error);
+    return;
+  }
+
+  resolve(@0);
 }
 
 - (void)forward:(NSString *)input

package/ios/RnExecutorch/ObjectDetection.mm

@@ -1,8 +1,6 @@
 #import "ObjectDetection.h"
 #import "models/object_detection/SSDLiteLargeModel.hpp"
 #import "utils/ImageProcessor.h"
-#import <ExecutorchLib/ETModel.h>
-#import <React/RCTBridgeModule.h>
 
 @implementation ObjectDetection {
   SSDLiteLargeModel *model;
@@ -10,28 +8,30 @@
 
 RCT_EXPORT_MODULE()
 
+- (void)releaseResources {
+  model = nil;
+}
+
 - (void)loadModule:(NSString *)modelSource
            resolve:(RCTPromiseResolveBlock)resolve
             reject:(RCTPromiseRejectBlock)reject {
   model = [[SSDLiteLargeModel alloc] init];
-  [model loadModel:[NSURL URLWithString:modelSource]
-        completion:^(BOOL success, NSNumber *errorCode) {
-          if (success) {
-            resolve(errorCode);
-            return;
-          }
-
-          NSError *error = [NSError
-              errorWithDomain:@"StyleTransferErrorDomain"
-                         code:[errorCode intValue]
-                     userInfo:@{
-                       NSLocalizedDescriptionKey : [NSString
-                           stringWithFormat:@"%ld", (long)[errorCode longValue]]
-                     }];
-
-          reject(@"init_module_error", error.localizedDescription, error);
-          return;
-        }];
+
+  NSNumber *errorCode = [model loadModel:modelSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    NSError *error = [NSError
+        errorWithDomain:@"StyleTransferErrorDomain"
+                   code:[errorCode intValue]
+               userInfo:@{
+                 NSLocalizedDescriptionKey : [NSString
+                     stringWithFormat:@"%ld", (long)[errorCode longValue]]
+               }];
+    reject(@"init_module_error", error.localizedDescription, error);
+    return;
+  }
+
+  resolve(@0);
 }
 
 - (void)forward:(NSString *)input

package/ios/RnExecutorch/SpeechToText.mm

@@ -1,13 +1,6 @@
 #import "SpeechToText.h"
-#import "./utils/ScalarType.h"
-#import "models/BaseModel.h"
 #import "models/stt/Moonshine.hpp"
-#import "models/stt/SpeechToTextBaseModel.hpp"
 #import "models/stt/Whisper.hpp"
-#import "utils/SFFT.hpp"
-#import <Accelerate/Accelerate.h>
-#import <ExecutorchLib/ETModel.h>
-#import <React/RCTBridgeModule.h>
 
 @implementation SpeechToText {
   Whisper *whisper;
@@ -16,6 +9,11 @@
 
 RCT_EXPORT_MODULE()
 
+- (void)releaseResources {
+  whisper = nil;
+  moonshine = nil;
+}
+
 - (void)generate:(NSArray *)waveform
          resolve:(RCTPromiseResolveBlock)resolve
           reject:(RCTPromiseRejectBlock)reject {
@@ -83,6 +81,7 @@ RCT_EXPORT_MODULE()
     [model loadModules:modelSources];
     resolve(@(0));
   } @catch (NSException *exception) {
+    [self releaseResources];
     reject(@"init_decoder_error",
            [NSString stringWithFormat:@"%@", exception.reason], nil);
   }

package/ios/RnExecutorch/StyleTransfer.mm

@@ -1,11 +1,6 @@
 #import "StyleTransfer.h"
 #import "ImageProcessor.h"
-#import "models/BaseModel.h"
-#import "models/StyleTransferModel.h"
-#import "utils/ETError.h"
-#import <ExecutorchLib/ETModel.h>
-#import <React/RCTBridgeModule.h>
-#import <opencv2/opencv.hpp>
+#import "models/style_transfer/StyleTransferModel.h"
 
 @implementation StyleTransfer {
   StyleTransferModel *model;
@@ -13,23 +8,25 @@
 
 RCT_EXPORT_MODULE()
 
+- (void)releaseResources {
+  model = nil;
+}
+
 - (void)loadModule:(NSString *)modelSource
            resolve:(RCTPromiseResolveBlock)resolve
             reject:(RCTPromiseRejectBlock)reject {
   model = [[StyleTransferModel alloc] init];
-  [model
-       loadModel:[NSURL URLWithString:modelSource]
-      completion:^(BOOL success, NSNumber *errorCode) {
-        if (success) {
-          resolve(errorCode);
-          return;
-        }
-
-        reject(@"init_module_error",
-               [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
-               nil);
-        return;
-      }];
+
+  NSNumber *errorCode = [model loadModel:modelSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    reject(@"init_module_error",
+           [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
+           nil);
+    return;
+  }
+
+  resolve(@0);
 }
 
 - (void)forward:(NSString *)input

package/ios/RnExecutorch/TextEmbeddings.h

@@ -0,0 +1,5 @@
+#import <RnExecutorchSpec/RnExecutorchSpec.h>
+
+@interface TextEmbeddings : NSObject <NativeTextEmbeddingsSpec>
+
+@end

package/ios/RnExecutorch/TextEmbeddings.mm

@@ -0,0 +1,62 @@
+#import "TextEmbeddings.h"
+#import "models/text_embeddings/TextEmbeddingsModel.h"
+
+@implementation TextEmbeddings {
+  TextEmbeddingsModel *model;
+}
+
+RCT_EXPORT_MODULE()
+
+- (void)releaseResources {
+  model = nil;
+}
+
+- (void)loadModule:(NSString *)modelSource
+    tokenizerSource:(NSString *)tokenizerSource
+            resolve:(RCTPromiseResolveBlock)resolve
+             reject:(RCTPromiseRejectBlock)reject {
+  model = [[TextEmbeddingsModel alloc] init];
+
+  @try {
+    [self->model loadTokenizer:tokenizerSource];
+  } @catch (NSException *exception) {
+    [self releaseResources];
+    reject(@"Tokenizer_Error",
+           [NSString stringWithFormat:@"Failed to load tokenizer from: %@",
+                                      tokenizerSource],
+           nil);
+    return;
+  }
+
+  NSNumber *errorCode = [model loadModel:modelSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    reject(@"init_module_error",
+           [NSString stringWithFormat:@"%ld", (long)[errorCode longValue]],
+           nil);
+    return;
+  }
+
+  resolve(@0);
+}
+
+- (void)forward:(NSString *)input
+        resolve:(RCTPromiseResolveBlock)resolve
+         reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    resolve([model runModel:input]);
+    return;
+  } @catch (NSException *exception) {
+    NSLog(@"An exception occurred: %@, %@", exception.name, exception.reason);
+    reject(@"forward_error",
+           [NSString stringWithFormat:@"%@", exception.reason], nil);
+    return;
+  }
+}
+
+- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
+    (const facebook::react::ObjCTurboModule::InitParams &)params {
+  return std::make_shared<facebook::react::NativeTextEmbeddingsSpecJSI>(params);
+}
+
+@end

package/ios/RnExecutorch/Tokenizer.h

@@ -0,0 +1,5 @@
+#import <RnExecutorchSpec/RnExecutorchSpec.h>
+
+@interface Tokenizer : NSObject <NativeTokenizerSpec>
+
+@end

package/ios/RnExecutorch/Tokenizer.mm

@@ -0,0 +1,83 @@
+#import "Tokenizer.h"
+#import <ExecutorchLib/HuggingFaceTokenizer.h>
+
+@implementation Tokenizer {
+  HuggingFaceTokenizer *tokenizer;
+}
+
+RCT_EXPORT_MODULE()
+
+- (void)loadModule:(NSString *)tokenizerSource
+           resolve:(RCTPromiseResolveBlock)resolve
+            reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    tokenizer =
+        [[HuggingFaceTokenizer alloc] initWithTokenizerPath:tokenizerSource];
+    resolve(@(0));
+  } @catch (NSException *exception) {
+    reject(@"Tokenizer_Error", @"Failed to load tokenizer", nil);
+  }
+}
+
+- (void)encode:(NSString *)input
+       resolve:(RCTPromiseResolveBlock)resolve
+        reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    resolve([tokenizer encode:input]);
+  } @catch (NSException *exception) {
+    reject(@"tokenizer_error",
+           [NSString stringWithFormat:@"%@", exception.reason], nil);
+  }
+}
+
+- (void)decode:(NSArray *)input
+    skipSpecialTokens:(BOOL)skipSpecialTokens
+              resolve:(RCTPromiseResolveBlock)resolve
+               reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    resolve([tokenizer decode:input skipSpecialTokens:skipSpecialTokens]);
+  } @catch (NSException *exception) {
+    reject(@"tokenizer_error",
+           [NSString stringWithFormat:@"%@", exception.reason], nil);
+  }
+}
+
+- (void)getVocabSize:(RCTPromiseResolveBlock)resolve
+              reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    resolve([NSNumber numberWithUnsignedInteger:[tokenizer getVocabSize]]);
+  } @catch (NSException *exception) {
+    reject(@"tokenizer_error",
+           [NSString stringWithFormat:@"%@", exception.reason], nil);
+  }
+}
+
+- (void)idToToken:(double)input
+          resolve:(RCTPromiseResolveBlock)resolve
+           reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    NSInteger tokenID = (NSInteger)input;
+    resolve([tokenizer idToToken:tokenID]);
+  } @catch (NSException *exception) {
+    reject(@"tokenizer_error",
+           [NSString stringWithFormat:@"%@", exception.reason], nil);
+  }
+}
+
+- (void)tokenToId:(NSString *)input
+          resolve:(RCTPromiseResolveBlock)resolve
+           reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    resolve([NSNumber numberWithInteger:[tokenizer tokenToId:input]]);
+  } @catch (NSException *exception) {
+    reject(@"tokenizer_error",
+           [NSString stringWithFormat:@"%@", exception.reason], nil);
+  }
+}
+
+- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
+    (const facebook::react::ObjCTurboModule::InitParams &)params {
+  return std::make_shared<facebook::react::NativeTokenizerSpecJSI>(params);
+}
+
+@end

package/ios/RnExecutorch/VerticalOCR.mm

@@ -1,13 +1,11 @@
 #import "VerticalOCR.h"
-#import "models/ocr/RecognitionHandler.h"
 #import "models/ocr/Recognizer.h"
 #import "models/ocr/VerticalDetector.h"
 #import "models/ocr/utils/CTCLabelConverter.h"
+#import "models/ocr/utils/Constants.h"
 #import "models/ocr/utils/OCRUtils.h"
 #import "models/ocr/utils/RecognizerUtils.h"
 #import "utils/ImageProcessor.h"
-#import <ExecutorchLib/ETModel.h>
-#import <React/RCTBridgeModule.h>
 
 @implementation VerticalOCR {
   VerticalDetector *detectorLarge;
@@ -19,6 +17,13 @@
 
 RCT_EXPORT_MODULE()
 
+- (void)releaseResources {
+  detectorLarge = nil;
+  detectorNarrow = nil;
+  recognizer = nil;
+  converter = nil;
+}
+
 - (void)loadModule:(NSString *)detectorLargeSource
      detectorNarrowSource:(NSString *)detectorNarrowSource
          recognizerSource:(NSString *)recognizerSource
@@ -26,42 +31,37 @@ RCT_EXPORT_MODULE()
     independentCharacters:(BOOL)independentCharacters
                   resolve:(RCTPromiseResolveBlock)resolve
                    reject:(RCTPromiseRejectBlock)reject {
-  detectorLarge = [[VerticalDetector alloc] initWithDetectSingleCharacters:NO];
   converter = [[CTCLabelConverter alloc] initWithCharacters:symbols
                                               separatorList:@{}];
   self->independentCharacters = independentCharacters;
-  [detectorLarge
-       loadModel:[NSURL URLWithString:detectorLargeSource]
-      completion:^(BOOL success, NSNumber *errorCode) {
-        if (!success) {
-          reject(@"init_module_error", @"Failed to initialize detector module",
-                 nil);
-          return;
-        }
-        self->detectorNarrow =
-            [[VerticalDetector alloc] initWithDetectSingleCharacters:YES];
-        [self->detectorNarrow
-             loadModel:[NSURL URLWithString:detectorNarrowSource]
-            completion:^(BOOL success, NSNumber *errorCode) {
-              if (!success) {
-                reject(@"init_module_error",
-                       @"Failed to initialize detector module", nil);
-                return;
-              }
-
-              self->recognizer = [[Recognizer alloc] init];
-              [self->recognizer
-                   loadModel:[NSURL URLWithString:recognizerSource]
-                  completion:^(BOOL success, NSNumber *errorCode) {
-                    if (!success) {
-                      reject(@"init_module_error",
-                             @"Failed to initialize recognizer module", nil);
-                    }
-
-                    resolve(@(YES));
-                  }];
-            }];
-      }];
+
+  detectorLarge = [[VerticalDetector alloc] initWithDetectSingleCharacters:NO];
+  NSNumber *errorCode = [detectorLarge loadModel:detectorLargeSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    reject(@"init_module_error", @"Failed to initialize detector module", nil);
+    return;
+  }
+
+  detectorNarrow =
+      [[VerticalDetector alloc] initWithDetectSingleCharacters:YES];
+  errorCode = [detectorNarrow loadModel:detectorNarrowSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    reject(@"init_module_error", @"Failed to initialize detector module", nil);
+    return;
+  }
+
+  recognizer = [[Recognizer alloc] init];
+  errorCode = [recognizer loadModel:recognizerSource];
+  if ([errorCode intValue] != 0) {
+    [self releaseResources];
+    reject(@"init_module_error", @"Failed to initialize recognizer module",
+           nil);
+    return;
+  }
+
+  resolve(@0);
 }
 
 - (void)forward:(NSString *)input

package/ios/RnExecutorch/models/BaseModel.h

@@ -1,13 +1,11 @@
 #import "ExecutorchLib/ETModel.h"
-#import <Foundation/Foundation.h>
-#import <UIKit/UIKit.h>
 
 @interface BaseModel : NSObject {
 @protected
   ETModel *module;
 }
 
-- (NSArray *)forward:(NSArray *)input;
+- (NSArray *)forward:(NSArray *)inputs;
 
 - (NSArray *)forward:(NSArray *)inputs
               shapes:(NSArray *)shapes
@@ -18,7 +16,6 @@
               shapes:(NSArray *)shapes
           inputTypes:(NSArray *)inputTypes;
 
-- (void)loadModel:(NSURL *)modelURL
-       completion:(void (^)(BOOL success, NSNumber *code))completion;
+- (NSNumber *)loadModel:(NSString *)modelSource;
 
 @end

package/ios/RnExecutorch/models/BaseModel.mm

@@ -1,9 +1,8 @@
 #import "BaseModel.h"
-#import "../utils/ETError.h"
 
 @implementation BaseModel
 
-- (NSArray *)forward:(NSArray *)input {
+- (NSArray *)forward:(NSArray *)inputs {
   NSMutableArray *shapes = [NSMutableArray new];
   NSMutableArray *inputTypes = [NSMutableArray new];
   NSNumber *numberOfInputs = [module getNumberOfInputs];
@@ -13,9 +12,8 @@
     [inputTypes addObject:[module getInputType:[NSNumber numberWithInt:i]]];
   }
 
-  NSArray *result = [module forward:@[ input ]
-                             shapes:shapes
-                         inputTypes:inputTypes];
+  NSArray *result = [module forward:inputs shapes:shapes inputTypes:inputTypes];
+
   return result;
 }
 
@@ -37,17 +35,9 @@
   return result;
 }
 
-- (void)loadModel:(NSURL *)modelURL
-       completion:(void (^)(BOOL success, NSNumber *code))completion {
+- (NSNumber *)loadModel:(NSString *)modelSource {
   module = [[ETModel alloc] init];
-  NSNumber *result = [self->module loadModel:modelURL.path];
-  if ([result intValue] != 0) {
-    completion(NO, result);
-    return;
-  }
-
-  completion(YES, result);
-  return;
+  return [self->module loadModel:modelSource];
 }
 
 @end

package/ios/RnExecutorch/models/classification/ClassificationModel.mm

@@ -1,8 +1,7 @@
 #import "ClassificationModel.h"
 #import "../../utils/ImageProcessor.h"
+#import "../../utils/Numerical.h"
 #import "Constants.h"
-#import "Utils.h"
-#import "opencv2/opencv.hpp"
 
 @implementation ClassificationModel
 
@@ -47,7 +46,7 @@
 
 - (NSDictionary *)runModel:(cv::Mat &)input {
   NSArray *modelInput = [self preprocess:input];
-  NSArray *modelOutput = [self forward:modelInput];
+  NSArray *modelOutput = [self forward:@[ modelInput ]];
   return [self postprocess:modelOutput];
 }
 

package/ios/RnExecutorch/models/classification/Constants.mm

@@ -1,5 +1,4 @@
 #import "Constants.h"
-#import <string>
 
 const std::string imagenet1k_v1_labels[] = {
     "tench, Tinca tinca",

package/ios/RnExecutorch/models/image_segmentation/Constants.h

@@ -0,0 +1,4 @@
+#import <string>
+#import <vector>
+
+extern const std::vector<std::string> deeplabv3_resnet50_labels;

package/ios/RnExecutorch/models/image_segmentation/Constants.mm

@@ -0,0 +1,8 @@
+#import "Constants.h"
+
+const std::vector<std::string> deeplabv3_resnet50_labels = {
+    "BACKGROUND", "AEROPLANE",   "BICYCLE", "BIRD",  "BOAT",
+    "BOTTLE",     "BUS",         "CAR",     "CAT",   "CHAIR",
+    "COW",        "DININGTABLE", "DOG",     "HORSE", "MOTORBIKE",
+    "PERSON",     "POTTEDPLANT", "SHEEP",   "SOFA",  "TRAIN",
+    "TVMONITOR"};

package/ios/RnExecutorch/models/image_segmentation/ImageSegmentationModel.h

@@ -0,0 +1,10 @@
+#import "../BaseModel.h"
+#import "opencv2/opencv.hpp"
+
+@interface ImageSegmentationModel : BaseModel
+- (cv::Size)getModelImageSize;
+- (NSDictionary *)runModel:(cv::Mat &)input
+             returnClasses:(NSArray *)classesOfInterest
+                    resize:(BOOL)resize;
+
+@end