react-native-executorch 0.4.8 → 0.5.1-rc.0

package/third-party/include/opencv2/video/video.hpp

@@ -0,0 +1,55 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this
+license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                          License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
+// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without
+modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright
+notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote
+products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is"
+and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are
+disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any
+direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#ifdef __OPENCV_BUILD
+#error this is a compatibility header which should not be used inside the OpenCV library
+#endif
+
+#include "opencv2/video.hpp"

package/third-party/include/opencv2/video.hpp

@@ -0,0 +1,65 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this
+license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                          License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
+// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without
+modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright
+notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote
+products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is"
+and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are
+disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any
+direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#ifndef OPENCV_VIDEO_HPP
+#define OPENCV_VIDEO_HPP
+
+/**
+  @defgroup video Video Analysis
+  @{
+    @defgroup video_motion Motion Analysis
+    @defgroup video_track Object Tracking
+  @}
+*/
+
+#include "opencv2/video/background_segm.hpp"
+#include "opencv2/video/tracking.hpp"
+
+#endif // OPENCV_VIDEO_HPP

package/third-party/include/tokenizers-cpp/tokenizers_c.h

@@ -0,0 +1,61 @@
+/*!
+ *  Copyright (c) 2023 by Contributors
+ * \file tokenizers_c.h
+ * \brief C binding to tokenizers rust library
+ */
+#ifndef TOKENIZERS_C_H_
+#define TOKENIZERS_C_H_
+
+// The C API
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef void *TokenizerHandle;
+
+typedef struct {
+  int *token_ids;
+  size_t len;
+} TokenizerEncodeResult;
+
+TokenizerHandle tokenizers_new_from_str(const char *json, size_t len);
+
+TokenizerHandle byte_level_bpe_tokenizers_new_from_str(
+    const char *vocab, size_t vocab_len, const char *merges, size_t merges_len,
+    const char *added_tokens, size_t added_tokens_len);
+
+void tokenizers_encode(TokenizerHandle handle, const char *data, size_t len,
+                       int add_special_token, TokenizerEncodeResult *result);
+
+void tokenizers_encode_batch(TokenizerHandle handle, const char **data,
+                             size_t *len, size_t num_seqs,
+                             int add_special_token,
+                             TokenizerEncodeResult *results);
+
+void tokenizers_free_encode_results(TokenizerEncodeResult *results,
+                                    size_t num_seqs);
+
+void tokenizers_decode(TokenizerHandle handle, const uint32_t *data, size_t len,
+                       int skip_special_token);
+
+void tokenizers_get_decode_str(TokenizerHandle handle, const char **data,
+                               size_t *len);
+
+void tokenizers_get_vocab_size(TokenizerHandle handle, size_t *size);
+
+void tokenizers_id_to_token(TokenizerHandle handle, uint32_t id,
+                            const char **data, size_t *len);
+
+// tokenizers_token_to_id stores -1 to *id if the token is not in the vocab
+void tokenizers_token_to_id(TokenizerHandle handle, const char *token,
+                            size_t len, int32_t *id);
+
+void tokenizers_free(TokenizerHandle handle);
+
+#ifdef __cplusplus
+}
+#endif
+#endif // TOKENIZERS_C_H_

package/third-party/include/tokenizers-cpp/tokenizers_cpp.h

@@ -0,0 +1,118 @@
+/*!
+ *  Copyright (c) 2023 by Contributors
+ * \file tokenizers_cpp.h
+ * \brief A C++ binding to common set of tokenizers
+ */
+#ifndef TOKENIZERS_CPP_H_
+#define TOKENIZERS_CPP_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace tokenizers {
+
+/*!
+ * \brief a universal tokenizer that loads
+ *  either HF's tokenizer or sentence piece,
+ *  depending on the constructor
+ */
+class Tokenizer {
+public:
+  /*! \brief virtual destructor */
+  virtual ~Tokenizer() {}
+
+  /*!
+   * \brief Encode text into ids.
+   * \param text The input text.
+   * \returns The encoded token ids.
+   */
+  virtual std::vector<int32_t> Encode(const std::string &text) = 0;
+
+  /*!
+   * \brief Encode a batch of texts into ids.
+   * \param texts The input texts.
+   * \returns The encoded token ids.
+   */
+  virtual std::vector<std::vector<int32_t>>
+  EncodeBatch(const std::vector<std::string> &texts) {
+    // Fall back when the derived class does not implement this function.
+    std::vector<std::vector<int32_t>> ret;
+    ret.reserve(texts.size());
+    for (const auto &text : texts) {
+      ret.push_back(Encode(text));
+    }
+    return ret;
+  }
+
+  /*!
+   * \brief Decode token ids into text.
+   * \param text The token ids.
+   * \returns The decoded text.
+   */
+  virtual std::string Decode(const std::vector<int32_t> &ids) = 0;
+
+  virtual std::string Decode(const std::vector<int32_t> &ids,
+                             bool skip_special_tokens) = 0;
+
+  /*!
+   * \brief Returns the vocabulary size. Special tokens are considered.
+   */
+  virtual size_t GetVocabSize() = 0;
+
+  /*!
+   * \brief Convert the given id to its corresponding token if it exists. If
+   * not, return an empty string.
+   */
+  virtual std::string IdToToken(int32_t token_id) = 0;
+
+  /*!
+   * \brief Convert the given token to its corresponding id if it exists. If
+   * not, return -1.
+   */
+  virtual int32_t TokenToId(const std::string &token) = 0;
+
+  //---------------------------------------------------
+  // Factory functions from byte-blobs
+  // These factory function takes in in-memory blobs
+  // so the library can be independent from filesystem
+  //---------------------------------------------------
+  /*!
+   * \brief Create HF tokenizer from a single in-memory json blob.
+   *
+   * \param json_blob The json blob.
+   * \return The created tokenzier.
+   */
+  static std::unique_ptr<Tokenizer> FromBlobJSON(const std::string &json_blob);
+  /*!
+   * \brief Create BPE tokenizer
+   *
+   * \param vocab_blob The blob that contains vocabs.
+   * \param merges_blob The blob that contains the merges.
+   * \param added_tokens The added tokens.
+   * \return The created tokenizer.
+   */
+  static std::unique_ptr<Tokenizer>
+  FromBlobByteLevelBPE(const std::string &vocab_blob,
+                       const std::string &merges_blob,
+                       const std::string &added_tokens = "");
+  /*!
+   * \brief Create SentencePiece.
+   *
+   * \param model_blob The blob that contains vocabs.
+   * \return The created tokenizer.
+   */
+  static std::unique_ptr<Tokenizer>
+  FromBlobSentencePiece(const std::string &model_blob);
+  /*!
+   * \brief Create RWKVWorldTokenizer.
+   *
+   * \param model_blob The blob that contains vocabs.
+   * \return The created tokenizer.
+   */
+  static std::unique_ptr<Tokenizer>
+  FromBlobRWKVWorld(const std::string &model_blob);
+};
+
+} // namespace tokenizers
+#endif // TOKENIZERS_CPP_H_

package/third-party/ios/ExecutorchLib/ExecutorchLib/Exported/ETModel.h

@@ -0,0 +1,27 @@
+#ifndef ETModel_hpp
+#define ETModel_hpp
+
+#import <UIKit/UIKit.h>
+
+@interface ETModel : NSObject
+
+- (NSNumber *)loadModel:(NSString *)filePath;
+- (NSNumber *)loadMethod:(NSString *)methodName;
+- (NSNumber *)loadForward;
+- (NSArray *)execute:(NSString *)methodName
+              inputs:(NSArray *)inputs
+              shapes:(NSArray *)shapes
+          inputTypes:(NSArray *)inputTypes;
+- (NSArray *)forward:(NSArray *)inputs
+              shapes:(NSArray *)shapes
+          inputTypes:(NSArray *)inputTypes;
+- (NSNumber *)getNumberOfInputs;
+- (NSNumber *)getInputType:(NSNumber *)index;
+- (NSArray *)getInputShape:(NSNumber *)index;
+- (NSNumber *)getNumberOfOutputs;
+- (NSNumber *)getOutputType:(NSNumber *)index;
+- (NSArray *)getOutputShape:(NSNumber *)index;
+
+@end
+
+#endif // ETModel_hpp

package/third-party/ios/ExecutorchLib/ExecutorchLib/Exported/ETModel.mm

@@ -0,0 +1,249 @@
+#import "ETModel.h"
+#include "Utils.hpp"
+#include <executorch/extension/module/module.h>
+#include <executorch/extension/tensor/tensor.h>
+#include <executorch/runtime/core/error.h>
+#include <executorch/runtime/core/result.h>
+#include <string>
+
+using namespace ::executorch::extension;
+using namespace ::torch::executor;
+
+@implementation ETModel {
+  std::unique_ptr<Module> _model;
+}
+
+- (NSNumber *)loadModel:(NSString *)filePath {
+  _model = std::make_unique<Module>(filePath.UTF8String);
+  Error err = _model->load();
+  return @((int)err);
+}
+
+- (NSNumber *)loadMethod:(NSString *)methodName {
+  Error err = _model->load_method([methodName UTF8String]);
+  return @((int)err);
+}
+
+- (NSNumber *)loadForward {
+  Error err = _model->load_forward();
+  return @((int)err);
+}
+
+- (NSNumber *)getNumberOfInputs {
+  const auto method_meta = _model->method_meta("forward");
+  if (!method_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_number_of_inputs_error"
+                   reason:[NSString stringWithFormat:@"%ld",
+                                                     (long)method_meta.error()]
+                 userInfo:nil];
+  }
+
+  return @(method_meta->num_inputs());
+}
+
+- (NSNumber *)getInputType:(NSNumber *)index {
+  const auto method_meta = _model->method_meta("forward");
+  if (!method_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_input_type_error"
+                   reason:[NSString stringWithFormat:@"%ld",
+                                                     (long)method_meta.error()]
+                 userInfo:nil];
+  }
+
+  const auto input_meta =
+      method_meta->input_tensor_meta([index unsignedLongValue]);
+  if (!input_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_input_type_error"
+                   reason:[NSString
+                              stringWithFormat:@"%ld", (long)input_meta.error()]
+                 userInfo:nil];
+  }
+
+  return scalarTypeToNSNumber(input_meta->scalar_type());
+};
+
+- (NSArray *)getInputShape:(NSNumber *)index {
+  const auto method_meta = _model->method_meta("forward");
+  if (!method_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_input_shape_error"
+                   reason:[NSString stringWithFormat:@"%ld",
+                                                     (long)method_meta.error()]
+                 userInfo:nil];
+  }
+
+  const auto input_meta =
+      method_meta->input_tensor_meta([index unsignedLongValue]);
+  if (!input_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_input_shape_error"
+                   reason:[NSString
+                              stringWithFormat:@"%ld", (long)input_meta.error()]
+                 userInfo:nil];
+  }
+
+  const auto shape = input_meta->sizes();
+  NSMutableArray *nsShape = [[NSMutableArray alloc] init];
+
+  for (int i = 0; i < shape.size(); i++) {
+    [nsShape addObject:@(shape[i])];
+  }
+
+  return [nsShape copy];
+};
+
+- (NSNumber *)getNumberOfOutputs {
+  const auto method_meta = _model->method_meta("forward");
+  if (!method_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_number_of_outputs_error"
+                   reason:[NSString stringWithFormat:@"%ld",
+                                                     (long)method_meta.error()]
+                 userInfo:nil];
+  }
+
+  return @(method_meta->num_outputs());
+}
+
+- (NSNumber *)getOutputType:(NSNumber *)index {
+  const auto method_meta = _model->method_meta("forward");
+  if (!method_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_output_type_error"
+                   reason:[NSString stringWithFormat:@"%ld",
+                                                     (long)method_meta.error()]
+                 userInfo:nil];
+  }
+
+  const auto output_meta =
+      method_meta->output_tensor_meta([index unsignedLongValue]);
+  if (!output_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_output_type_error"
+                   reason:[NSString stringWithFormat:@"%ld",
+                                                     (long)output_meta.error()]
+                 userInfo:nil];
+  }
+
+  return scalarTypeToNSNumber(output_meta->scalar_type());
+};
+
+- (NSArray *)getOutputShape:(NSNumber *)index {
+  const auto method_meta = _model->method_meta("forward");
+  if (!method_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_output_shape_error"
+                   reason:[NSString stringWithFormat:@"%ld",
+                                                     (long)method_meta.error()]
+                 userInfo:nil];
+  }
+
+  const auto output_meta =
+      method_meta->output_tensor_meta([index unsignedLongValue]);
+  if (!output_meta.ok()) {
+    @throw [NSException
+        exceptionWithName:@"get_output_shape_error"
+                   reason:[NSString stringWithFormat:@"%ld",
+                                                     (long)output_meta.error()]
+                 userInfo:nil];
+  }
+
+  const auto shape = output_meta->sizes();
+  NSMutableArray *nsShape = [[NSMutableArray alloc] init];
+
+  for (int i = 0; i < shape.size(); i++) {
+    [nsShape addObject:@(shape[i])];
+  }
+
+  return [nsShape copy];
+};
+
+/**
+ * @brief Processes inputs through the forward pass of the model.
+ *
+ * This method takes input tensors, their corresponding shapes, and types,
+ * and performs a forward pass using _model. It supports both
+ * single and multiple inputs.
+ *
+ * @param inputs NSArray* of inputs where each element is an NSArray
+ *               representing the data for a tensor.
+ * @param shapes An array of shapes corresponding to the input tensors.
+ *               Each element is an NSArray of integers defining the dimensions.
+ * @param inputTypes An array of NSNumber objects representing the ScalarType of
+ *                   the input tensors
+ *
+ * @return An NSArray containing the results of the forward pass. Each element
+ *         represents the output of the corresponding input.
+ *
+ * @throws NSException Throws an exception with name "forward_error" if
+ *                     an error occurs during input processing or model
+ * execution.
+ *
+ * @warning Ensure that the inputs, shapes, and inputTypes arrays have the
+ *          same number of elements. Mismatched sizes can lead to runtime
+ *          errors.
+ **/
+- (NSArray *)execute:(NSString *)methodName
+              inputs:(NSArray *)inputs
+              shapes:(NSArray *)shapes
+          inputTypes:(NSArray *)inputTypes {
+  std::vector<EValue> inputTensors;
+  std::vector<TensorPtr> inputTensorPtrs;
+
+  for (NSUInteger i = 0; i < [inputTypes count]; i++) {
+    NSArray *inputShapeNSArray = [shapes objectAtIndex:i];
+
+    std::vector<int> inputShape = NSArrayToIntVector(inputShapeNSArray);
+    int inputType = [[inputTypes objectAtIndex:i] intValue];
+
+    NSArray *input = [inputs objectAtIndex:i];
+
+    TensorPtr currentTensor = NSArrayToTensorPtr(input, inputShape, inputType);
+    if (!currentTensor) {
+      throw [NSException
+          exceptionWithName:@"forward_error"
+                     reason:[NSString
+                                stringWithFormat:@"%d", Error::InvalidArgument]
+                   userInfo:nil];
+    }
+
+    // Since pushing back to inputTensors would cast to EValue (forward accepts
+    // a vector of EValues) We also push back to inputTensorPtrs to keep the
+    // underlying tensor alive. inputTensorPtrs vector retains shared ownership
+    // to prevent premature destruction
+    inputTensors.push_back(*currentTensor);
+    inputTensorPtrs.push_back(currentTensor);
+  }
+
+  Result result = _model->execute([methodName UTF8String], inputTensors);
+
+  if (!result.ok()) {
+    throw [NSException
+        exceptionWithName:@"forward_error"
+                   reason:[NSString stringWithFormat:@"%d", result.error()]
+                 userInfo:nil];
+  }
+
+  NSMutableArray *output = [NSMutableArray new];
+  for (int i = 0; i < result->size(); i++) {
+    auto currentResultTensor = result->at(i).toTensor();
+    NSArray *currentOutput = arrayToNsArray(
+        currentResultTensor.const_data_ptr(), currentResultTensor.numel(),
+        currentResultTensor.scalar_type());
+    [output addObject:currentOutput];
+  }
+  return output;
+}
+- (NSArray *)forward:(NSArray *)inputs
+              shapes:(NSArray *)shapes
+          inputTypes:(NSArray *)inputTypes {
+  return [self execute:@"forward"
+                inputs:inputs
+                shapes:shapes
+            inputTypes:inputTypes];
+}
+
+@end

package/third-party/ios/ExecutorchLib/ExecutorchLib/Exported/HuggingFaceTokenizer.h

@@ -0,0 +1,14 @@
+#import <Foundation/Foundation.h>
+
+@interface HuggingFaceTokenizer : NSObject
+
+- (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
+- (NSArray<NSNumber *> *)encode:(NSString *)text;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds
+    skipSpecialTokens:(BOOL)skipSpecialTokens;
+- (NSUInteger)getVocabSize;
+- (NSString *)idToToken:(NSInteger)tokenId;
+- (NSInteger)tokenToId:(NSString *)token;
+
+@end

package/third-party/ios/ExecutorchLib/ExecutorchLib/Exported/HuggingFaceTokenizer.mm

@@ -0,0 +1,80 @@
+#import "HuggingFaceTokenizer.h"
+#include <fstream>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <tokenizers-cpp/tokenizers_cpp.h>
+
+std::string loadBytesFromFile(const std::string &path) {
+  std::ifstream fs(path, std::ios::in | std::ios::binary);
+  if (fs.fail()) {
+    throw std::runtime_error("Failed to open tokenizer file");
+  }
+  std::string data;
+  fs.seekg(0, std::ios::end);
+  size_t size = static_cast<size_t>(fs.tellg());
+  fs.seekg(0, std::ios::beg);
+  data.resize(size);
+  fs.read(data.data(), size);
+  return data;
+}
+
+@implementation HuggingFaceTokenizer {
+  std::unique_ptr<tokenizers::Tokenizer> _tokenizer;
+}
+
+- (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath {
+  self = [super init];
+  if (self) {
+    auto blob = loadBytesFromFile([tokenizerPath UTF8String]);
+    _tokenizer = tokenizers::Tokenizer::FromBlobJSON(blob);
+  }
+  return self;
+}
+
+- (NSArray<NSNumber *> *)encode:(NSString *)text {
+  std::vector<int32_t> result = _tokenizer->Encode([text UTF8String]);
+  NSMutableArray<NSNumber *> *encodedResult =
+      [[NSMutableArray alloc] initWithCapacity:result.size()];
+  for (int32_t tokenId : result) {
+    [encodedResult addObject:@(tokenId)];
+  }
+
+  return encodedResult;
+}
+
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds {
+  return [self decode:tokenIds skipSpecialTokens:NO];
+}
+
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds
+    skipSpecialTokens:(BOOL)skipSpecialTokens {
+  std::vector<int32_t> stdTokenIds;
+  stdTokenIds.reserve([tokenIds count]);
+  for (NSNumber *tokenId in tokenIds) {
+    stdTokenIds.push_back([tokenId intValue]);
+  }
+  std::string decodedString =
+      _tokenizer->Decode(stdTokenIds, skipSpecialTokens);
+  return [NSString stringWithUTF8String:decodedString.c_str()];
+}
+
+- (NSUInteger)getVocabSize {
+  return (NSUInteger)_tokenizer->GetVocabSize();
+}
+
+- (NSString *)idToToken:(NSInteger)tokenId {
+  std::string token = _tokenizer->IdToToken(static_cast<int32_t>(tokenId));
+  return [NSString stringWithUTF8String:token.c_str()];
+}
+
+- (NSInteger)tokenToId:(NSString *)token {
+  std::string stdToken = [token UTF8String];
+  return (NSInteger)_tokenizer->TokenToId(stdToken);
+}
+
+- (void)dealloc {
+  _tokenizer.reset();
+}
+
+@end