react-native-executorch 0.4.8 → 0.5.0

package/common/rnexecutorch/models/image_segmentation/ImageSegmentation.cpp

@@ -0,0 +1,173 @@
+#include "ImageSegmentation.h"
+
+#include <future>
+
+#include <executorch/extension/tensor/tensor.h>
+
+#include <rnexecutorch/data_processing/ImageProcessing.h>
+#include <rnexecutorch/data_processing/Numerical.h>
+#include <rnexecutorch/host_objects/JsiConversions.h>
+#include <rnexecutorch/models/image_segmentation/Constants.h>
+
+namespace rnexecutorch {
+
+ImageSegmentation::ImageSegmentation(
+    const std::string &modelSource,
+    std::shared_ptr<react::CallInvoker> callInvoker)
+    : BaseModel(modelSource, callInvoker) {
+  auto inputShapes = getAllInputShapes();
+  if (inputShapes.size() == 0) {
+    throw std::runtime_error("Model seems to not take any input tensors.");
+  }
+  std::vector<int32_t> modelInputShape = inputShapes[0];
+  if (modelInputShape.size() < 2) {
+    char errorMessage[100];
+    std::snprintf(errorMessage, sizeof(errorMessage),
+                  "Unexpected model input size, expected at least 2 dimentions "
+                  "but got: %zu.",
+                  modelInputShape.size());
+    throw std::runtime_error(errorMessage);
+  }
+  modelImageSize = cv::Size(modelInputShape[modelInputShape.size() - 1],
+                            modelInputShape[modelInputShape.size() - 2]);
+  numModelPixels = modelImageSize.area();
+}
+
+std::shared_ptr<jsi::Object> ImageSegmentation::generate(
+    std::string imageSource,
+    std::set<std::string, std::less<>> classesOfInterest, bool resize) {
+  auto [inputTensor, originalSize] =
+      imageprocessing::readImageToTensor(imageSource, getAllInputShapes()[0]);
+
+  auto forwardResult = BaseModel::forward(inputTensor);
+  if (!forwardResult.ok()) {
+    throw std::runtime_error(
+        "Failed to forward, error: " +
+        std::to_string(static_cast<uint32_t>(forwardResult.error())));
+  }
+
+  return postprocess(forwardResult->at(0).toTensor(), originalSize,
+                     classesOfInterest, resize);
+}
+
+std::shared_ptr<jsi::Object> ImageSegmentation::postprocess(
+    const Tensor &tensor, cv::Size originalSize,
+    std::set<std::string, std::less<>> classesOfInterest, bool resize) {
+
+  auto dataPtr = static_cast<const float *>(tensor.const_data_ptr());
+  auto resultData = std::span(dataPtr, tensor.numel());
+
+  // We copy the ET-owned data to jsi array buffers that can be directly
+  // returned to JS
+  std::vector<std::shared_ptr<OwningArrayBuffer>> resultClasses;
+  resultClasses.reserve(numClasses);
+  for (std::size_t cl = 0; cl < numClasses; ++cl) {
+    auto classBuffer =
+        std::make_shared<OwningArrayBuffer>(numModelPixels * sizeof(float));
+    resultClasses.push_back(classBuffer);
+    std::memcpy(classBuffer->data(), &resultData[cl * numModelPixels],
+                numModelPixels * sizeof(float));
+  }
+
+  // Apply softmax per each pixel across all classes
+  for (std::size_t pixel = 0; pixel < numModelPixels; ++pixel) {
+    std::vector<float> classValues(numClasses);
+    for (std::size_t cl = 0; cl < numClasses; ++cl) {
+      classValues[cl] =
+          reinterpret_cast<float *>(resultClasses[cl]->data())[pixel];
+    }
+    numerical::softmax(classValues);
+    for (std::size_t cl = 0; cl < numClasses; ++cl) {
+      reinterpret_cast<float *>(resultClasses[cl]->data())[pixel] =
+          classValues[cl];
+    }
+  }
+
+  // Calculate the maximum class for each pixel
+  auto argmax =
+      std::make_shared<OwningArrayBuffer>(numModelPixels * sizeof(int32_t));
+  for (std::size_t pixel = 0; pixel < numModelPixels; ++pixel) {
+    float max = reinterpret_cast<float *>(resultClasses[0]->data())[pixel];
+    int maxInd = 0;
+    for (int cl = 1; cl < numClasses; ++cl) {
+      if (reinterpret_cast<float *>(resultClasses[cl]->data())[pixel] > max) {
+        maxInd = cl;
+        max = reinterpret_cast<float *>(resultClasses[cl]->data())[pixel];
+      }
+    }
+    reinterpret_cast<int32_t *>(argmax->data())[pixel] = maxInd;
+  }
+
+  auto buffersToReturn = std::make_shared<std::unordered_map<
+      std::string_view, std::shared_ptr<OwningArrayBuffer>>>();
+  for (std::size_t cl = 0; cl < numClasses; ++cl) {
+    if (classesOfInterest.contains(DEEPLABV3_RESNET50_LABELS[cl])) {
+      (*buffersToReturn)[DEEPLABV3_RESNET50_LABELS[cl]] = resultClasses[cl];
+    }
+  }
+
+  // Resize selected classes and argmax
+  if (resize) {
+    cv::Mat argmaxMat(modelImageSize, CV_32SC1, argmax->data());
+    cv::resize(argmaxMat, argmaxMat, originalSize, 0, 0,
+               cv::InterpolationFlags::INTER_NEAREST);
+    argmax = std::make_shared<OwningArrayBuffer>(originalSize.area() *
+                                                 sizeof(int32_t));
+    std::memcpy(argmax->data(), argmaxMat.data,
+                originalSize.area() * sizeof(int32_t));
+
+    for (auto &[label, arrayBuffer] : *buffersToReturn) {
+      cv::Mat classMat(modelImageSize, CV_32FC1, arrayBuffer->data());
+      cv::resize(classMat, classMat, originalSize);
+      arrayBuffer = std::make_shared<OwningArrayBuffer>(originalSize.area() *
+                                                        sizeof(float));
+      std::memcpy(arrayBuffer->data(), classMat.data,
+                  originalSize.area() * sizeof(float));
+    }
+  }
+  return populateDictionary(argmax, buffersToReturn);
+}
+
+std::shared_ptr<jsi::Object> ImageSegmentation::populateDictionary(
+    std::shared_ptr<OwningArrayBuffer> argmax,
+    std::shared_ptr<std::unordered_map<std::string_view,
+                                       std::shared_ptr<OwningArrayBuffer>>>
+        classesToOutput) {
+  // Synchronize the invoked thread to return when the dict is constructed
+  auto promisePtr = std::make_shared<std::promise<void>>();
+  std::future<void> doneFuture = promisePtr->get_future();
+
+  std::shared_ptr<jsi::Object> dictPtr = nullptr;
+  callInvoker->invokeAsync(
+      [argmax, classesToOutput, &dictPtr, promisePtr](jsi::Runtime &runtime) {
+        dictPtr = std::make_shared<jsi::Object>(runtime);
+        auto argmaxArrayBuffer = jsi::ArrayBuffer(runtime, argmax);
+
+        auto int32ArrayCtor =
+            runtime.global().getPropertyAsFunction(runtime, "Int32Array");
+        auto int32Array =
+            int32ArrayCtor.callAsConstructor(runtime, argmaxArrayBuffer)
+                .getObject(runtime);
+        dictPtr->setProperty(runtime, "ARGMAX", int32Array);
+
+        for (auto &[classLabel, owningBuffer] : *classesToOutput) {
+          auto classArrayBuffer = jsi::ArrayBuffer(runtime, owningBuffer);
+
+          auto float32ArrayCtor =
+              runtime.global().getPropertyAsFunction(runtime, "Float32Array");
+          auto float32Array =
+              float32ArrayCtor.callAsConstructor(runtime, classArrayBuffer)
+                  .getObject(runtime);
+
+          dictPtr->setProperty(
+              runtime, jsi::String::createFromAscii(runtime, classLabel.data()),
+              float32Array);
+        }
+        promisePtr->set_value();
+      });
+
+  doneFuture.wait();
+  return dictPtr;
+}
+
+} // namespace rnexecutorch

package/common/rnexecutorch/models/image_segmentation/ImageSegmentation.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <set>
+#include <utility>
+
+#include <executorch/extension/tensor/tensor_ptr.h>
+#include <jsi/jsi.h>
+#include <opencv2/opencv.hpp>
+
+#include <rnexecutorch/jsi/OwningArrayBuffer.h>
+#include <rnexecutorch/models/BaseModel.h>
+#include <rnexecutorch/models/image_segmentation/Constants.h>
+
+namespace rnexecutorch {
+using namespace facebook;
+
+using executorch::aten::Tensor;
+using executorch::extension::TensorPtr;
+
+class ImageSegmentation : public BaseModel {
+public:
+  ImageSegmentation(const std::string &modelSource,
+                    std::shared_ptr<react::CallInvoker> callInvoker);
+  std::shared_ptr<jsi::Object>
+  generate(std::string imageSource,
+           std::set<std::string, std::less<>> classesOfInterest, bool resize);
+
+private:
+  std::shared_ptr<jsi::Object>
+  postprocess(const Tensor &tensor, cv::Size originalSize,
+              std::set<std::string, std::less<>> classesOfInterest,
+              bool resize);
+  std::shared_ptr<jsi::Object> populateDictionary(
+      std::shared_ptr<OwningArrayBuffer> argmax,
+      std::shared_ptr<std::unordered_map<std::string_view,
+                                         std::shared_ptr<OwningArrayBuffer>>>
+          classesToOutput);
+
+  static constexpr std::size_t numClasses{DEEPLABV3_RESNET50_LABELS.size()};
+  cv::Size modelImageSize;
+  std::size_t numModelPixels;
+};
+} // namespace rnexecutorch

package/{ios/RnExecutorch/utils/Constants.mm → common/rnexecutorch/models/object_detection/Constants.h}

@@ -1,6 +1,12 @@
-#include "Constants.h"
+#pragma once
 
-const std::unordered_map<int, std::string> cocoLabelsMap = {
+#include <string>
+#include <unordered_map>
+
+namespace rnexecutorch {
+inline constexpr float IOU_THRESHOLD = 0.55;
+
+inline const std::unordered_map<int, std::string> COCO_LABELS_MAP = {
     {1, "PERSON"},         {2, "BICYCLE"},       {3, "CAR"},
     {4, "MOTORCYCLE"},     {5, "AIRPLANE"},      {6, "BUS"},
     {7, "TRAIN"},          {8, "TRUCK"},         {9, "BOAT"},
@@ -32,3 +38,4 @@ const std::unordered_map<int, std::string> cocoLabelsMap = {
     {85, "CLOCK"},         {86, "VASE"},         {87, "SCISSORS"},
     {88, "TEDDY_BEAR"},    {89, "HAIR_DRIER"},   {90, "TOOTHBRUSH"},
     {91, "HAIR_BRUSH"}};
+} // namespace rnexecutorch

package/common/rnexecutorch/models/object_detection/ObjectDetection.cpp

@@ -0,0 +1,82 @@
+#include "ObjectDetection.h"
+
+#include <rnexecutorch/data_processing/ImageProcessing.h>
+
+namespace rnexecutorch {
+
+ObjectDetection::ObjectDetection(
+    const std::string &modelSource,
+    std::shared_ptr<react::CallInvoker> callInvoker)
+    : BaseModel(modelSource, callInvoker) {
+  auto inputTensors = getAllInputShapes();
+  if (inputTensors.size() == 0) {
+    throw std::runtime_error("Model seems to not take any input tensors.");
+  }
+  std::vector<int32_t> modelInputShape = inputTensors[0];
+  if (modelInputShape.size() < 2) {
+    char errorMessage[100];
+    std::snprintf(errorMessage, sizeof(errorMessage),
+                  "Unexpected model input size, expected at least 2 dimentions "
+                  "but got: %zu.",
+                  modelInputShape.size());
+    throw std::runtime_error(errorMessage);
+  }
+  modelImageSize = cv::Size(modelInputShape[modelInputShape.size() - 1],
+                            modelInputShape[modelInputShape.size() - 2]);
+}
+
+std::vector<Detection>
+ObjectDetection::postprocess(const std::vector<EValue> &tensors,
+                             cv::Size originalSize, double detectionThreshold) {
+  float widthRatio =
+      static_cast<float>(originalSize.width) / modelImageSize.width;
+  float heightRatio =
+      static_cast<float>(originalSize.height) / modelImageSize.height;
+
+  std::vector<Detection> detections;
+  auto bboxTensor = tensors.at(0).toTensor();
+  std::span<const float> bboxes(
+      static_cast<const float *>(bboxTensor.const_data_ptr()),
+      bboxTensor.numel());
+
+  auto scoreTensor = tensors.at(1).toTensor();
+  std::span<const float> scores(
+      static_cast<const float *>(scoreTensor.const_data_ptr()),
+      scoreTensor.numel());
+
+  auto labelTensor = tensors.at(2).toTensor();
+  std::span<const float> labels(
+      static_cast<const float *>(labelTensor.const_data_ptr()),
+      labelTensor.numel());
+
+  for (std::size_t i = 0; i < scores.size(); ++i) {
+    if (scores[i] < detectionThreshold) {
+      continue;
+    }
+    float x1 = bboxes[i * 4] * widthRatio;
+    float y1 = bboxes[i * 4 + 1] * heightRatio;
+    float x2 = bboxes[i * 4 + 2] * widthRatio;
+    float y2 = bboxes[i * 4 + 3] * heightRatio;
+    detections.emplace_back(x1, y1, x2, y2, static_cast<int>(labels[i]),
+                            scores[i]);
+  }
+
+  std::vector<Detection> output = nonMaxSuppression(detections);
+  return output;
+}
+
+std::vector<Detection> ObjectDetection::generate(std::string imageSource,
+                                                 double detectionThreshold) {
+  auto [inputTensor, originalSize] =
+      imageprocessing::readImageToTensor(imageSource, getAllInputShapes()[0]);
+
+  auto forwardResult = BaseModel::forward(inputTensor);
+  if (!forwardResult.ok()) {
+    throw std::runtime_error(
+        "Failed to forward, error: " +
+        std::to_string(static_cast<uint32_t>(forwardResult.error())));
+  }
+
+  return postprocess(forwardResult.get(), originalSize, detectionThreshold);
+}
+} // namespace rnexecutorch

package/common/rnexecutorch/models/object_detection/ObjectDetection.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <unordered_map>
+
+#include <executorch/extension/tensor/tensor_ptr.h>
+#include <executorch/runtime/core/evalue.h>
+#include <opencv2/opencv.hpp>
+
+#include <rnexecutorch/models/BaseModel.h>
+#include <rnexecutorch/models/object_detection/Utils.h>
+
+namespace rnexecutorch {
+using executorch::extension::TensorPtr;
+using executorch::runtime::EValue;
+
+class ObjectDetection : public BaseModel {
+public:
+  ObjectDetection(const std::string &modelSource,
+                  std::shared_ptr<react::CallInvoker> callInvoker);
+  std::vector<Detection> generate(std::string imageSource,
+                                  double detectionThreshold);
+
+private:
+  std::vector<Detection> postprocess(const std::vector<EValue> &tensors,
+                                     cv::Size originalSize,
+                                     double detectionThreshold);
+
+  cv::Size modelImageSize{0, 0};
+};
+
+} // namespace rnexecutorch

package/{ios/RnExecutorch/utils/ObjectDetectionUtils.mm → common/rnexecutorch/models/object_detection/Utils.cpp}

@@ -1,30 +1,8 @@
-#include "ObjectDetectionUtils.hpp"
+#include "Utils.h"
 #include "Constants.h"
 
-NSString *floatLabelToNSString(float label) {
-  int intLabel = static_cast<int>(label);
-  auto it = cocoLabelsMap.find(intLabel);
-  if (it != cocoLabelsMap.end()) {
-    return [NSString stringWithUTF8String:it->second.c_str()];
-  } else {
-    return [NSString stringWithUTF8String:"unknown"];
-  }
-}
-
-NSDictionary *detectionToNSDictionary(const Detection &detection) {
-  return @{
-    @"bbox" : @{
-      @"x1" : @(detection.x1),
-      @"y1" : @(detection.y1),
-      @"x2" : @(detection.x2),
-      @"y2" : @(detection.y2),
-    },
-    @"label" : floatLabelToNSString(detection.label),
-    @"score" : @(detection.score)
-  };
-}
-
-float iou(const Detection &a, const Detection &b) {
+namespace rnexecutorch {
+float intersectionOverUnion(const Detection &a, const Detection &b) {
   float x1 = std::max(a.x1, b.x1);
   float y1 = std::max(a.y1, b.y1);
   float x2 = std::min(a.x2, b.x2);
@@ -36,10 +14,9 @@ float iou(const Detection &a, const Detection &b) {
   float unionArea = areaA + areaB - intersectionArea;
 
   return intersectionArea / unionArea;
-};
+}
 
-std::vector<Detection> nms(std::vector<Detection> detections,
-                           float iouThreshold) {
+std::vector<Detection> nonMaxSuppression(std::vector<Detection> detections) {
   if (detections.empty()) {
     return {};
   }
@@ -70,8 +47,9 @@ std::vector<Detection> nms(std::vector<Detection> detections,
       filteredLabelDetections.push_back(current);
       labelDetections.erase(
           std::remove_if(labelDetections.begin(), labelDetections.end(),
-                         [&](const Detection &other) {
-                           return iou(current, other) > iouThreshold;
+                         [&current](const Detection &other) {
+                           return intersectionOverUnion(current, other) >
+                                  IOU_THRESHOLD;
                          }),
           labelDetections.end());
     }
@@ -80,3 +58,5 @@ std::vector<Detection> nms(std::vector<Detection> detections,
   }
   return result;
 }
+
+} // namespace rnexecutorch

package/common/rnexecutorch/models/object_detection/Utils.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <vector>
+
+namespace rnexecutorch {
+struct Detection {
+  float x1;
+  float y1;
+  float x2;
+  float y2;
+  int label;
+  float score;
+};
+
+float intersectionOverUnion(const Detection &a, const Detection &b);
+std::vector<Detection> nonMaxSuppression(std::vector<Detection> detections);
+} // namespace rnexecutorch

package/common/rnexecutorch/models/ocr/CTCLabelConverter.cpp

@@ -0,0 +1,88 @@
+#include "CTCLabelConverter.h"
+#include <algorithm>
+#include <optional>
+
+namespace rnexecutorch::ocr {
+CTCLabelConverter::CTCLabelConverter(const std::string &characters)
+    : ignoreIdx(0),
+      character({"[blank]"}) // blank character is ignored character (index 0).
+{
+  for (size_t i = 0; i < characters.length();) {
+    size_t char_len = 0;
+    unsigned char first_byte = characters[i];
+
+    if ((first_byte & 0x80) == 0) { // 0xxxxxxx -> 1-byte character
+      char_len = 1;
+    } else if ((first_byte & 0xE0) == 0xC0) { // 110xxxxx -> 2-byte character
+      char_len = 2;
+    } else if ((first_byte & 0xF0) == 0xE0) { // 1110xxxx -> 3-byte character
+      char_len = 3;
+    } else if ((first_byte & 0xF8) == 0xF0) { // 11110xxx -> 4-byte character
+      char_len = 4;
+    } else {
+      // Invalid UTF-8 start byte, treat as a single byte character to avoid
+      // infinite loop
+      char_len = 1;
+    }
+
+    // Ensure we don't read past the end of the string
+    if (i + char_len <= characters.length()) {
+      character.push_back(characters.substr(i, char_len));
+    }
+    i += char_len;
+  }
+}
+
+std::vector<std::string>
+CTCLabelConverter::decodeGreedy(const std::vector<int32_t> &textIndex,
+                                size_t length) {
+  /*
+   The current strategy used for decoding is greedy approach
+   which iterates through the list of indices and process
+   each index using following steps:
+    1. Ignore if idx == 0
+    2. Ignore if idx is the same as last idx
+    3. decode idx -> char and append it to returned text.
+
+   Note that ignoring repeated indices, does not mean decoding
+   won't handle repeated letters in a word, since in most cases
+   actual chars are already seperated by blank tokens.
+  */
+  std::vector<std::string> texts;
+  size_t index = 0;
+
+  while (index < textIndex.size()) {
+    size_t segmentLength = std::min(length, textIndex.size() - index);
+
+    std::vector<int32_t> subArray(textIndex.begin() + index,
+                                  textIndex.begin() + index + segmentLength);
+
+    std::string text;
+
+    if (!subArray.empty()) {
+      std::optional<int32_t> lastChar;
+      for (int32_t currentChar : subArray) {
+        bool isRepeated =
+            lastChar.has_value() && lastChar.value() == currentChar;
+        bool isIgnored = currentChar == ignoreIdx;
+        lastChar = currentChar;
+
+        if (currentChar >= 0 &&
+            currentChar < static_cast<int32_t>(character.size()) &&
+            !isRepeated && !isIgnored) {
+          text += character[currentChar];
+        }
+      }
+    }
+
+    texts.push_back(std::move(text));
+    index += segmentLength;
+
+    if (segmentLength < length) {
+      break;
+    }
+  }
+
+  return texts;
+}
+} // namespace rnexecutorch::ocr

package/common/rnexecutorch/models/ocr/CTCLabelConverter.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace rnexecutorch::ocr {
+/*
+ CTC (Connectionist Temporal Classification) Label Converter
+ is used for decoding the returned list of indices by Recognizer into
+ actual characters.
+ For each Recognizer there is an 1:1 correspondence between
+ an index and a character. CTC Label Converter operates on this
+ mapping. Symbol corresponding to the first index is a [blank]
+ character, meaning "no character to decode here".
+ The decoder ignores [blank] char.
+*/
+
+class CTCLabelConverter final {
+public:
+  explicit CTCLabelConverter(const std::string &characters);
+
+  std::vector<std::string> decodeGreedy(const std::vector<int32_t> &textIndex,
+                                        size_t length);
+
+private:
+  std::vector<std::string> character;
+  int32_t ignoreIdx;
+};
+} // namespace rnexecutorch::ocr

package/common/rnexecutorch/models/ocr/Constants.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <cstdint>
+#include <opencv2/opencv.hpp>
+
+namespace rnexecutorch::ocr {
+
+inline constexpr float TEXT_THRESHOLD = 0.4;
+inline constexpr float TEXT_THRESHOLD_VERTICAL = 0.3;
+inline constexpr float LINK_THRESHOLD = 0.4;
+inline constexpr float LOW_TEXT_THRESHOLD = 0.7;
+inline constexpr float CENTER_THRESHOLD = 0.5;
+inline constexpr float DISTANCE_THRESHOLD = 2.0;
+inline constexpr float HEIGHT_THRESHOLD = 2.0;
+inline constexpr float SINGLE_CHARACTER_CENTER_THRESHOLD = 0.3;
+inline constexpr float LOW_CONFIDENCE_THRESHOLD = 0.3;
+inline constexpr float ADJUST_CONTRAST = 0.2;
+inline constexpr int32_t MIN_SIDE_THRESHOLD = 15;
+inline constexpr int32_t MAX_SIDE_THRESHOLD = 30;
+inline constexpr int32_t RECOGNIZER_HEIGHT = 64;
+inline constexpr int32_t LARGE_RECOGNIZER_WIDTH = 512;
+inline constexpr int32_t MEDIUM_RECOGNIZER_WIDTH = 256;
+inline constexpr int32_t SMALL_RECOGNIZER_WIDTH = 128;
+inline constexpr int32_t SMALL_VERTICAL_RECOGNIZER_WIDTH = 64;
+inline constexpr int32_t MAX_WIDTH =
+    LARGE_RECOGNIZER_WIDTH + (LARGE_RECOGNIZER_WIDTH * 0.15);
+inline constexpr int32_t SINGLE_CHARACTER_MIN_SIZE = 70;
+inline constexpr int32_t RECOGNIZER_IMAGE_SIZE = 1280;
+inline constexpr int32_t VERTICAL_LINE_THRESHOLD = 20;
+
+inline const cv::Scalar MEAN(0.485, 0.456, 0.406);
+inline const cv::Scalar VARIANCE(0.229, 0.224, 0.225);
+
+} // namespace rnexecutorch::ocr