react-native-executorch 0.4.10 → 0.5.0-nightly-6f4dd53-20251211

package/common/rnexecutorch/models/text_to_image/TextToImage.cpp

@@ -0,0 +1,141 @@
+#include "TextToImage.h"
+
+#include <cmath>
+#include <random>
+#include <span>
+
+#include <executorch/extension/tensor/tensor.h>
+
+#include <rnexecutorch/Log.h>
+#include <rnexecutorch/models/text_to_image/Constants.h>
+
+namespace rnexecutorch::models::text_to_image {
+
+using namespace executorch::extension;
+
+TextToImage::TextToImage(const std::string &tokenizerSource,
+                         const std::string &encoderSource,
+                         const std::string &unetSource,
+                         const std::string &decoderSource,
+                         float schedulerBetaStart, float schedulerBetaEnd,
+                         int32_t schedulerNumTrainTimesteps,
+                         int32_t schedulerStepsOffset,
+                         std::shared_ptr<react::CallInvoker> callInvoker)
+    : callInvoker(callInvoker),
+      scheduler(std::make_unique<Scheduler>(
+          schedulerBetaStart, schedulerBetaEnd, schedulerNumTrainTimesteps,
+          schedulerStepsOffset, callInvoker)),
+      encoder(std::make_unique<Encoder>(tokenizerSource, encoderSource,
+                                        callInvoker)),
+      unet(std::make_unique<UNet>(unetSource, callInvoker)),
+      decoder(std::make_unique<Decoder>(decoderSource, callInvoker)) {}
+
+void TextToImage::setImageSize(int32_t imageSize) {
+  if (imageSize % 32 != 0) {
+    throw std::runtime_error("Image size must be a multiple of 32.");
+  }
+  this->imageSize = imageSize;
+  constexpr int32_t latentDownsample = 8;
+  latentImageSize = std::floor(imageSize / latentDownsample);
+  unet->latentImageSize = latentImageSize;
+  decoder->latentImageSize = latentImageSize;
+}
+
+void TextToImage::setSeed(int32_t &seed) {
+  // Seed argument is provided
+  if (seed >= 0) {
+    return;
+  }
+  std::random_device rd;
+  seed = rd();
+}
+
+std::shared_ptr<OwningArrayBuffer>
+TextToImage::generate(std::string input, int32_t imageSize,
+                      size_t numInferenceSteps, int32_t seed,
+                      std::shared_ptr<jsi::Function> callback) {
+  setImageSize(imageSize);
+  setSeed(seed);
+
+  std::vector<float> embeddings = encoder->generate(input);
+  std::vector<int32_t> embeddingsShape = {2, 77, 768};
+  auto embeddingsTensor =
+      make_tensor_ptr(embeddingsShape, embeddings.data(), ScalarType::Float);
+
+  int32_t latentsSize = numChannels * latentImageSize * latentImageSize;
+  std::vector<float> latents(latentsSize);
+  std::mt19937 gen(seed);
+  std::normal_distribution<float> dist(0.0, 1.0);
+  for (auto &val : latents) {
+    val = dist(gen);
+  }
+
+  scheduler->setTimesteps(numInferenceSteps);
+  std::vector<int32_t> timesteps = scheduler->timesteps;
+
+  auto nativeCallback = [this, callback](size_t stepIdx) {
+    this->callInvoker->invokeAsync([callback, stepIdx](jsi::Runtime &runtime) {
+      callback->call(runtime, jsi::Value(static_cast<int32_t>(stepIdx)));
+    });
+  };
+  for (size_t t = 0; t < numInferenceSteps + 1 && !interrupted; t++) {
+    log(LOG_LEVEL::Debug, "Step:", t, "/", numInferenceSteps);
+
+    std::vector<float> noisePred =
+        unet->generate(latents, timesteps[t], embeddingsTensor);
+
+    size_t noiseSize = noisePred.size() / 2;
+    std::span<const float> noisePredSpan{noisePred};
+    std::span<const float> noiseUncond = noisePredSpan.subspan(0, noiseSize);
+    std::span<const float> noiseText =
+        noisePredSpan.subspan(noiseSize, noiseSize);
+    std::vector<float> noise(noiseSize);
+    for (size_t i = 0; i < noiseSize; i++) {
+      noise[i] =
+          noiseUncond[i] * (1 - guidanceScale) + noiseText[i] * guidanceScale;
+    }
+    latents = scheduler->step(latents, noise, timesteps[t]);
+
+    nativeCallback(t);
+  }
+  if (interrupted) {
+    interrupted = false;
+    return std::make_shared<OwningArrayBuffer>(0);
+  }
+
+  for (auto &val : latents) {
+    val /= latentsScale;
+  }
+
+  std::vector<float> output = decoder->generate(latents);
+  return postprocess(output);
+}
+
+std::shared_ptr<OwningArrayBuffer>
+TextToImage::postprocess(const std::vector<float> &output) const {
+  // Convert RGB to RGBA
+  int32_t imagePixelCount = imageSize * imageSize;
+  std::vector<uint8_t> outputRgba(imagePixelCount * 4);
+  for (int32_t i = 0; i < imagePixelCount; i++) {
+    outputRgba[i * 4 + 0] = output[i * 3 + 0];
+    outputRgba[i * 4 + 1] = output[i * 3 + 1];
+    outputRgba[i * 4 + 2] = output[i * 3 + 2];
+    outputRgba[i * 4 + 3] = 255;
+  }
+  return std::make_shared<OwningArrayBuffer>(outputRgba);
+}
+
+void TextToImage::interrupt() noexcept { interrupted = true; }
+
+size_t TextToImage::getMemoryLowerBound() const noexcept {
+  return encoder->getMemoryLowerBound() + unet->getMemoryLowerBound() +
+         decoder->getMemoryLowerBound();
+}
+
+void TextToImage::unload() noexcept {
+  encoder->unload();
+  unet->unload();
+  decoder->unload();
+}
+
+} // namespace rnexecutorch::models::text_to_image

package/common/rnexecutorch/models/text_to_image/TextToImage.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <ReactCommon/CallInvoker.h>
+#include <jsi/jsi.h>
+
+#include <rnexecutorch/jsi/OwningArrayBuffer.h>
+#include <rnexecutorch/metaprogramming/ConstructorHelpers.h>
+
+#include <rnexecutorch/models/text_to_image/Decoder.h>
+#include <rnexecutorch/models/text_to_image/Encoder.h>
+#include <rnexecutorch/models/text_to_image/Scheduler.h>
+#include <rnexecutorch/models/text_to_image/UNet.h>
+
+namespace rnexecutorch {
+namespace models::text_to_image {
+using namespace facebook;
+
+class TextToImage final {
+public:
+  explicit TextToImage(const std::string &tokenizerSource,
+                       const std::string &encoderSource,
+                       const std::string &unetSource,
+                       const std::string &decoderSource,
+                       float schedulerBetaStart, float schedulerBetaEnd,
+                       int32_t schedulerNumTrainTimesteps,
+                       int32_t schedulerStepsOffset,
+                       std::shared_ptr<react::CallInvoker> callInvoker);
+  std::shared_ptr<OwningArrayBuffer>
+  generate(std::string input, int32_t imageSize, size_t numInferenceSteps,
+           int32_t seed, std::shared_ptr<jsi::Function> callback);
+  void interrupt() noexcept;
+  size_t getMemoryLowerBound() const noexcept;
+  void unload() noexcept;
+
+private:
+  void setImageSize(int32_t imageSize);
+  void setSeed(int32_t &seed);
+  std::shared_ptr<OwningArrayBuffer>
+  postprocess(const std::vector<float> &output) const;
+
+  size_t memorySizeLowerBound;
+  int32_t imageSize;
+  int32_t latentImageSize;
+  static constexpr int32_t numChannels = 4;
+  static constexpr float guidanceScale = 7.5f;
+  static constexpr float latentsScale = 0.18215f;
+  bool interrupted = false;
+
+  std::shared_ptr<react::CallInvoker> callInvoker;
+  std::unique_ptr<Scheduler> scheduler;
+  std::unique_ptr<Encoder> encoder;
+  std::unique_ptr<UNet> unet;
+  std::unique_ptr<Decoder> decoder;
+};
+} // namespace models::text_to_image
+
+REGISTER_CONSTRUCTOR(models::text_to_image::TextToImage, std::string,
+                     std::string, std::string, std::string, float, float,
+                     int32_t, int32_t, std::shared_ptr<react::CallInvoker>);
+} // namespace rnexecutorch

package/common/rnexecutorch/models/text_to_image/UNet.cpp

@@ -0,0 +1,38 @@
+#include "UNet.h"
+
+namespace rnexecutorch::models::text_to_image {
+
+using namespace executorch::extension;
+
+UNet::UNet(const std::string &modelSource,
+           std::shared_ptr<react::CallInvoker> callInvoker)
+    : BaseModel(modelSource, callInvoker) {}
+
+std::vector<float> UNet::generate(std::vector<float> &latents, int32_t timestep,
+                                  TensorPtr &embeddingsTensor) const {
+  std::vector<float> latentsConcat;
+  latentsConcat.reserve(2 * latentImageSize);
+  latentsConcat.insert(latentsConcat.end(), latents.begin(), latents.end());
+  latentsConcat.insert(latentsConcat.end(), latents.begin(), latents.end());
+
+  std::vector<int32_t> latentsShape = {2, numChannels, latentImageSize,
+                                       latentImageSize};
+
+  auto timestepTensor =
+      make_tensor_ptr<int64_t>({static_cast<int64_t>(timestep)});
+  auto latentsTensor =
+      make_tensor_ptr(latentsShape, latentsConcat.data(), ScalarType::Float);
+
+  auto forwardResult =
+      BaseModel::forward({latentsTensor, timestepTensor, embeddingsTensor});
+  if (!forwardResult.ok()) {
+    throw std::runtime_error(
+        "Function forward in UNet failed with error code: " +
+        std::to_string(static_cast<uint32_t>(forwardResult.error())));
+  }
+
+  auto forwardResultTensor = forwardResult->at(0).toTensor();
+  const auto *dataPtr = forwardResultTensor.const_data_ptr<float>();
+  return {dataPtr, dataPtr + forwardResultTensor.numel()};
+}
+} // namespace rnexecutorch::models::text_to_image

package/common/rnexecutorch/models/text_to_image/UNet.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <executorch/extension/tensor/tensor.h>
+
+#include <ReactCommon/CallInvoker.h>
+#include <rnexecutorch/models/BaseModel.h>
+
+namespace rnexecutorch::models::text_to_image {
+
+using namespace executorch::extension;
+
+class UNet final : public BaseModel {
+public:
+  explicit UNet(const std::string &modelSource,
+                std::shared_ptr<react::CallInvoker> callInvoker);
+  std::vector<float> generate(std::vector<float> &latents, int32_t timestep,
+                              TensorPtr &embeddingsTensor) const;
+
+  int32_t latentImageSize;
+
+private:
+  static constexpr int32_t numChannels = 4;
+};
+} // namespace rnexecutorch::models::text_to_image

package/common/rnexecutorch/models/vertical_ocr/VerticalDetector.cpp

@@ -0,0 +1,93 @@
+#include "VerticalDetector.h"
+
+#include <rnexecutorch/data_processing/ImageProcessing.h>
+#include <rnexecutorch/models/ocr/Constants.h>
+#include <rnexecutorch/models/ocr/utils/DetectorUtils.h>
+
+#include <executorch/extension/tensor/tensor_ptr.h>
+
+namespace rnexecutorch::models::ocr {
+VerticalDetector::VerticalDetector(
+    const std::string &modelSource, bool detectSingleCharacters,
+    std::shared_ptr<react::CallInvoker> callInvoker)
+    : BaseModel(modelSource, callInvoker) {
+  this->detectSingleCharacters = detectSingleCharacters;
+  auto inputShapes = getAllInputShapes();
+  if (inputShapes.empty()) {
+    throw std::runtime_error(
+        "Detector model seems to not take any input tensors.");
+  }
+  std::vector<int32_t> modelInputShape = inputShapes[0];
+  if (modelInputShape.size() < 2) {
+    throw std::runtime_error("Unexpected detector model input size, expected "
+                             "at least 2 dimensions but got: " +
+                             std::to_string(modelInputShape.size()) + ".");
+  }
+  modelImageSize = cv::Size(modelInputShape[modelInputShape.size() - 1],
+                            modelInputShape[modelInputShape.size() - 2]);
+}
+
+cv::Size VerticalDetector::getModelImageSize() const noexcept {
+  return modelImageSize;
+}
+
+std::vector<types::DetectorBBox>
+VerticalDetector::generate(const cv::Mat &inputImage) {
+  auto inputShapes = getAllInputShapes();
+  cv::Mat resizedInputImage =
+      image_processing::resizePadded(inputImage, getModelImageSize());
+  TensorPtr inputTensor = image_processing::getTensorFromMatrix(
+      inputShapes[0], resizedInputImage, constants::kNormalizationMean,
+      constants::kNormalizationVariance);
+  auto forwardResult = BaseModel::forward(inputTensor);
+  if (!forwardResult.ok()) {
+    throw std::runtime_error(
+        "Failed to forward, error: " +
+        std::to_string(static_cast<uint32_t>(forwardResult.error())));
+  }
+  return postprocess(forwardResult->at(0).toTensor());
+}
+
+std::vector<types::DetectorBBox>
+VerticalDetector::postprocess(const Tensor &tensor) const {
+  /*
+   The output of the model consists of two matrices (heat maps):
+   1. ScoreText(Score map) - The probability of a region containing character.
+   2. ScoreAffinity(Affinity map) - affinity between characters, used to to
+   group each character into a single instance (sequence) Both matrices are
+   H/2xW/2.
+
+   The result of this step is a list of bounding boxes that contain text.
+   */
+  std::span<const float> tensorData(tensor.const_data_ptr<float>(),
+                                    tensor.numel());
+  /*
+   The output of the model is a matrix half the size of the input image
+   containing two channels representing the heatmaps.
+   */
+  auto [scoreTextMat, scoreAffinityMat] = utils::interleavedArrayToMats(
+      tensorData,
+      cv::Size(modelImageSize.width / 2, modelImageSize.height / 2));
+  float txtThreshold = this->detectSingleCharacters
+                           ? constants::kTextThreshold
+                           : constants::kTextThresholdVertical;
+  std::vector<types::DetectorBBox> bBoxesList =
+      utils::getDetBoxesFromTextMapVertical(
+          scoreTextMat, scoreAffinityMat, txtThreshold,
+          constants::kLinkThreshold, this->detectSingleCharacters);
+  const float restoreRatio = utils::calculateRestoreRatio(
+      scoreTextMat.rows, constants::kRecognizerImageSize);
+  utils::restoreBboxRatio(bBoxesList, restoreRatio);
+
+  // if this is Narrow Detector, do not group boxes.
+  if (!this->detectSingleCharacters) {
+    bBoxesList = utils::groupTextBoxes(
+        bBoxesList, constants::kCenterThreshold, constants::kDistanceThreshold,
+        constants::kHeightThreshold, constants::kMinSideThreshold,
+        constants::kMaxSideThreshold, constants::kMaxWidth);
+  }
+
+  return bBoxesList;
+}
+
+} // namespace rnexecutorch::models::ocr

package/common/rnexecutorch/models/vertical_ocr/VerticalDetector.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <executorch/extension/tensor/tensor_ptr.h>
+#include <opencv2/opencv.hpp>
+
+#include <rnexecutorch/models/BaseModel.h>
+#include <rnexecutorch/models/ocr/Types.h>
+
+namespace rnexecutorch::models::ocr {
+
+/*
+ Vertical Detector is an sligtly modified Detector tuned for detecting Vertical
+ text. For more details about standard detector, refer to the file
+ ocr/Detector.cpp.
+
+  In Vertical OCR pipeline we make use of Detector two times:
+
+  1. Large Detector -- The differences between Detector used in standard OCR and
+ Large Detector used in Vertical OCR is: a) To obtain detected boxes from heeat
+ maps it utilizes `getDetBoxesFromTextMapVertical()` function rather than
+ 'getDetBoxesFromTextMap()`. Other than that, refer to the standard OCR
+ Detector.
+
+  2. Narrow Detector -- it is designed to detect a single characters bounding
+ boxes. `getDetBoxesFromTextMapVertical()` function acts differently for Narrow
+ Detector and different textThreshold Value is passed. Additionally, the
+ grouping of detected boxes is completely omited.
+
+  Vertical Detector pipeline differentiate the Large Detector and Narrow
+ Detector based on `detectSingleCharacters` flag passed to the constructor.
+*/
+
+using executorch::aten::Tensor;
+using executorch::extension::TensorPtr;
+
+class VerticalDetector final : public BaseModel {
+public:
+  explicit VerticalDetector(const std::string &modelSource,
+                            bool detectSingleCharacters,
+                            std::shared_ptr<react::CallInvoker> callInvoker);
+  std::vector<types::DetectorBBox> generate(const cv::Mat &inputImage);
+  cv::Size getModelImageSize() const noexcept;
+
+private:
+  bool detectSingleCharacters;
+  std::vector<types::DetectorBBox> postprocess(const Tensor &tensor) const;
+  cv::Size modelImageSize;
+};
+} // namespace rnexecutorch::models::ocr

package/common/rnexecutorch/models/vertical_ocr/VerticalOCR.cpp

@@ -0,0 +1,180 @@
+#include "VerticalOCR.h"
+#include <future>
+#include <rnexecutorch/data_processing/ImageProcessing.h>
+#include <rnexecutorch/data_processing/Numerical.h>
+#include <rnexecutorch/models/ocr/Types.h>
+#include <tuple>
+
+namespace rnexecutorch::models::ocr {
+VerticalOCR::VerticalOCR(const std::string &detectorLargeSource,
+                         const std::string &detectorNarrowSource,
+                         const std::string &recognizerSource,
+                         std::string symbols, bool independentChars,
+                         std::shared_ptr<react::CallInvoker> invoker)
+    : detectorLarge(detectorLargeSource, false, invoker),
+      detectorNarrow(detectorNarrowSource, true, invoker),
+      recognizer(recognizerSource, invoker), converter(symbols),
+      independentCharacters(independentChars), callInvoker(invoker) {}
+
+std::vector<types::OCRDetection> VerticalOCR::generate(std::string input) {
+  cv::Mat image = image_processing::readImage(input);
+  if (image.empty()) {
+    throw std::runtime_error("Failed to load image from path: " + input);
+  }
+  // 1. Large Detector
+  std::vector<types::DetectorBBox> largeBoxes = detectorLarge.generate(image);
+
+  cv::Size largeDetectorSize = detectorLarge.getModelImageSize();
+  cv::Mat resizedImage =
+      image_processing::resizePadded(image, largeDetectorSize);
+  types::PaddingInfo imagePaddings =
+      utils::calculateResizeRatioAndPaddings(image.size(), largeDetectorSize);
+
+  std::vector<types::OCRDetection> predictions;
+  predictions.reserve(largeBoxes.size());
+
+  for (auto &box : largeBoxes) {
+    predictions.push_back(
+        _processSingleTextBox(box, image, resizedImage, imagePaddings));
+  }
+
+  return predictions;
+}
+
+std::size_t VerticalOCR::getMemoryLowerBound() const noexcept {
+  return detectorLarge.getMemoryLowerBound() +
+         detectorNarrow.getMemoryLowerBound() +
+         recognizer.getMemoryLowerBound();
+}
+
+// Strategy 1: Recognize each character individually
+std::pair<std::string, float> VerticalOCR::_handleIndependentCharacters(
+    const types::DetectorBBox &box, const cv::Mat &originalImage,
+    const std::vector<types::DetectorBBox> &characterBoxes,
+    const types::PaddingInfo &paddingsBox,
+    const types::PaddingInfo &imagePaddings) {
+  std::string text;
+  float confidenceScore = 0.0f;
+  float totalScore = 0.0f;
+  for (const auto &characterBox : characterBoxes) {
+
+    /*
+     Prepare for Recognition by following steps:
+     1. Crop image to the character bounding box,
+     2. Convert Image to gray.
+     3. Resize it to [VerticalSmallRecognizerWidth x RecognizerHeight] (64 x
+     64),
+    */
+    auto croppedChar = utils::prepareForRecognition(
+        originalImage, characterBox.bbox, box.bbox, paddingsBox, imagePaddings);
+
+    /*
+     To make Recognition simpler, we convert cropped character image
+     to a bit mask with white character and black background.
+    */
+    croppedChar = utils::characterBitMask(croppedChar);
+    croppedChar = utils::normalizeForRecognizer(
+        croppedChar, constants::kRecognizerHeight, 0.0, true);
+
+    const auto &[predIndex, score] = recognizer.generate(croppedChar);
+    if (!predIndex.empty()) {
+      text += converter.decodeGreedy(predIndex, predIndex.size())[0];
+    }
+    totalScore += score;
+  }
+  confidenceScore = totalScore / characterBoxes.size();
+  return {text, confidenceScore};
+}
+
+// Strategy 2: Concatenate characters and recognize as a single line
+std::pair<std::string, float> VerticalOCR::_handleJointCharacters(
+    const types::DetectorBBox &box, const cv::Mat &originalImage,
+    const std::vector<types::DetectorBBox> &characterBoxes,
+    const types::PaddingInfo &paddingsBox,
+    const types::PaddingInfo &imagePaddings) {
+  std::string text;
+  std::vector<cv::Mat> croppedCharacters;
+  croppedCharacters.reserve(characterBoxes.size());
+  for (const auto &characterBox : characterBoxes) {
+    /*
+     Prepare for Recognition by following steps:
+     1. Crop image to the character bounding box,
+     2. Convert Image to gray.
+     3. Resize it to [kSmallVerticalRecognizerWidth x kRecognizerHeight] (64 x
+     64). The same height is required for horizontal concatenation of single
+     characters into one image.
+    */
+    auto croppedChar = utils::prepareForRecognition(
+        originalImage, characterBox.bbox, box.bbox, paddingsBox, imagePaddings);
+    croppedCharacters.push_back(croppedChar);
+  }
+
+  cv::Mat mergedCharacters;
+  cv::hconcat(croppedCharacters, mergedCharacters);
+  mergedCharacters = image_processing::resizePadded(
+      mergedCharacters,
+      cv::Size(constants::kLargeRecognizerWidth, constants::kRecognizerHeight));
+  mergedCharacters = utils::normalizeForRecognizer(
+      mergedCharacters, constants::kRecognizerHeight, 0.0, false);
+
+  const auto &[predIndex, confidenceScore] =
+      recognizer.generate(mergedCharacters);
+  if (!predIndex.empty()) {
+    text = converter.decodeGreedy(predIndex, predIndex.size())[0];
+  }
+  return {text, confidenceScore};
+}
+
+types::OCRDetection VerticalOCR::_processSingleTextBox(
+    types::DetectorBBox &box, const cv::Mat &originalImage,
+    const cv::Mat &resizedLargeImage, const types::PaddingInfo &imagePaddings) {
+  cv::Rect boundingBox = utils::extractBoundingBox(box.bbox);
+
+  // Crop the image for detection of single characters.
+  cv::Rect safeRect =
+      boundingBox & cv::Rect(0, 0, resizedLargeImage.cols,
+                             resizedLargeImage.rows); // ensure valid box
+  cv::Mat croppedLargeBox = resizedLargeImage(safeRect);
+
+  // 2. Narrow Detector - detects single characters
+  std::vector<types::DetectorBBox> characterBoxes =
+      detectorNarrow.generate(croppedLargeBox);
+
+  std::string text;
+  float confidenceScore = 0.0;
+  if (!characterBoxes.empty()) {
+    // Prepare information useful for proper boxes shifting and image cropping.
+    const int32_t boxWidth =
+        static_cast<int32_t>(box.bbox[2].x - box.bbox[0].x);
+    const int32_t boxHeight =
+        static_cast<int32_t>(box.bbox[2].y - box.bbox[0].y);
+    cv::Size narrowRecognizerSize = detectorNarrow.getModelImageSize();
+    types::PaddingInfo paddingsBox = utils::calculateResizeRatioAndPaddings(
+        cv::Size(boxWidth, boxHeight), narrowRecognizerSize);
+
+    // 3. Recognition - decide between Strategy 1 and Strategy 2.
+    std::tie(text, confidenceScore) =
+        independentCharacters
+            ? _handleIndependentCharacters(box, originalImage, characterBoxes,
+                                           paddingsBox, imagePaddings)
+            : _handleJointCharacters(box, originalImage, characterBoxes,
+                                     paddingsBox, imagePaddings);
+  }
+  // Modify the returned boxes to match the original image size
+  std::array<types::Point, 4> finalBbox;
+  for (size_t i = 0; i < box.bbox.size(); ++i) {
+    finalBbox[i].x =
+        (box.bbox[i].x - imagePaddings.left) * imagePaddings.resizeRatio;
+    finalBbox[i].y =
+        (box.bbox[i].y - imagePaddings.top) * imagePaddings.resizeRatio;
+  }
+
+  return {finalBbox, text, confidenceScore};
+}
+
+void VerticalOCR::unload() noexcept {
+  detectorLarge.unload();
+  detectorNarrow.unload();
+  recognizer.unload();
+}
+} // namespace rnexecutorch::models::ocr

package/common/rnexecutorch/models/vertical_ocr/VerticalOCR.h

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <executorch/extension/tensor/tensor_ptr.h>
+#include <opencv2/opencv.hpp>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "rnexecutorch/metaprogramming/ConstructorHelpers.h"
+#include <rnexecutorch/models/BaseModel.h>
+#include <rnexecutorch/models/ocr/CTCLabelConverter.h>
+#include <rnexecutorch/models/ocr/Recognizer.h>
+#include <rnexecutorch/models/ocr/utils/DetectorUtils.h>
+#include <rnexecutorch/models/ocr/utils/RecognitionHandlerUtils.h>
+#include <rnexecutorch/models/ocr/utils/RecognizerUtils.h>
+#include <rnexecutorch/models/vertical_ocr/VerticalDetector.h>
+
+namespace rnexecutorch {
+namespace models::ocr {
+
+/*
+  Vertical OCR is OCR designed to handle vertical texts.
+  Vertical OCR pipeline consists of:
+  1. Large Detector -- detects regions where text is located.
+     Almost identical to the Detector in standard OCR.
+     The result of this phase is a list of bounding boxes.
+  Each detected box is then processed individually through the following steps:
+    2. Narrow Detector -- designed for detecting where single characters
+       are located.
+    There are two different strategies used for vertical recognition:
+      Strategy 1 "Independent Characters":
+        Treating each character region found  by Narrow Detector
+        as compeletely independent.
+        3. Each character is forwarded to Small Recognizer (64 x 64).
+      Strategy 2 "Joint Characters":
+        The bounding boxes found by Narrow Detector are
+        horizontally merged to create one wide image.
+        3. One wide image is forwarded to Large Recognzer (512 x 64).
+    Vertical OCR differentiate between those two strategies based on
+    `independentChars` flag passed to the constructor.
+*/
+
+using executorch::aten::Tensor;
+using executorch::extension::TensorPtr;
+
+class VerticalOCR final {
+public:
+  explicit VerticalOCR(const std::string &detectorLargeSource,
+                       const std::string &detectorNarrowSource,
+                       const std::string &recognizerSource, std::string symbols,
+                       bool indpendentCharacters,
+                       std::shared_ptr<react::CallInvoker> callInvoker);
+  std::vector<types::OCRDetection> generate(std::string input);
+  std::size_t getMemoryLowerBound() const noexcept;
+  void unload() noexcept;
+
+private:
+  std::pair<std::string, float> _handleIndependentCharacters(
+      const types::DetectorBBox &box, const cv::Mat &originalImage,
+      const std::vector<types::DetectorBBox> &characterBoxes,
+      const types::PaddingInfo &paddingsBox,
+      const types::PaddingInfo &imagePaddings);
+  std::pair<std::string, float>
+  _handleJointCharacters(const types::DetectorBBox &box,
+                         const cv::Mat &originalImage,
+                         const std::vector<types::DetectorBBox> &characterBoxes,
+                         const types::PaddingInfo &paddingsBox,
+                         const types::PaddingInfo &imagePaddings);
+  types::OCRDetection
+  _processSingleTextBox(types::DetectorBBox &box, const cv::Mat &originalImage,
+                        const cv::Mat &resizedLargeImage,
+                        const types::PaddingInfo &imagePaddings);
+  VerticalDetector detectorLarge;
+  VerticalDetector detectorNarrow;
+  Recognizer recognizer;
+  CTCLabelConverter converter;
+  bool independentCharacters;
+  std::shared_ptr<react::CallInvoker> callInvoker;
+};
+} // namespace models::ocr
+
+REGISTER_CONSTRUCTOR(models::ocr::VerticalOCR, std::string, std::string,
+                     std::string, std::string, bool,
+                     std::shared_ptr<react::CallInvoker>);
+} // namespace rnexecutorch