react-native-executorch 0.5.2 → 0.5.4

package/common/rnexecutorch/models/ocr/{DetectorUtils.h → utils/DetectorUtils.h}

@@ -4,7 +4,7 @@
 #include <rnexecutorch/models/ocr/Types.h>
 #include <span>
 
-namespace rnexecutorch::ocr {
+namespace rnexecutorch::models::ocr::utils {
 std::pair<cv::Mat, cv::Mat> interleavedArrayToMats(std::span<const float> data,
                                                    cv::Size size);
 /**
@@ -27,19 +27,20 @@ std::pair<cv::Mat, cv::Mat> interleavedArrayToMats(std::span<const float> data,
  * detected text box.
  *  - "angle": a float representing the rotation angle of the box.
  */
-std::vector<DetectorBBox> getDetBoxesFromTextMap(cv::Mat &textMap,
-                                                 cv::Mat &affinityMap,
-                                                 float textThreshold,
-                                                 float linkThreshold,
-                                                 float lowTextThreshold);
-std::vector<DetectorBBox>
+std::vector<types::DetectorBBox> getDetBoxesFromTextMap(cv::Mat &textMap,
+                                                        cv::Mat &affinityMap,
+                                                        float textThreshold,
+                                                        float linkThreshold,
+                                                        float lowTextThreshold);
+std::vector<types::DetectorBBox>
 getDetBoxesFromTextMapVertical(cv::Mat &textMap, cv::Mat &affinityMap,
                                float textThreshold, float linkThreshold,
                                bool independentCharacters);
 
 float calculateRestoreRatio(int32_t currentSize, int32_t desiredSize);
 
-void restoreBboxRatio(std::vector<DetectorBBox> &boxes, float restoreRatio);
+void restoreBboxRatio(std::vector<types::DetectorBBox> &boxes,
+                      float restoreRatio);
 /**
  * This method processes a vector of DetectorBBox bounding boxes, each
  * containing details about individual text boxes, and attempts to group and
@@ -72,9 +73,9 @@ void restoreBboxRatio(std::vector<DetectorBBox> &boxes, float restoreRatio);
  * 3. Post-processing to remove any boxes that are too small.
  * 4. Sort the final array of boxes by their vertical positions.
  */
-std::vector<DetectorBBox>
-groupTextBoxes(std::vector<DetectorBBox> &boxes, float centerThreshold,
+std::vector<types::DetectorBBox>
+groupTextBoxes(std::vector<types::DetectorBBox> &boxes, float centerThreshold,
                float distanceThreshold, float heightThreshold,
                int32_t minSideThreshold, int32_t maxSideThreshold,
                int32_t maxWidth);
-} // namespace rnexecutorch::ocr
+} // namespace rnexecutorch::models::ocr::utils

package/common/rnexecutorch/models/ocr/{RecognitionHandlerUtils.cpp → utils/RecognitionHandlerUtils.cpp}

@@ -3,9 +3,9 @@
 #include <rnexecutorch/data_processing/ImageProcessing.h>
 #include <rnexecutorch/models/ocr/Constants.h>
 
-namespace rnexecutorch::ocr {
-PaddingInfo calculateResizeRatioAndPaddings(cv::Size size,
-                                            cv::Size desiredSize) {
+namespace rnexecutorch::models::ocr::utils {
+types::PaddingInfo calculateResizeRatioAndPaddings(cv::Size size,
+                                                   cv::Size desiredSize) {
   const auto newRatioH = static_cast<float>(desiredSize.height) / size.height;
   const auto newRatioW = static_cast<float>(desiredSize.width) / size.width;
   auto resizeRatio = std::min(newRatioH, newRatioW);
@@ -40,7 +40,8 @@ void computeRatioAndResize(cv::Mat &img, cv::Size size, int32_t modelHeight) {
   cv::resize(img, img, resizedSize, 0.0, 0.0, cv::INTER_LANCZOS4);
 }
 
-cv::Mat cropImage(DetectorBBox box, cv::Mat &image, int32_t modelHeight) {
+cv::Mat cropImage(types::DetectorBBox box, cv::Mat &image,
+                  int32_t modelHeight) {
   // Convert custom points to cv::Point2f
   std::array<cv::Point2f, 4> points;
 #pragma unroll
@@ -126,13 +127,14 @@ void adjustContrastGrey(cv::Mat &img, double target) {
 
 int32_t getDesiredWidth(const cv::Mat &img, bool isVertical) {
 
-  if (img.cols >= LARGE_RECOGNIZER_WIDTH) {
-    return LARGE_RECOGNIZER_WIDTH;
+  if (img.cols >= constants::kLargeRecognizerWidth) {
+    return constants::kLargeRecognizerWidth;
   }
-  if (img.cols >= MEDIUM_RECOGNIZER_WIDTH) {
-    return MEDIUM_RECOGNIZER_WIDTH;
+  if (img.cols >= constants::kMediumRecognizerWidth) {
+    return constants::kMediumRecognizerWidth;
   }
-  return isVertical ? SMALL_VERTICAL_RECOGNIZER_WIDTH : SMALL_RECOGNIZER_WIDTH;
+  return isVertical ? constants::kSmallVerticalRecognizerWidth
+                    : constants::kSmallRecognizerWidth;
 }
 
 cv::Mat normalizeForRecognizer(const cv::Mat &image, int32_t modelHeight,
@@ -144,10 +146,11 @@ cv::Mat normalizeForRecognizer(const cv::Mat &image, int32_t modelHeight,
 
   int32_t desiredWidth = getDesiredWidth(image, isVertical);
 
-  img = imageprocessing::resizePadded(img, cv::Size(desiredWidth, modelHeight));
+  img =
+      image_processing::resizePadded(img, cv::Size(desiredWidth, modelHeight));
   img.convertTo(img, CV_32F, 1.0f / 255.0f);
   img -= 0.5f;
   img *= 2.0f;
   return img;
 }
-} // namespace rnexecutorch::ocr
+} // namespace rnexecutorch::models::ocr::utils

package/common/rnexecutorch/models/ocr/{RecognitionHandlerUtils.h → utils/RecognitionHandlerUtils.h}

@@ -3,7 +3,7 @@
 #include <opencv2/opencv.hpp>
 #include <rnexecutorch/models/ocr/Types.h>
 
-namespace rnexecutorch::ocr {
+namespace rnexecutorch::models::ocr::utils {
 /**
  * @brief Calculates the resize ratio and padding offsets needed to fit an image
  *  into a target size while maintaining aspect ratio.
@@ -12,8 +12,8 @@ namespace rnexecutorch::ocr {
  * @return Struct containing the scaling factor and top/left padding amounts for
  * centering the image.
  */
-PaddingInfo calculateResizeRatioAndPaddings(cv::Size size,
-                                            cv::Size desiredSize);
+types::PaddingInfo calculateResizeRatioAndPaddings(cv::Size size,
+                                                   cv::Size desiredSize);
 /**
  * @brief Resizes an image proportionally to match a target height while
  * maintaining aspect ratio.
@@ -41,7 +41,7 @@ void computeRatioAndResize(cv::Mat &img, cv::Size size, int32_t modelHeight);
  * @param modelHeight Target height for output (width scales proportionally)
  * @return Cropped, aligned and resized image region (empty if invalid box)
  */
-cv::Mat cropImage(DetectorBBox box, cv::Mat &image, int32_t modelHeight);
+cv::Mat cropImage(types::DetectorBBox box, cv::Mat &image, int32_t modelHeight);
 void adjustContrastGrey(cv::Mat &img, double target);
 /**
  * @brief Prepares an image for recognition models by standardizing size,
@@ -69,4 +69,4 @@ void adjustContrastGrey(cv::Mat &img, double target);
 cv::Mat normalizeForRecognizer(const cv::Mat &image, int32_t modelHeight,
                                double adjustContrast = 0.0,
                                bool isVertical = false);
-} // namespace rnexecutorch::ocr
+} // namespace rnexecutorch::models::ocr::utils

package/common/rnexecutorch/models/ocr/{RecognizerUtils.cpp → utils/RecognizerUtils.cpp}

@@ -1,6 +1,6 @@
 #include "RecognizerUtils.h"
 
-namespace rnexecutorch::ocr {
+namespace rnexecutorch::models::ocr::utils {
 cv::Mat softmax(const cv::Mat &inputs) {
   cv::Mat maxVal;
   cv::reduce(inputs, maxVal, 1, cv::REDUCE_MAX, CV_32F);
@@ -28,9 +28,9 @@ void divideMatrixByRows(cv::Mat &matrix, const std::vector<float> &rowSums) {
   }
 }
 
-ValuesAndIndices findMaxValuesIndices(const cv::Mat &mat) {
+types::ValuesAndIndices findMaxValuesIndices(const cv::Mat &mat) {
   CV_Assert(mat.type() == CV_32F);
-  ValuesAndIndices result{};
+  types::ValuesAndIndices result{};
   result.values.reserve(mat.rows);
   result.indices.reserve(mat.rows);
 
@@ -66,7 +66,7 @@ float confidenceScore(const std::vector<float> &values,
   return std::pow(product, exponent);
 }
 
-cv::Rect extractBoundingBox(std::array<Point, 4> &points) {
+cv::Rect extractBoundingBox(std::array<types::Point, 4> &points) {
   cv::Mat pointsMat(4, 1, CV_32FC2, points.data());
   return cv::boundingRect(pointsMat);
 }
@@ -107,10 +107,11 @@ cv::Mat characterBitMask(const cv::Mat &img) {
 
   const int32_t height = thresh.rows;
   const int32_t width = thresh.cols;
-  const int32_t minX = ocr::SINGLE_CHARACTER_CENTER_THRESHOLD * width;
-  const int32_t maxX = (1 - ocr::SINGLE_CHARACTER_CENTER_THRESHOLD) * width;
-  const int32_t minY = ocr::SINGLE_CHARACTER_CENTER_THRESHOLD * height;
-  const int32_t maxY = (1 - ocr::SINGLE_CHARACTER_CENTER_THRESHOLD) * height;
+  const int32_t minX = constants::kSingleCharacterCenterThreshold * width;
+  const int32_t maxX = (1 - constants::kSingleCharacterCenterThreshold) * width;
+  const int32_t minY = constants::kSingleCharacterCenterThreshold * height;
+  const int32_t maxY =
+      (1 - constants::kSingleCharacterCenterThreshold) * height;
 
   int32_t selectedComponent = -1;
   int32_t maxArea = -1;
@@ -120,8 +121,8 @@ cv::Mat characterBitMask(const cv::Mat &img) {
     const double cy = centroids.at<double>(i, 1);
 
     if ((minX < cx && cx < maxX && minY < cy &&
-         cy < maxY &&                              // check if centered
-         area > ocr::SINGLE_CHARACTER_MIN_SIZE) && // check if large enough
+         cy < maxY &&                                  // check if centered
+         area > constants::kSingleCharacterMinSize) && // check if large enough
         area > maxArea) {
       selectedComponent = i;
       maxArea = area;
@@ -142,21 +143,22 @@ cv::Mat characterBitMask(const cv::Mat &img) {
   return resultImage;
 }
 
-cv::Mat cropImageWithBoundingBox(const cv::Mat &img,
-                                 const std::array<Point, 4> &bbox,
-                                 const std::array<Point, 4> &originalBbox,
-                                 const PaddingInfo &paddings,
-                                 const PaddingInfo &originalPaddings) {
+cv::Mat
+cropImageWithBoundingBox(const cv::Mat &img,
+                         const std::array<types::Point, 4> &bbox,
+                         const std::array<types::Point, 4> &originalBbox,
+                         const types::PaddingInfo &paddings,
+                         const types::PaddingInfo &originalPaddings) {
   if (originalBbox.empty()) {
     throw std::runtime_error("Original bounding box cannot be empty.");
   }
-  const Point topLeft = originalBbox[0];
+  const types::Point topLeft = originalBbox[0];
 
   std::vector<cv::Point2f> points;
   points.reserve(bbox.size());
 
   for (const auto &point : bbox) {
-    Point transformedPoint = point;
+    types::Point transformedPoint = point;
 
     transformedPoint.x -= paddings.left;
     transformedPoint.y -= paddings.top;
@@ -186,17 +188,17 @@ cv::Mat cropImageWithBoundingBox(const cv::Mat &img,
 }
 
 cv::Mat prepareForRecognition(const cv::Mat &originalImage,
-                              const std::array<Point, 4> &bbox,
-                              const std::array<Point, 4> &originalBbox,
-                              const PaddingInfo &paddings,
-                              const PaddingInfo &originalPaddings) {
+                              const std::array<types::Point, 4> &bbox,
+                              const std::array<types::Point, 4> &originalBbox,
+                              const types::PaddingInfo &paddings,
+                              const types::PaddingInfo &originalPaddings) {
   auto croppedChar = cropImageWithBoundingBox(originalImage, bbox, originalBbox,
                                               paddings, originalPaddings);
   cv::cvtColor(croppedChar, croppedChar, cv::COLOR_BGR2GRAY);
-  cv::resize(
-      croppedChar, croppedChar,
-      cv::Size(ocr::SMALL_VERTICAL_RECOGNIZER_WIDTH, ocr::RECOGNIZER_HEIGHT), 0,
-      0, cv::INTER_AREA);
+  cv::resize(croppedChar, croppedChar,
+             cv::Size(constants::kSmallVerticalRecognizerWidth,
+                      constants::kRecognizerHeight),
+             0, 0, cv::INTER_AREA);
   return croppedChar;
 }
-} // namespace rnexecutorch::ocr
+} // namespace rnexecutorch::models::ocr::utils

package/common/rnexecutorch/models/ocr/{RecognizerUtils.h → utils/RecognizerUtils.h}

@@ -6,7 +6,7 @@
 #include <rnexecutorch/models/ocr/Types.h>
 #include <vector>
 
-namespace rnexecutorch::ocr {
+namespace rnexecutorch::models::ocr::utils {
 /**
  * @brief Computes per row softmax funcion.
  * Formula: softmax(x_i) = exp(x_i - max(x)) / sum(exp(x_j - max(x))) for each
@@ -18,10 +18,10 @@ cv::Mat softmax(const cv::Mat &inputs);
  * @brief For each row of matrix computes {maxValue, index} pair. Returns a list
  * of maxValues and a list of corresponding indices.
  */
-ValuesAndIndices findMaxValuesIndices(const cv::Mat &mat);
+types::ValuesAndIndices findMaxValuesIndices(const cv::Mat &mat);
 std::vector<float> sumProbabilityRows(const cv::Mat &matrix);
 void divideMatrixByRows(cv::Mat &matrix, const std::vector<float> &rowSums);
-cv::Rect extractBoundingBox(std::array<Point, 4> &points);
+cv::Rect extractBoundingBox(std::array<types::Point, 4> &points);
 
 /**
  * @brief Computes confidence score for given values and indices vectors.
@@ -43,11 +43,12 @@ cv::Mat characterBitMask(const cv::Mat &img);
  * with internal bounding box and padding.
  * It does so to preserve the best possible image quality.
  */
-cv::Mat cropImageWithBoundingBox(const cv::Mat &img,
-                                 const std::array<Point, 4> &bbox,
-                                 const std::array<Point, 4> &originalBbox,
-                                 const PaddingInfo &paddings,
-                                 const PaddingInfo &originalPaddings);
+cv::Mat
+cropImageWithBoundingBox(const cv::Mat &img,
+                         const std::array<types::Point, 4> &bbox,
+                         const std::array<types::Point, 4> &originalBbox,
+                         const types::PaddingInfo &paddings,
+                         const types::PaddingInfo &originalPaddings);
 
 /**
  * @brief Perform cropping, resizing and convert to grayscale to prepare image
@@ -56,15 +57,15 @@ cv::Mat cropImageWithBoundingBox(const cv::Mat &img,
  * Prepare for Recognition by following steps:
  * 1. Crop image to the character bounding box,
  * 2. Convert Image to gray.
- * 3. Resize it to [SMALL_VERTICAL_RECOGNIZER_WIDTH x RECOGNIZER_HEIGHT] (64 x
+ * 3. Resize it to [kSmallVerticalRecognizerWidth x kRecognizerHeight] (64 x
  * 64).
  *
  * @details it utilizes cropImageWithBoundingBox to perform specific cropping.
  */
 
 cv::Mat prepareForRecognition(const cv::Mat &originalImage,
-                              const std::array<Point, 4> &bbox,
-                              const std::array<Point, 4> &originalBbox,
-                              const PaddingInfo &paddings,
-                              const PaddingInfo &originalPaddings);
-} // namespace rnexecutorch::ocr
+                              const std::array<types::Point, 4> &bbox,
+                              const std::array<types::Point, 4> &originalBbox,
+                              const types::PaddingInfo &paddings,
+                              const types::PaddingInfo &originalPaddings);
+} // namespace rnexecutorch::models::ocr::utils

package/common/rnexecutorch/models/speech_to_text/SpeechToText.cpp

@@ -1,64 +1,125 @@
-#include <rnexecutorch/models/speech_to_text/SpeechToText.h>
-#include <rnexecutorch/models/speech_to_text/WhisperStrategy.h>
-#include <stdexcept>
+#include <thread>
 
-namespace rnexecutorch {
+#include "SpeechToText.h"
+
+namespace rnexecutorch::models::speech_to_text {
 
 using namespace ::executorch::extension;
 
-SpeechToText::SpeechToText(const std::string &encoderPath,
-                           const std::string &decoderPath,
-                           const std::string &modelName,
+SpeechToText::SpeechToText(const std::string &encoderSource,
+                           const std::string &decoderSource,
+                           const std::string &tokenizerSource,
                            std::shared_ptr<react::CallInvoker> callInvoker)
-    : EncoderDecoderBase(encoderPath, decoderPath, callInvoker),
-      modelName(modelName) {
-  initializeStrategy();
+    : callInvoker(std::move(callInvoker)),
+      encoder(std::make_unique<BaseModel>(encoderSource, this->callInvoker)),
+      decoder(std::make_unique<BaseModel>(decoderSource, this->callInvoker)),
+      tokenizer(std::make_unique<TokenizerModule>(tokenizerSource,
+                                                  this->callInvoker)),
+      asr(std::make_unique<ASR>(this->encoder.get(), this->decoder.get(),
+                                this->tokenizer.get())),
+      processor(std::make_unique<OnlineASRProcessor>(this->asr.get())),
+      isStreaming(false), readyToProcess(false) {}
+
+std::shared_ptr<OwningArrayBuffer>
+SpeechToText::encode(std::span<float> waveform) const {
+  std::vector<float> encoderOutput = this->asr->encode(waveform);
+  return this->makeOwningBuffer(encoderOutput);
 }
 
-void SpeechToText::initializeStrategy() {
-  if (modelName == "whisper") {
-    strategy = std::make_unique<WhisperStrategy>();
-  } else {
-    throw std::runtime_error("Unsupported STT model: " + modelName +
-                             ". Only 'whisper' is supported.");
-  }
+std::shared_ptr<OwningArrayBuffer>
+SpeechToText::decode(std::span<int32_t> tokens,
+                     std::span<float> encoderOutput) const {
+  std::vector<float> decoderOutput = this->asr->decode(tokens, encoderOutput);
+  return this->makeOwningBuffer(decoderOutput);
 }
 
-void SpeechToText::encode(std::span<float> waveform) {
-  const auto modelInputTensor = strategy->prepareAudioInput(waveform);
+std::string SpeechToText::transcribe(std::span<float> waveform,
+                                     std::string languageOption) const {
+  std::vector<Segment> segments =
+      this->asr->transcribe(waveform, DecodingOptions(languageOption));
+  std::string transcription;
+
+  size_t transcriptionLength = 0;
+  for (auto &segment : segments) {
+    for (auto &word : segment.words) {
+      transcriptionLength += word.content.size();
+    }
+  }
+  transcription.reserve(transcriptionLength);
 
-  const auto result = encoder_->forward(modelInputTensor);
-  if (!result.ok()) {
-    throw std::runtime_error(
-        "Forward pass failed during encoding, error code: " +
-        std::to_string(static_cast<int>(result.error())));
+  for (auto &segment : segments) {
+    for (auto &word : segment.words) {
+      transcription += word.content;
+    }
   }
+  return transcription;
+}
 
-  encoderOutput = result.get().at(0);
+size_t SpeechToText::getMemoryLowerBound() const noexcept {
+  return this->encoder->getMemoryLowerBound() +
+         this->decoder->getMemoryLowerBound() +
+         this->tokenizer->getMemoryLowerBound();
 }
 
 std::shared_ptr<OwningArrayBuffer>
-SpeechToText::decode(std::vector<int64_t> prevTokens) {
-  if (encoderOutput.isNone()) {
-    throw std::runtime_error("Empty encodings on decode call, make sure to "
-                             "call encode() prior to decode()!");
+SpeechToText::makeOwningBuffer(std::span<const float> vectorView) const {
+  auto owningArrayBuffer =
+      std::make_shared<OwningArrayBuffer>(vectorView.size_bytes());
+  std::memcpy(owningArrayBuffer->data(), vectorView.data(),
+              vectorView.size_bytes());
+  return owningArrayBuffer;
+}
+
+void SpeechToText::stream(std::shared_ptr<jsi::Function> callback,
+                          std::string languageOption) {
+  if (this->isStreaming) {
+    throw std::runtime_error("Streaming is already in progress");
+  }
+
+  auto nativeCallback = [this, callback](const std::string &committed,
+                                         const std::string &nonCommitted,
+                                         bool isDone) {
+    this->callInvoker->invokeAsync(
+        [callback, committed, nonCommitted, isDone](jsi::Runtime &rt) {
+          callback->call(rt, jsi::String::createFromUtf8(rt, committed),
+                         jsi::String::createFromUtf8(rt, nonCommitted),
+                         jsi::Value(isDone));
+        });
+  };
+
+  this->resetStreamState();
+
+  this->isStreaming = true;
+  while (this->isStreaming) {
+    if (!this->readyToProcess ||
+        this->processor->audioBuffer.size() < SpeechToText::kMinAudioSamples) {
+      std::this_thread::sleep_for(std::chrono::milliseconds(100));
+      continue;
+    }
+    ProcessResult res =
+        this->processor->processIter(DecodingOptions(languageOption));
+    nativeCallback(res.committed, res.nonCommitted, false);
+    this->readyToProcess = false;
   }
 
-  const auto prevTokensTensor = strategy->prepareTokenInput(prevTokens);
+  std::string committed = this->processor->finish();
+  nativeCallback(committed, "", true);
+}
 
-  const auto decoderMethod = strategy->getDecoderMethod();
-  const auto decoderResult =
-      decoder_->execute(decoderMethod, {prevTokensTensor, encoderOutput});
+void SpeechToText::streamStop() { this->isStreaming = false; }
 
-  if (!decoderResult.ok()) {
-    throw std::runtime_error(
-        "Forward pass failed during decoding, error code: " +
-        std::to_string(static_cast<int>(decoderResult.error())));
+void SpeechToText::streamInsert(std::span<float> waveform) {
+  if (!this->isStreaming) {
+    throw std::runtime_error("Streaming is not started");
   }
+  this->processor->insertAudioChunk(waveform);
+  this->readyToProcess = true;
+}
 
-  const auto decoderOutputTensor = decoderResult.get().at(0).toTensor();
-  const auto innerDim = decoderOutputTensor.size(1);
-  return strategy->extractOutputToken(decoderOutputTensor);
+void SpeechToText::resetStreamState() {
+  this->isStreaming = false;
+  this->readyToProcess = false;
+  this->processor = std::make_unique<OnlineASRProcessor>(this->asr.get());
 }
 
-} // namespace rnexecutorch
+} // namespace rnexecutorch::models::speech_to_text

package/common/rnexecutorch/models/speech_to_text/SpeechToText.h

@@ -1,31 +1,60 @@
 #pragma once
 
-#include "ReactCommon/CallInvoker.h"
-#include "executorch/runtime/core/evalue.h"
-#include <cstdint>
-#include <memory>
-#include <rnexecutorch/models/EncoderDecoderBase.h>
-#include <rnexecutorch/models/speech_to_text/SpeechToTextStrategy.h>
-#include <span>
-#include <string>
-#include <vector>
+#include "rnexecutorch/models/speech_to_text/stream/OnlineASRProcessor.h"
 
 namespace rnexecutorch {
-class SpeechToText : public EncoderDecoderBase {
+
+namespace models::speech_to_text {
+
+using namespace asr;
+using namespace types;
+using namespace stream;
+
+class SpeechToText {
 public:
-  explicit SpeechToText(const std::string &encoderPath,
-                        const std::string &decoderPath,
-                        const std::string &modelName,
+  explicit SpeechToText(const std::string &encoderSource,
+                        const std::string &decoderSource,
+                        const std::string &tokenizerSource,
                         std::shared_ptr<react::CallInvoker> callInvoker);
-  void encode(std::span<float> waveform);
-  std::shared_ptr<OwningArrayBuffer> decode(std::vector<int64_t> prevTokens);
+
+  std::shared_ptr<OwningArrayBuffer> encode(std::span<float> waveform) const;
+  std::shared_ptr<OwningArrayBuffer>
+  decode(std::span<int32_t> tokens, std::span<float> encoderOutput) const;
+  std::string transcribe(std::span<float> waveform,
+                         std::string languageOption) const;
+
+  size_t getMemoryLowerBound() const noexcept;
+
+  // Stream
+  void stream(std::shared_ptr<jsi::Function> callback,
+              std::string languageOption);
+  void streamStop();
+  void streamInsert(std::span<float> waveform);
 
 private:
-  const std::string modelName;
-  executorch::runtime::EValue encoderOutput;
-  std::unique_ptr<SpeechToTextStrategy> strategy;
+  std::unique_ptr<BaseModel> encoder;
+  std::unique_ptr<BaseModel> decoder;
+  std::unique_ptr<TokenizerModule> tokenizer;
+  std::unique_ptr<ASR> asr;
 
-  void initializeStrategy();
+  std::shared_ptr<OwningArrayBuffer>
+  makeOwningBuffer(std::span<const float> vectorView) const;
+
+  // Stream
+  std::shared_ptr<react::CallInvoker> callInvoker;
+  std::unique_ptr<OnlineASRProcessor> processor;
+  bool isStreaming;
+  bool readyToProcess;
+
+  constexpr static int32_t kMinAudioSamples = 16000; // 1 second
+
+  void resetStreamState();
 };
 
+} // namespace models::speech_to_text
+
+REGISTER_CONSTRUCTOR(models::speech_to_text::SpeechToText, std::string,
+                     std::string, std::string,
+                     std::shared_ptr<react::CallInvoker>);
+
 } // namespace rnexecutorch