react-native-executorch 0.8.0-nightly-48610bf-20260324 → 0.8.0

package/android/src/main/cpp/ETInstallerModule.h

@@ -2,7 +2,6 @@
 
 #include <ReactCommon/CallInvokerHolder.h>
 #include <fbjni/fbjni.h>
-#include <react/jni/CxxModuleWrapper.h>
 #include <react/jni/JMessageQueueThread.h>
 
 #include <memory>
@@ -40,4 +39,4 @@ private:
       const std::shared_ptr<facebook::react::CallInvoker> &jsCallInvoker);
 };
 
-} // namespace rnexecutorch
+} // namespace rnexecutorch

package/common/rnexecutorch/models/object_detection/ObjectDetection.cpp

@@ -1,6 +1,8 @@
 #include "ObjectDetection.h"
 #include "Constants.h"
 
+#include <set>
+
 #include <rnexecutorch/Error.h>
 #include <rnexecutorch/ErrorCodes.h>
 #include <rnexecutorch/Log.h>
@@ -18,21 +20,6 @@ ObjectDetection::ObjectDetection(
     std::shared_ptr<react::CallInvoker> callInvoker)
     : VisionModel(modelSource, callInvoker),
       labelNames_(std::move(labelNames)) {
-  auto inputTensors = getAllInputShapes();
-  if (inputTensors.empty()) {
-    throw RnExecutorchError(RnExecutorchErrorCode::UnexpectedNumInputs,
-                            "Model seems to not take any input tensors.");
-  }
-  modelInputShape_ = inputTensors[0];
-  if (modelInputShape_.size() < 2) {
-    char errorMessage[100];
-    std::snprintf(errorMessage, sizeof(errorMessage),
-                  "Unexpected model input size, expected at least 2 dimensions "
-                  "but got: %zu.",
-                  modelInputShape_.size());
-    throw RnExecutorchError(RnExecutorchErrorCode::UnexpectedNumInputs,
-                            errorMessage);
-  }
   if (normMean.size() == 3) {
     normMean_ = cv::Scalar(normMean[0], normMean[1], normMean[2]);
   } else if (!normMean.empty()) {
@@ -47,14 +34,67 @@ ObjectDetection::ObjectDetection(
   }
 }
 
+cv::Size ObjectDetection::modelInputSize() const {
+  if (currentlyLoadedMethod_.empty()) {
+    return VisionModel::modelInputSize();
+  }
+  auto inputShapes = getAllInputShapes(currentlyLoadedMethod_);
+  if (inputShapes.empty() || inputShapes[0].size() < 2) {
+    throw RnExecutorchError(RnExecutorchErrorCode::UnexpectedNumInputs,
+                            "Could not determine input shape for method: " +
+                                currentlyLoadedMethod_);
+  }
+  const auto &shape = inputShapes[0];
+  return {static_cast<int32_t>(shape[shape.size() - 2]),
+          static_cast<int32_t>(shape[shape.size() - 1])};
+}
+
+void ObjectDetection::ensureMethodLoaded(const std::string &methodName) {
+  if (methodName.empty()) {
+    throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
+                            "methodName cannot be empty");
+  }
+  if (currentlyLoadedMethod_ == methodName) {
+    return;
+  }
+  if (!module_) {
+    throw RnExecutorchError(RnExecutorchErrorCode::ModuleNotLoaded,
+                            "Model module is not loaded");
+  }
+  if (!currentlyLoadedMethod_.empty()) {
+    module_->unload_method(currentlyLoadedMethod_);
+  }
+  auto loadResult = module_->load_method(methodName);
+  if (loadResult != executorch::runtime::Error::Ok) {
+    throw RnExecutorchError(
+        loadResult, "Failed to load method '" + methodName +
+                        "'. Ensure the method exists in the exported model.");
+  }
+  currentlyLoadedMethod_ = methodName;
+}
+
+std::set<int32_t> ObjectDetection::prepareAllowedClasses(
+    const std::vector<int32_t> &classIndices) const {
+  std::set<int32_t> allowedClasses;
+  if (!classIndices.empty()) {
+    allowedClasses.insert(classIndices.begin(), classIndices.end());
+  }
+  return allowedClasses;
+}
+
 std::vector<types::Detection>
 ObjectDetection::postprocess(const std::vector<EValue> &tensors,
-                             cv::Size originalSize, double detectionThreshold) {
+                             cv::Size originalSize, double detectionThreshold,
+                             double iouThreshold,
+                             const std::vector<int32_t> &classIndices) {
   const cv::Size inputSize = modelInputSize();
   float widthRatio = static_cast<float>(originalSize.width) / inputSize.width;
   float heightRatio =
       static_cast<float>(originalSize.height) / inputSize.height;
 
+  // Prepare allowed classes set for filtering
+  auto allowedClasses = prepareAllowedClasses(classIndices);
+
   std::vector<types::Detection> detections;
   auto bboxTensor = tensors.at(0).toTensor();
   std::span<const float> bboxes(
@@ -75,12 +115,21 @@ ObjectDetection::postprocess(const std::vector<EValue> &tensors,
     if (scores[i] < detectionThreshold) {
       continue;
     }
+
+    auto labelIdx = static_cast<int32_t>(labels[i]);
+
+    // Filter by class if classesOfInterest is specified
+    if (!allowedClasses.empty() &&
+        allowedClasses.find(labelIdx) == allowedClasses.end()) {
+      continue;
+    }
+
     float x1 = bboxes[i * 4] * widthRatio;
     float y1 = bboxes[i * 4 + 1] * heightRatio;
     float x2 = bboxes[i * 4 + 2] * widthRatio;
     float y2 = bboxes[i * 4 + 3] * heightRatio;
-    auto labelIdx = static_cast<std::size_t>(labels[i]);
-    if (labelIdx >= labelNames_.size()) {
+
+    if (std::cmp_greater_equal(labelIdx, labelNames_.size())) {
       throw RnExecutorchError(
           RnExecutorchErrorCode::InvalidConfig,
           "Model output class index " + std::to_string(labelIdx) +
@@ -88,23 +137,40 @@ ObjectDetection::postprocess(const std::vector<EValue> &tensors,
               ". Ensure the labelMap covers all model output classes.");
     }
     detections.emplace_back(utils::computer_vision::BBox{x1, y1, x2, y2},
-                            labelNames_[labelIdx],
-                            static_cast<int32_t>(labelIdx), scores[i]);
+                            labelNames_[labelIdx], labelIdx, scores[i]);
   }
 
-  return utils::computer_vision::nonMaxSuppression(detections,
-                                                   constants::IOU_THRESHOLD);
+  return utils::computer_vision::nonMaxSuppression(detections, iouThreshold);
 }
 
-std::vector<types::Detection>
-ObjectDetection::runInference(cv::Mat image, double detectionThreshold) {
+std::vector<types::Detection> ObjectDetection::runInference(
+    cv::Mat image, double detectionThreshold, double iouThreshold,
+    const std::vector<int32_t> &classIndices, const std::string &methodName) {
   if (detectionThreshold < 0.0 || detectionThreshold > 1.0) {
     throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
                             "detectionThreshold must be in range [0, 1]");
   }
+  if (iouThreshold < 0.0 || iouThreshold > 1.0) {
+    throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
+                            "iouThreshold must be in range [0, 1]");
+  }
+
   std::scoped_lock lock(inference_mutex_);
 
+  // Ensure the correct method is loaded
+  ensureMethodLoaded(methodName);
+
   cv::Size originalSize = image.size();
+
+  // Query input shapes for the currently loaded method
+  auto inputShapes = getAllInputShapes(methodName);
+  if (inputShapes.empty() || inputShapes[0].size() < 2) {
+    throw RnExecutorchError(RnExecutorchErrorCode::UnexpectedNumInputs,
+                            "Could not determine input shape for method: " +
+                                methodName);
+  }
+  modelInputShape_ = inputShapes[0];
+
   cv::Mat preprocessed = preprocess(image);
 
   auto inputTensor =
@@ -114,46 +180,52 @@ ObjectDetection::runInference(cv::Mat image, double detectionThreshold) {
           : image_processing::getTensorFromMatrix(modelInputShape_,
                                                   preprocessed);
 
-  auto forwardResult = BaseModel::forward(inputTensor);
-  if (!forwardResult.ok()) {
-    throw RnExecutorchError(forwardResult.error(),
-                            "The model's forward function did not succeed. "
-                            "Ensure the model input is correct.");
+  auto executeResult = execute(methodName, {inputTensor});
+  if (!executeResult.ok()) {
+    throw RnExecutorchError(executeResult.error(),
+                            "The model's " + methodName +
+                                " method did not succeed. "
+                                "Ensure the model input is correct.");
   }
 
-  return postprocess(forwardResult.get(), originalSize, detectionThreshold);
+  return postprocess(executeResult.get(), originalSize, detectionThreshold,
+                     iouThreshold, classIndices);
 }
 
-std::vector<types::Detection>
-ObjectDetection::generateFromString(std::string imageSource,
-                                    double detectionThreshold) {
+std::vector<types::Detection> ObjectDetection::generateFromString(
+    std::string imageSource, double detectionThreshold, double iouThreshold,
+    std::vector<int32_t> classIndices, std::string methodName) {
   cv::Mat imageBGR = image_processing::readImage(imageSource);
 
   cv::Mat imageRGB;
   cv::cvtColor(imageBGR, imageRGB, cv::COLOR_BGR2RGB);
 
-  return runInference(imageRGB, detectionThreshold);
+  return runInference(imageRGB, detectionThreshold, iouThreshold, classIndices,
+                      methodName);
 }
 
-std::vector<types::Detection>
-ObjectDetection::generateFromFrame(jsi::Runtime &runtime,
-                                   const jsi::Value &frameData,
-                                   double detectionThreshold) {
+std::vector<types::Detection> ObjectDetection::generateFromFrame(
+    jsi::Runtime &runtime, const jsi::Value &frameData,
+    double detectionThreshold, double iouThreshold,
+    std::vector<int32_t> classIndices, std::string methodName) {
   auto orient = ::rnexecutorch::utils::readFrameOrientation(runtime, frameData);
   cv::Mat frame = extractFromFrame(runtime, frameData);
   cv::Mat rotated = ::rnexecutorch::utils::rotateFrameForModel(frame, orient);
-  auto detections = runInference(rotated, detectionThreshold);
+  auto detections = runInference(rotated, detectionThreshold, iouThreshold,
+                                 classIndices, methodName);
+
   for (auto &det : detections) {
     ::rnexecutorch::utils::inverseRotateBbox(det.bbox, orient, rotated.size());
   }
   return detections;
 }
 
-std::vector<types::Detection>
-ObjectDetection::generateFromPixels(JSTensorViewIn pixelData,
-                                    double detectionThreshold) {
+std::vector<types::Detection> ObjectDetection::generateFromPixels(
+    JSTensorViewIn pixelData, double detectionThreshold, double iouThreshold,
+    std::vector<int32_t> classIndices, std::string methodName) {
   cv::Mat image = extractFromPixels(pixelData);
 
-  return runInference(image, detectionThreshold);
+  return runInference(image, detectionThreshold, iouThreshold, classIndices,
+                      methodName);
 }
 } // namespace rnexecutorch::models::object_detection

package/common/rnexecutorch/models/object_detection/ObjectDetection.h

@@ -57,6 +57,13 @@ public:
    * @param imageSource        URI or file path of the input image.
    * @param detectionThreshold Minimum confidence score in (0, 1] for a
    *                           detection to be included in the output.
+   * @param iouThreshold       IoU threshold for non-maximum suppression.
+   * @param classIndices       Optional list of class indices to filter results.
+   *                           Only detections matching these classes will be
+   *                           returned. Pass empty vector to include all
+   * classes.
+   * @param methodName         Name of the method to execute (e.g., "forward",
+   *                           "forward_384", "forward_512", "forward_640").
    *
    * @return A vector of @ref types::Detection objects with bounding boxes,
    *         label strings (resolved via the label names passed to the
@@ -66,16 +73,33 @@ public:
    *         fails.
    */
   [[nodiscard("Registered non-void function")]] std::vector<types::Detection>
-  generateFromString(std::string imageSource, double detectionThreshold);
+  generateFromString(std::string imageSource, double detectionThreshold,
+                     double iouThreshold, std::vector<int32_t> classIndices,
+                     std::string methodName);
   [[nodiscard("Registered non-void function")]] std::vector<types::Detection>
   generateFromFrame(jsi::Runtime &runtime, const jsi::Value &frameData,
-                    double detectionThreshold);
+                    double detectionThreshold, double iouThreshold,
+                    std::vector<int32_t> classIndices, std::string methodName);
   [[nodiscard("Registered non-void function")]] std::vector<types::Detection>
-  generateFromPixels(JSTensorViewIn pixelData, double detectionThreshold);
+  generateFromPixels(JSTensorViewIn pixelData, double detectionThreshold,
+                     double iouThreshold, std::vector<int32_t> classIndices,
+                     std::string methodName);
 
 protected:
-  std::vector<types::Detection> runInference(cv::Mat image,
-                                             double detectionThreshold);
+  /**
+   * @brief Returns the model input size based on the currently loaded method.
+   *
+   * Overrides VisionModel::modelInputSize() to support multi-method models
+   * where each method may have different input dimensions.
+   *
+   * @return The expected input size for the currently loaded method.
+   */
+  cv::Size modelInputSize() const override;
+
+  std::vector<types::Detection>
+  runInference(cv::Mat image, double detectionThreshold, double iouThreshold,
+               const std::vector<int32_t> &classIndices,
+               const std::string &methodName);
 
 private:
   /**
@@ -88,15 +112,37 @@ private:
    *                           bounding boxes back to input coordinates.
    * @param detectionThreshold Confidence threshold below which detections
    *                           are discarded.
+   * @param iouThreshold       IoU threshold for non-maximum suppression.
+   * @param classIndices       Optional list of class indices to filter results.
    *
    * @return Non-max-suppressed detections above the threshold.
    *
    * @throws RnExecutorchError if the model outputs a class index that exceeds
    *         the size of @ref labelNames_.
    */
-  std::vector<types::Detection> postprocess(const std::vector<EValue> &tensors,
-                                            cv::Size originalSize,
-                                            double detectionThreshold);
+  std::vector<types::Detection>
+  postprocess(const std::vector<EValue> &tensors, cv::Size originalSize,
+              double detectionThreshold, double iouThreshold,
+              const std::vector<int32_t> &classIndices);
+
+  /**
+   * @brief Ensures the specified method is loaded, unloading any previous
+   * method if necessary.
+   *
+   * @param methodName Name of the method to load (e.g., "forward",
+   * "forward_384").
+   * @throws RnExecutorchError if the method cannot be loaded.
+   */
+  void ensureMethodLoaded(const std::string &methodName);
+
+  /**
+   * @brief Prepares a set of allowed class indices for filtering detections.
+   *
+   * @param classIndices Vector of class indices to allow.
+   * @return A set containing the allowed class indices.
+   */
+  std::set<int32_t>
+  prepareAllowedClasses(const std::vector<int32_t> &classIndices) const;
 
   /// Optional per-channel mean for input normalisation (set in constructor).
   std::optional<cv::Scalar> normMean_;
@@ -106,6 +152,9 @@ private:
 
   /// Ordered label strings mapping class indices to human-readable names.
   std::vector<std::string> labelNames_;
+
+  /// Name of the currently loaded method (for multi-method models).
+  std::string currentlyLoadedMethod_;
 };
 } // namespace models::object_detection