react-native-image-stitcher 0.15.2 → 0.16.0

package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm

@@ -58,6 +58,11 @@
 // The header lives in the SDK's `cpp/` dir and is on the pod's
 // HEADER_SEARCH_PATHS (see RNImageStitcher.podspec).
 #import "stitcher.hpp"
+// item-7 cropToQuad: the OpenCV-free quad geometry (quadDstRect /
+// isQuadAcceptable) + the shared canvas OOM guard.  Same `cpp/`
+// HEADER_SEARCH_PATHS as stitcher.hpp.
+#import "crop_quad.hpp"
+#import "warp_guard.hpp"
 #import <UIKit/UIKit.h>
 #import <AVFoundation/AVFoundation.h>
 #import <os/log.h>
@@ -262,6 +267,12 @@ NSString *const RNImageStitcherErrorDomain = @"RNImageStitcherErrorDomain";
 // RNStitchResult
 // ─────────────────────────────────────────────────────────────────────
 
+// Redeclare debugSummary as readwrite internally so it can be set after the
+// designated initializer (keeps the init signature unchanged).
+@interface RNStitchResult ()
+@property (nonatomic, copy, readwrite) NSString *debugSummary;
+@end
+
 @implementation RNStitchResult
 
 - (instancetype)initWithOutputPath:(NSString *)outputPath
@@ -280,6 +291,7 @@ NSString *const RNImageStitcherErrorDomain = @"RNImageStitcherErrorDomain";
     _framesRequested = framesRequested;
     _framesIncluded = framesIncluded;
     _finalConfidenceThresh = finalConfidenceThresh;
+    _debugSummary = @"";
   }
   return self;
 }
@@ -411,6 +423,7 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
                                   captureOrientation:(NSString *)captureOrientation
                                 useInscribedRectCrop:(BOOL)useInscribedRectCrop
                                           stitchMode:(NSString *)stitchMode
+                                   useManualPipeline:(BOOL)useManualPipeline
                                                error:(NSError **)error {
   // ── Phase 2 (2026-05-16): delegated to shared C++ ───────────────────
   //
@@ -438,7 +451,7 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
   // working until they are updated.  The shared C++ has its own
   // defaults but we want the wrapper to be tolerant of nil inputs
   // from Swift / Obj-C callers that grew up against the legacy API.
-  if (warperType == nil || warperType.length == 0) warperType = @"plane";
+  if (warperType == nil || warperType.length == 0) warperType = @"spherical";
   if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
   if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
   if (captureOrientation == nil || captureOrientation.length == 0) captureOrientation = @"portrait";
@@ -479,12 +492,22 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
   cfg.availableRamMB =
       (double)NSProcessInfo.processInfo.physicalMemory
       / (1024.0 * 1024.0);
-  // Route to the manual cv::detail::* pipeline; the high-level
-  // cv::Stitcher::create path (Android's default) is unsuitable for
-  // iOS's shelf-pan capture shape (compose-MP defaults, graphcut at
-  // compose-MP, BA convergence params — see stitcher.hpp comment
-  // block).
-  cfg.useManualPipeline = true;
+  // 2026-06-15 — DEFAULT to the MANUAL cv::detail pipeline.  ALL the memory/OOM
+  // hardening lives on the manual path (PreStitchMemoryAbort, RAM-aware
+  // canvas-budget downscale, STREAM/BATCH held-set routing, the black-canvas
+  // utilization guard); the high-level cv::Stitcher path calls NONE of it.  So
+  // manual is both the user's preferred output AND the memory-safe one.
+  //
+  // WARPER: NOT hardcoded — cfg.warperType carries the caller's choice (set from
+  // the JS `warperType`, which defaults to "spherical" and is settable via the
+  // ⚙️ panel / the host's `defaultWarper` prop).  The JS default is the single
+  // source of truth now.  Choosing "plane" re-arms the dynamic plane→spherical
+  // fallback + divergence switch in the manual pipeline (they only fire when
+  // warperType != "spherical").
+  //
+  // The pipeline is caller-driven: batch capture passes YES (manual, default
+  // output); the on-demand high-level tab re-stitches with NO.
+  cfg.useManualPipeline = useManualPipeline;
 
   // Marshal NSArray<NSString*> → std::vector<std::string>.  Strip the
   // `file://` scheme that some callers attach so the shared C++ can
@@ -566,6 +589,18 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
              framesRequested:framesRequested
               framesIncluded:(NSInteger)r.framesIncluded
        finalConfidenceThresh:r.finalConfidenceThresh];
+      if (!r.debugSummary.empty()) {
+        result.debugSummary =
+            [NSString stringWithUTF8String:r.debugSummary.c_str()];
+      }
+      // 2026-06-15 — the eager A/B harness that ALSO stitched the high-level
+      // alt on EVERY capture has been REMOVED.  Manual is now the default (this
+      // method), so computing high-level eagerly was pure wasted work —
+      // especially while profiling memory/perf — when the user isn't viewing
+      // it.  The keyframe JPEGs are retained on disk so high-level can be
+      // produced ON DEMAND (follow-up: a `useManualPipeline` param on this
+      // method lets `refinePanorama` re-stitch them via the high-level path
+      // when the user switches to the high-level tab).
     } else {
       // Map StitchErrorCode → NSError.code.  Preserves the existing
       // 9001/9002/9003/1001/9007 sentinels the JS UX layer already
@@ -863,6 +898,7 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
           captureOrientation:nil
         useInscribedRectCrop:NO
                   stitchMode:nil
+           useManualPipeline:NO  // legacy video path keeps high-level cv::Stitcher
                        error:&stitchErr];
 
   // Always tear down the tmp dir, success or fail — leaving
@@ -1063,6 +1099,154 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
   };
 }
 
+// item-7 — free-quad perspective crop.  Mirrors cropToRectAtPath, but
+// instead of an axis-aligned sub-rectangle it takes 4 user-dragged
+// corners in IMAGE-PIXEL space (ordered TL, TR, BR, BL by the JS editor's
+// orderQuadCorners) and rectifies them to an upright rectangle via
+// cv::getPerspectiveTransform + cv::warpPerspective.  The destination
+// size + the convex/min-area/in-bounds gate come from the shared OpenCV-
+// free cpp/crop_quad.hpp so iOS / Android / JS agree bit-for-bit; the
+// output canvas is GUARDED with the same canvasExceedsGuard the stitch
+// pipeline uses so a near-collinear quad can't OOM a multi-MP panorama.
++ (NSDictionary<NSString *, NSNumber *> *)cropToQuadAtPath:(NSString *)imagePath
+                                                      tlX:(double)tlX
+                                                      tlY:(double)tlY
+                                                      trX:(double)trX
+                                                      trY:(double)trY
+                                                      brX:(double)brX
+                                                      brY:(double)brY
+                                                      blX:(double)blX
+                                                      blY:(double)blY
+                                                  quality:(NSInteger)quality
+                                                    error:(NSError **)error {
+  NSString *cleaned = normalizeImagePath(imagePath);
+  if (![[NSFileManager defaultManager] fileExistsAtPath:cleaned]) {
+    if (error) {
+      *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
+                                   code:1020
+                               userInfo:@{
+        NSLocalizedDescriptionKey:
+          [NSString stringWithFormat:@"Image not found: %@", imagePath],
+      }];
+    }
+    return nil;
+  }
+
+  std::string nativePath(cleaned.UTF8String);
+  cv::Mat img = cv::imread(nativePath, cv::IMREAD_COLOR);
+  if (img.empty()) {
+    if (error) {
+      *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
+                                   code:1021
+                               userInfo:@{
+        NSLocalizedDescriptionKey:
+          [NSString stringWithFormat:@"Could not decode image at %@", imagePath],
+      }];
+    }
+    return nil;
+  }
+
+  retailens::CropQuad quad;
+  quad.tl = {tlX, tlY};
+  quad.tr = {trX, trY};
+  quad.br = {brX, brY};
+  quad.bl = {blX, blY};
+
+  // Geometry gate — convex, non-degenerate, inside the decoded image.
+  if (!retailens::isQuadAcceptable(quad, (double)img.cols, (double)img.rows)) {
+    if (error) {
+      *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
+                                   code:1023
+                               userInfo:@{
+        NSLocalizedDescriptionKey:
+          @"Crop quad is degenerate (non-convex, zero-area, or out of bounds)",
+      }];
+    }
+    return nil;
+  }
+
+  const retailens::QuadDstSize dst = retailens::quadDstRect(quad);
+  // Output-canvas OOM net — the same guard the stitch pipeline uses.
+  if (dst.width <= 0 || dst.height <= 0 ||
+      retailens::canvasExceedsGuard(dst.width, dst.height)) {
+    if (error) {
+      *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
+                                   code:1024
+                               userInfo:@{
+        NSLocalizedDescriptionKey:
+          [NSString stringWithFormat:
+            @"Crop quad output canvas is degenerate or exceeds the size guard (%dx%d)",
+            dst.width, dst.height],
+      }];
+    }
+    return nil;
+  }
+
+  const cv::Point2f src[4] = {
+    cv::Point2f((float)tlX, (float)tlY),
+    cv::Point2f((float)trX, (float)trY),
+    cv::Point2f((float)brX, (float)brY),
+    cv::Point2f((float)blX, (float)blY),
+  };
+  const cv::Point2f dstPts[4] = {
+    cv::Point2f(0.0f, 0.0f),
+    cv::Point2f((float)dst.width, 0.0f),
+    cv::Point2f((float)dst.width, (float)dst.height),
+    cv::Point2f(0.0f, (float)dst.height),
+  };
+
+  cv::Mat warped;
+  // OpenCV throws cv::Exception (a C++ exception) — catch with a C++
+  // try/catch, NOT @try/@catch (which only traps NSException).
+  try {
+    cv::Mat transform = cv::getPerspectiveTransform(src, dstPts);
+    cv::warpPerspective(img, warped, transform,
+                        cv::Size(dst.width, dst.height), cv::INTER_LINEAR);
+  } catch (const cv::Exception &e) {
+    if (error) {
+      *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
+                                   code:1025
+                               userInfo:@{
+        NSLocalizedDescriptionKey:
+          [NSString stringWithFormat:@"Perspective warp failed: %s", e.what()],
+      }];
+    }
+    return nil;
+  }
+  if (warped.empty()) {
+    if (error) {
+      *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
+                                   code:1025
+                               userInfo:@{
+        NSLocalizedDescriptionKey: @"Perspective warp produced an empty image",
+      }];
+    }
+    return nil;
+  }
+
+  int q = (int)quality;
+  if (q < 1) { q = 1; }
+  if (q > 100) { q = 100; }
+  std::vector<int> writeParams = { cv::IMWRITE_JPEG_QUALITY, q };
+  bool ok = cv::imwrite(nativePath, warped, writeParams);
+  if (!ok) {
+    if (error) {
+      *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
+                                   code:1022
+                               userInfo:@{
+        NSLocalizedDescriptionKey:
+          [NSString stringWithFormat:@"Could not rewrite image at %@", imagePath],
+      }];
+    }
+    return nil;
+  }
+
+  return @{
+    @"width":  @((NSInteger)warped.cols),
+    @"height": @((NSInteger)warped.rows),
+  };
+}
+
 + (NSDictionary *)debugMaskOverlayAtPath:(NSString *)imagePath
                                threshold:(NSInteger)threshold
                                    error:(NSError **)error {

package/ios/Sources/RNImageStitcher/RNSARSession.swift

@@ -154,6 +154,10 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         attributes: .concurrent
     )
     private static let MAX_POSE_LOG = 600  // ~10 s @ 60Hz
+    /// AR keyframe long-edge budget (px) — downscale every device's frame to
+    /// this before encoding so stitch memory is consistent cross-device.
+    /// Mirrors Android's AR_KEYFRAME_MAX_LONG_EDGE.
+    private static let arKeyframeMaxLongEdge: CGFloat = 640
 
     /// Latest tracking state.  Read by JS for UI feedback.
     @objc public private(set) var currentTrackingState: RNSARTrackingState = .notAvailable
@@ -472,6 +476,18 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         // small text and packaging detail.
         config.isAutoFocusEnabled = true
 
+        // Option B — prefer the 4:3 videoFormat for full sensor FOV; the
+        // keyframe is downscaled to arKeyframeMaxLongEdge below so memory
+        // stays consistent across devices regardless of the format res.
+        if let fmt = ARWorldTrackingConfiguration.supportedVideoFormats.min(by: { a, b in
+            let da = abs(a.imageResolution.width / a.imageResolution.height - 4.0 / 3.0)
+            let db = abs(b.imageResolution.width / b.imageResolution.height - 4.0 / 3.0)
+            if abs(da - db) > 0.001 { return da < db }
+            return a.imageResolution.width * a.imageResolution.height
+                 < b.imageResolution.width * b.imageResolution.height
+        }) {
+            config.videoFormat = fmt
+        }
         arSession.run(config, options: [.resetTracking, .removeExistingAnchors])
         // V16-diag — log the chosen video format so we can correlate
         // batch-keyframe memory with ARFrame resolution.  iPhone Pro
@@ -855,8 +871,16 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         case "portrait-upside-down":  exifOrientation = .left
         default:                       exifOrientation = .right  // portrait + unknown
         }
-        let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
+        var ciImage = CIImage(cvPixelBuffer: pixelBuffer)
             .oriented(exifOrientation)
+        // AR keyframe downscale guard — normalise long edge to the budget so
+        // every device produces a ~0.3 MP keyframe (cross-device-consistent
+        // stitch memory).  Mirrors Android's downscale in YuvImageConverter.
+        let kfLongEdge = max(ciImage.extent.width, ciImage.extent.height)
+        if kfLongEdge > Self.arKeyframeMaxLongEdge {
+            let kfScale = Self.arKeyframeMaxLongEdge / kfLongEdge
+            ciImage = ciImage.transformed(by: CGAffineTransform(scaleX: kfScale, y: kfScale))
+        }
         let context = CIContext(options: nil)
         guard let cgImage = context.createCGImage(
             ciImage,

package/ios/Sources/RNImageStitcher/Stitcher.swift

@@ -161,7 +161,11 @@ public enum Stitcher {
         // expose stitchMode in its options dict yet.  nil falls
         // through to Panorama in OpenCVStitcher.mm (preserves
         // historical behaviour).
-        stitchMode: nil
+        stitchMode: nil,
+        // Generic one-shot API keeps the high-level cv::Stitcher pipeline
+        // (its historical behaviour); the batch capture is what defaults to
+        // manual.  warperType "plane" above only matters on the manual path.
+        useManualPipeline: false
       )
       return StitchResult(
         outputPath: result.outputPath,
@@ -239,6 +243,35 @@ public enum Stitcher {
     }
   }
 
+  /// item-7 — perspective-rectify the quadrilateral with corners
+  /// `(tlX,tlY) (trX,trY) (brX,brY) (blX,blY)` (IMAGE-PIXEL space, ordered
+  /// TL→TR→BR→BL) out of `imagePath` into an upright rectangle, overwrite
+  /// in place, re-encode at `quality`.  Pairs with the axis-aligned
+  /// `cropToRect`; the JS editor picks this when the dragged quad isn't
+  /// ~rectangular.  Returns the rectified `{ width, height }`.
+  public static func cropToQuad(
+    imagePath: String,
+    tlX: Double, tlY: Double,
+    trX: Double, trY: Double,
+    brX: Double, brY: Double,
+    blX: Double, blY: Double,
+    quality: Int
+  ) throws -> (width: Int, height: Int) {
+    do {
+      let d = try OpenCVStitcher.cropToQuad(
+        atPath: imagePath,
+        tlX: tlX, tlY: tlY,
+        trX: trX, trY: trY,
+        brX: brX, brY: brY,
+        blX: blX, blY: blY,
+        quality: quality
+      )
+      return (width: d["width"]?.intValue ?? 0, height: d["height"]?.intValue ?? 0)
+    } catch let nsError as NSError {
+      throw StitcherError.fromNSError(nsError)
+    }
+  }
+
   /// v0.15 debug — write a red-tinted mask overlay (excluded pixels =
   /// red) next to the image and report what fraction the brightness mask
   /// drops. Pairs with the inscribed-rect debug harness.

package/ios/Sources/RNImageStitcher/StitcherBridge.m

@@ -34,6 +34,11 @@ RCT_EXTERN_METHOD(cropToRect:(NSDictionary *)options
                   resolver:(RCTPromiseResolveBlock)resolver
                   rejecter:(RCTPromiseRejectBlock)rejecter)
 
+// item-7 perspective crop (free-quad rectify; pairs with cropToRect).
+RCT_EXTERN_METHOD(cropToQuad:(NSDictionary *)options
+                  resolver:(RCTPromiseResolveBlock)resolver
+                  rejecter:(RCTPromiseRejectBlock)rejecter)
+
 RCT_EXTERN_METHOD(debugMaskOverlay:(NSDictionary *)options
                   resolver:(RCTPromiseResolveBlock)resolver
                   rejecter:(RCTPromiseRejectBlock)rejecter)

package/ios/Sources/RNImageStitcher/StitcherBridge.swift

@@ -265,6 +265,62 @@ public class StitcherBridge: NSObject {
     }
   }
 
+  /// item-7 — perspective-rectify a user-dragged quad.  `options` carries
+  /// `imagePath` + the 4 IMAGE-PIXEL corners as a flat `quad` array of 8
+  /// numbers `[tlX, tlY, trX, trY, brX, brY, blX, blY]` (ordered
+  /// TL→TR→BR→BL by the JS editor) + optional `quality` (default 90).
+  /// Resolves the rectified `{ width, height }`.
+  @objc(cropToQuad:resolver:rejecter:)
+  public func cropToQuad(
+    options: NSDictionary,
+    resolver: @escaping RCTPromiseResolveBlock,
+    rejecter: @escaping RCTPromiseRejectBlock
+  ) {
+    guard let imagePath = options["imagePath"] as? String else {
+      rejecter("invalid-options", "imagePath must be a string", nil)
+      return
+    }
+    guard let quad = options["quad"] as? [NSNumber], quad.count == 8 else {
+      rejecter(
+        "invalid-options",
+        "quad must be an array of 8 numbers [tlX,tlY,trX,trY,brX,brY,blX,blY]",
+        nil
+      )
+      return
+    }
+    let p = quad.map { $0.doubleValue }
+    let quality = (options["quality"] as? NSNumber)?.intValue ?? 90
+    DispatchQueue.global(qos: .userInitiated).async {
+      do {
+        let dims = try Stitcher.cropToQuad(
+          imagePath: imagePath,
+          tlX: p[0], tlY: p[1],
+          trX: p[2], trY: p[3],
+          brX: p[4], brY: p[5],
+          blX: p[6], blY: p[7],
+          quality: quality
+        )
+        resolver([
+          "width": dims.width,
+          "height": dims.height,
+        ])
+      } catch let err as StitcherError {
+        switch err {
+        case .insufficientFrames(let count):
+          rejecter("insufficient-frames", "(unexpected for cropToQuad) frames=\(count)", err)
+        case .readFailed(let path):
+          rejecter("read-failed", "Could not read image: \(path)", err)
+        case .writeFailed(let path):
+          rejecter("write-failed", "Could not write image: \(path)", err)
+        case .opencvFailed(let code, let message):
+          rejecter("opencv-failed-\(code)", message, err)
+        }
+      } catch {
+        rejecter("unknown", "Unexpected cropToQuad failure: \(error)", error)
+      }
+    }
+  }
+
   /// v0.15 debug — write a red-tinted mask overlay for `imagePath`
   /// (excluded pixels red). `threshold` optional (default 1, matching the
   /// inscribed-rect mask). Resolves

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-image-stitcher",
-  "version": "0.15.2",
+  "version": "0.16.0",
   "description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,6 +8,8 @@
     "dist/**/*.js",
     "dist/**/*.d.ts",
     "src",
+    "!src/**/__tests__",
+    "!src/**/__tests__/**",
     "ios/Sources",
     "ios/Package.swift",
     "android/build.gradle",
@@ -15,6 +17,8 @@
     "android/src/main/cpp",
     "android/src/main/AndroidManifest.xml",
     "cpp",
+    "!cpp/tests",
+    "!cpp/tests/**",
     "scripts/postinstall-fetch-binaries.js",
     "scripts/opencv-version.txt",
     "RNImageStitcher.podspec",