react-native-image-stitcher 0.15.2 → 0.16.1

package/cpp/crop_quad.cpp

@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// crop_quad.cpp — OpenCV implementation of the free-quad perspective
+// crop (`cropToQuad`), the only net-new native code for item-7 of the
+// first-time-user guidance flow.
+//
+// Pairs with the EXISTING axis-aligned crop (cropToRectAtPath on iOS /
+// cropToRect on Android).  Where that crop slices a sub-rectangle out of
+// the panorama, this one takes 4 user-dragged corners (a skewed
+// quadrilateral) and perspective-rectifies them into an upright rectangle
+// — the editor in src/camera/RectCropPreview.tsx picks `cropToQuad` over
+// `cropToRect` whenever the dragged quad isn't ~axis-aligned.
+//
+// Why a separate .cpp (not folded into stitcher.cpp): the geometry core
+// (quadDstRect / isQuadAcceptable) is the OpenCV-FREE, unit-tested
+// cpp/crop_quad.hpp; this file is JUST the thin cv:: warp around it, kept
+// out of stitcher.cpp's translation unit so the test suite can link the
+// header without pulling in the whole stitch pipeline.
+//
+// Both platform bridges (iOS OpenCVStitcher.mm, Android BatchStitcher.kt)
+// duplicate this warp in their own native language today — see the
+// integrator notes in the item-7 handoff — so this file is the shared
+// C++ reference + the home of the canvasExceedsGuard wiring.  Wire it
+// into a translation unit (the pod / the JNI lib) when the native crop is
+// routed through shared C++ rather than per-platform OpenCV bindings.
+
+// OpenCV's headers redefine NO/YES on platforms whose prefix.pch already
+// has the ObjC bool macros; undef defensively (no-op off iOS).
+#ifdef NO
+#undef NO
+#endif
+#ifdef YES
+#undef YES
+#endif
+
+#include <opencv2/opencv.hpp>
+#include <opencv2/imgproc.hpp>
+
+#include <string>
+#include <vector>
+
+#include "crop_quad.hpp"
+#include "warp_guard.hpp"
+
+namespace retailens {
+
+// Result of a cropToQuad call.  `ok == false` carries a human-readable
+// `error` (the bridge maps it to NSError / Promise.reject); on success
+// `width`/`height` are the written output dimensions.
+struct CropQuadResult {
+  bool ok = false;
+  std::string error;
+  int width = 0;
+  int height = 0;
+};
+
+// Perspective-rectify the quad `q` (4 ORDERED [TL, TR, BR, BL] image-pixel
+// corners) out of the image at `inPath` into an upright w×h rectangle,
+// written to `outPath` as a JPEG at `quality` (clamped to [1, 100]).
+//
+// Guards, in order (each is a hard reject — NO partial output is written):
+//   1. inPath decodes (cv::imread non-empty).
+//   2. isQuadAcceptable — convex + min-area + within the decoded image
+//      bounds.  A degenerate / non-convex / out-of-bounds quad is rejected
+//      here, before any allocation.
+//   3. quadDstRect yields a positive w×h.
+//   4. canvasExceedsGuard(w, h) — the SAME blend-canvas guard the stitch
+//      pipeline uses.  A near-collinear quad whose averaged opposite-edge
+//      lengths still multiply to a multi-gigapixel output (e.g. a 1 px ×
+//      40000 px sliver dragged across a wide pano) can't OOM the device:
+//      the output Mat is never allocated when this fires.
+//
+// The warp itself is cv::getPerspectiveTransform(src → axis-aligned dst)
+// + cv::warpPerspective to a w×h canvas.  `outPath == inPath` is allowed
+// (overwrite in place), matching cropToRect's contract.
+CropQuadResult cropQuadToFile(const std::string& inPath,
+                              const std::string& outPath,
+                              const CropQuad& q,
+                              int quality) {
+  CropQuadResult result;
+
+  cv::Mat img = cv::imread(inPath, cv::IMREAD_COLOR);
+  if (img.empty()) {
+    result.error = "Could not decode image at " + inPath;
+    return result;
+  }
+
+  // Guard 2 — geometry: convex, non-degenerate, inside the decoded image.
+  if (!isQuadAcceptable(q, static_cast<double>(img.cols),
+                        static_cast<double>(img.rows))) {
+    result.error =
+        "Crop quad is degenerate (non-convex, zero-area, or out of bounds)";
+    return result;
+  }
+
+  // Guard 3 — positive destination size.
+  const QuadDstSize dst = quadDstRect(q);
+  if (dst.width <= 0 || dst.height <= 0) {
+    result.error = "Crop quad rectifies to a non-positive rectangle";
+    return result;
+  }
+
+  // Guard 4 — output-canvas OOM net (shared with the stitch pipeline).
+  if (canvasExceedsGuard(dst.width, dst.height)) {
+    result.error = "Crop quad output canvas exceeds the size guard (" +
+                   std::to_string(dst.width) + "x" +
+                   std::to_string(dst.height) + ")";
+    return result;
+  }
+
+  const cv::Point2f src[4] = {
+      cv::Point2f(static_cast<float>(q.tl.x), static_cast<float>(q.tl.y)),
+      cv::Point2f(static_cast<float>(q.tr.x), static_cast<float>(q.tr.y)),
+      cv::Point2f(static_cast<float>(q.br.x), static_cast<float>(q.br.y)),
+      cv::Point2f(static_cast<float>(q.bl.x), static_cast<float>(q.bl.y)),
+  };
+  const cv::Point2f dstPts[4] = {
+      cv::Point2f(0.0f, 0.0f),
+      cv::Point2f(static_cast<float>(dst.width), 0.0f),
+      cv::Point2f(static_cast<float>(dst.width),
+                  static_cast<float>(dst.height)),
+      cv::Point2f(0.0f, static_cast<float>(dst.height)),
+  };
+
+  cv::Mat warped;
+  try {
+    const cv::Mat transform = cv::getPerspectiveTransform(src, dstPts);
+    cv::warpPerspective(img, warped, transform,
+                        cv::Size(dst.width, dst.height),
+                        cv::INTER_LINEAR);
+  } catch (const cv::Exception& e) {
+    result.error = std::string("Perspective warp failed: ") + e.what();
+    return result;
+  }
+  if (warped.empty()) {
+    result.error = "Perspective warp produced an empty image";
+    return result;
+  }
+
+  int q255 = quality;
+  if (q255 < 1) q255 = 1;
+  if (q255 > 100) q255 = 100;
+  const std::vector<int> writeParams = {cv::IMWRITE_JPEG_QUALITY, q255};
+  bool wrote = false;
+  try {
+    wrote = cv::imwrite(outPath, warped, writeParams);
+  } catch (const cv::Exception& e) {
+    result.error = std::string("Could not write cropped image: ") + e.what();
+    return result;
+  }
+  if (!wrote) {
+    result.error = "Could not write cropped image to " + outPath;
+    return result;
+  }
+
+  result.ok = true;
+  result.width = warped.cols;
+  result.height = warped.rows;
+  return result;
+}
+
+}  // namespace retailens

package/cpp/crop_quad.hpp

@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+#include <cmath>
+#include <cstdint>
+
+// ─────────────────────────────────────────────────────────────────────
+// Free-quad crop geometry — the OpenCV-FREE, unit-testable core of the
+// item-7 draggable-corner crop (`cropToQuad`).  The actual perspective
+// warp (cv::getPerspectiveTransform + cv::warpPerspective) lives next to
+// the existing axis-aligned crop in the platform bridges; this header is
+// JUST the two pure predicates worth testing in isolation:
+//
+//   - quadDstRect(quad)       → the {w,h} of the upright destination
+//                               rectangle the quad rectifies INTO.
+//   - isQuadAcceptable(quad)  → convex + min-area + within-bounds gate
+//                               the warp must pass before allocating.
+//
+// Header-only + zero cv:: dependency on purpose — same posture as
+// warp_guard.hpp.  These are the C++ twins of the JS-side helpers in
+// src/camera/cropGeometry.ts (rectSizeForQuad / isQuadValid); the math
+// is duplicated per the repo's "duplicate stage code, DRY when proven"
+// convention so a native caller never has to round-trip through JS to
+// validate a quad.
+//
+// Corner-order contract: every function below assumes the 4 points are
+// in canonical [TL, TR, BR, BL] (clockwise from top-left) winding — the
+// order src/camera/cropGeometry.ts:orderQuadCorners produces and the
+// order RectCropResult.quad carries.  Pass un-ordered points and the
+// edge-length / convexity math is meaningless.
+// ─────────────────────────────────────────────────────────────────────
+
+namespace retailens {
+
+// A single corner in image-pixel space (origin = image top-left).
+struct QuadPoint {
+  double x = 0.0;
+  double y = 0.0;
+};
+
+// Exactly four corners, in [TL, TR, BR, BL] order.
+struct CropQuad {
+  QuadPoint tl;
+  QuadPoint tr;
+  QuadPoint br;
+  QuadPoint bl;
+};
+
+// Integer destination-rectangle size the quad rectifies into.
+struct QuadDstSize {
+  int width = 0;
+  int height = 0;
+};
+
+// Euclidean distance between two corners.
+inline double quadPointDistance(const QuadPoint& a, const QuadPoint& b) {
+  const double dx = a.x - b.x;
+  const double dy = a.y - b.y;
+  return std::sqrt(dx * dx + dy * dy);
+}
+
+// Target rectangle size for the perspective `dst` quad, derived from the
+// 4 ORDERED ([TL, TR, BR, BL]) image-pixel corners:
+//   - width  = average of the top edge (TL→TR) and bottom edge (BL→BR).
+//   - height = average of the left edge (TL→BL) and right edge (TR→BR).
+//
+// Averaging opposite edges gives a stable output size for a skewed quad
+// (each pair of opposite edges differs under perspective; the mean is the
+// least-distorting target).  Rounds to whole pixels — the warp allocates
+// an integer-sized output Mat.  Mirrors cropGeometry.ts:rectSizeForQuad
+// so iOS / Android / JS agree on the output dimensions bit-for-bit.
+//
+// A degenerate quad (all-collinear, zero-size) yields a 0×0 or 1×N size;
+// the caller GUARDS this with isQuadAcceptable + canvasExceedsGuard before
+// allocating, so this function never has to reject — it only measures.
+inline QuadDstSize quadDstRect(const CropQuad& q) {
+  const double top = quadPointDistance(q.tl, q.tr);
+  const double bottom = quadPointDistance(q.bl, q.br);
+  const double left = quadPointDistance(q.tl, q.bl);
+  const double right = quadPointDistance(q.tr, q.br);
+  QuadDstSize size;
+  size.width = static_cast<int>(std::lround((top + bottom) / 2.0));
+  size.height = static_cast<int>(std::lround((left + right) / 2.0));
+  return size;
+}
+
+// 2× the signed area of the quad via the shoelace formula.  Positive for
+// one winding, negative for the other, ~0 for a degenerate (collinear)
+// quad.  Sign is winding-dependent; callers that care about magnitude
+// (min-area) take the absolute value.
+inline double quadSignedArea2(const CropQuad& q) {
+  const QuadPoint p[4] = {q.tl, q.tr, q.br, q.bl};
+  double sum = 0.0;
+  for (int i = 0; i < 4; ++i) {
+    const QuadPoint& a = p[i];
+    const QuadPoint& b = p[(i + 1) % 4];
+    sum += a.x * b.y - b.x * a.y;
+  }
+  return sum;
+}
+
+// Convexity test: all four consecutive edge cross-products share one sign
+// (zero allowed for a straight, axis-aligned corner).  Rejects the self-
+// intersecting "bowtie" quads a free-drag editor can produce, which a
+// perspective warp can't rectify.  Winding-agnostic.  Mirrors the
+// `isConvex` helper in cropGeometry.ts.
+inline bool quadIsConvex(const CropQuad& q) {
+  const QuadPoint p[4] = {q.tl, q.tr, q.br, q.bl};
+  int sign = 0;
+  for (int i = 0; i < 4; ++i) {
+    const QuadPoint& a = p[i];
+    const QuadPoint& b = p[(i + 1) % 4];
+    const QuadPoint& c = p[(i + 2) % 4];
+    const double cross =
+        (b.x - a.x) * (c.y - b.y) - (b.y - a.y) * (c.x - b.x);
+    if (cross != 0.0) {
+      const int s = (cross > 0.0) ? 1 : -1;
+      if (sign == 0) {
+        sign = s;
+      } else if (s != sign) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+// True when the 4 ordered ([TL, TR, BR, BL]) corners form a quad the
+// perspective warp can safely rectify:
+//   1. **Convex** — no self-intersection (quadIsConvex).
+//   2. **Non-degenerate area** — |signed area| ≥ `minArea` (default 1 px²);
+//      rejects all-collinear / zero-size quads.
+//   3. **Within bounds** — every corner lies inside [0..imageW]×[0..imageH]
+//      (a half-pixel epsilon absorbs the lround in the JS letterbox
+//      inverse).  Pass imageW <= 0 OR imageH <= 0 to SKIP the bounds
+//      check (the caller doesn't know the image size yet).
+//
+// The companion to warp_guard.hpp:canvasExceedsGuard — that guards the
+// OUTPUT canvas against an OOM; this guards the INPUT quad against being
+// geometrically unwarpable.  Both must pass before warpPerspective runs.
+inline bool isQuadAcceptable(const CropQuad& q,
+                             double imageW = 0.0,
+                             double imageH = 0.0,
+                             double minArea = 1.0) {
+  if (!quadIsConvex(q)) {
+    return false;
+  }
+  if (std::fabs(quadSignedArea2(q)) < minArea * 2.0) {
+    return false;
+  }
+  if (imageW > 0.0 && imageH > 0.0) {
+    const QuadPoint p[4] = {q.tl, q.tr, q.br, q.bl};
+    const double eps = 0.5;
+    for (int i = 0; i < 4; ++i) {
+      if (p[i].x < -eps || p[i].x > imageW + eps ||
+          p[i].y < -eps || p[i].y > imageH + eps) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+}  // namespace retailens

package/cpp/keyframe_gate.cpp

@@ -356,6 +356,11 @@ struct KeyframeGate::Impl {
     // the downscale factor.  Re-set whenever prevFrameGrayWork is.
     int32_t                     prevFrameOrigWidth  = 0;
     int32_t                     prevFrameOrigHeight = 0;
+    // 2026-06-16 (audit #4) — the CURRENT frame's downscaled working image,
+    // produced by ingestWorkingFrame() (under the JNI pin) and consumed by
+    // evaluateWithWorkingMat() (after the pin is released).  Empty unless an
+    // ingest is pending.  Owned (downscaleToWorking clones / resizes).
+    cv::Mat                     pendingWorkMat;
 };
 
 // Compile-time layout check on the shared POD struct — ensures iOS
@@ -750,6 +755,45 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
     int32_t height,
     int32_t stride,
     int64_t monotonicNowMs)
+{
+    // 2026-06-16 (audit #4) — thin wrapper = ingest (reads pixels) + evaluate.
+    // Kept for iOS / non-pinned callers; the Android JNI calls the two halves
+    // separately so the heavy OpenCV runs OUTSIDE the GC-pinned critical region.
+    ingestWorkingFrame(grayData, width, height, stride);
+    return evaluateWithWorkingMat(pose, latchedPlane, width, height, monotonicNowMs);
+}
+
+void KeyframeGate::ingestWorkingFrame(
+    const uint8_t* grayData,
+    int32_t width,
+    int32_t height,
+    int32_t stride)
+{
+    Impl& s = *pImpl_;
+    s.pendingWorkMat.release();
+    // Only the Flow strategy reads pixels; disabled / force-accept / Pose ignore
+    // them (the Pose path is deliberately OpenCV-free).  For those — and for
+    // invalid input — stash nothing; evaluateWithWorkingMat short-circuits
+    // without a frame.  This is the ONLY step that touches `grayData`, so the
+    // JNI can ReleasePrimitiveArrayCritical immediately after it returns.
+    if (!s.enabled || s.forceAcceptNext || s.strategy != GateStrategy::Flow
+        || grayData == nullptr || width <= 0 || height <= 0 || stride < width) {
+        return;
+    }
+    // Non-owning view over the (pinned) bytes; downscaleToWorking returns an
+    // OWNED Mat (clone or resize output), so it stays valid after the pin drops.
+    cv::Mat currGrayFull(height, width, CV_8UC1,
+                        const_cast<uint8_t*>(grayData),
+                        static_cast<size_t>(stride));
+    s.pendingWorkMat = downscaleToWorking(currGrayFull);
+}
+
+KeyframeGateDecision KeyframeGate::evaluateWithWorkingMat(
+    const Pose& pose,
+    const PlaneTransform* latchedPlane,
+    int32_t origWidth,
+    int32_t origHeight,
+    int64_t monotonicNowMs)
 {
     Impl& s = *pImpl_;
     // Stash the monotonic stamp (see evaluate()).  The Pose-strategy
@@ -785,19 +829,14 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
         return evaluate(pose, latchedPlane, s.currentNowMs);
     }
 
-    // Flow path — wrap incoming pixel data as a non-owning cv::Mat
-    // and downscale to working resolution.  The non-owning view is
-    // SAFE because we deep-copy (via clone) before storing on Impl.
-    if (grayData == nullptr || width <= 0 || height <= 0 || stride < width) {
-        // Defensive: caller forgot to supply image data despite
-        // strategy=Flow.  Fall back to pose path so we degrade
-        // gracefully rather than crashing on a null deref.
+    // Flow path — operate on the working frame ingestWorkingFrame() stashed.
+    // Empty ⇒ ingest declined (null/invalid grayData under Flow); fall back to
+    // the pose path so we degrade gracefully rather than deref a null frame.
+    cv::Mat currGrayWork = std::move(s.pendingWorkMat);
+    s.pendingWorkMat.release();
+    if (currGrayWork.empty()) {
         return evaluate(pose, latchedPlane, s.currentNowMs);
     }
-    cv::Mat currGrayFull(height, width, CV_8UC1,
-                        const_cast<uint8_t*>(grayData),
-                        static_cast<size_t>(stride));
-    cv::Mat currGrayWork = downscaleToWorking(currGrayFull);
 
     // §4 — first-frame accept under Flow.  No prev to track against;
     // we anchor here and detect features so subsequent frames have
@@ -811,8 +850,8 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
             s.flowMinDistance);
         s.prevFrameGrayWork = currGrayWork;  // clone-owned via downscale path
         s.prevFeatures      = std::move(features);
-        s.prevFrameOrigWidth  = width;
-        s.prevFrameOrigHeight = height;
+        s.prevFrameOrigWidth  = origWidth;
+        s.prevFrameOrigHeight = origHeight;
         s.lastAcceptedPose    = pose;
         s.lastAcceptSteadyMs  = s.currentNowMs;
         s.acceptedCount       = 1;
@@ -970,8 +1009,8 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
         s.flowMinDistance);
     s.prevFrameGrayWork = currGrayWork;   // owned via downscale's clone
     s.prevFeatures      = std::move(nextFeatures);
-    s.prevFrameOrigWidth  = width;
-    s.prevFrameOrigHeight = height;
+    s.prevFrameOrigWidth  = origWidth;
+    s.prevFrameOrigHeight = origHeight;
     s.lastAcceptedPose    = pose;
     s.lastAcceptSteadyMs  = s.currentNowMs;
     s.acceptedCount      += 1;

package/cpp/keyframe_gate.hpp

@@ -248,6 +248,39 @@ public:
                                            int32_t stride,
                                            int64_t monotonicNowMs = -1);
 
+    // ── Split entry point (2026-06-16, audit #4) ──────────────────
+    //
+    // Android JNI pins the gray byte[] with GetPrimitiveArrayCritical, which
+    // blocks the GC for the pin's whole duration.  Running the heavy OpenCV
+    // (goodFeaturesToTrack / optical flow, multi-ms) inside that window stalls
+    // the frame-rate producer thread and violates the JNI no-blocking-between-
+    // critical rule.  This pair splits the work so ONLY the pinned-buffer read
+    // happens under the pin:
+    //
+    //   ingestWorkingFrame(grayData, w, h, stride)   // <- under the pin
+    //       Builds the downscaled working frame (the only step that reads the
+    //       pinned pixels) and stashes it INSIDE the gate.  Does nothing (no
+    //       pixel read) for Pose/disabled/force-accept/invalid input — those
+    //       don't need the frame.  cv::Mat is kept out of this header so the
+    //       JNI needn't link OpenCV; the frame lives on the gate's Impl.
+    //
+    //   evaluateWithWorkingMat(pose, plane, origW, origH)   // <- pin released
+    //       Runs the rest of the gate (the heavy OpenCV) on the stashed working
+    //       frame, OUTSIDE the critical section.  origW/origH are the ORIGINAL
+    //       full-res dims (the working frame is downscaled).
+    //
+    // evaluateWithFrame() above is exactly ingest+evaluate in sequence — kept
+    // for iOS and any non-pinned caller (DRY: it delegates to these two).
+    void ingestWorkingFrame(const uint8_t* grayData,
+                            int32_t width,
+                            int32_t height,
+                            int32_t stride);
+    KeyframeGateDecision evaluateWithWorkingMat(const Pose& pose,
+                                                const PlaneTransform* latchedPlane,
+                                                int32_t origWidth,
+                                                int32_t origHeight,
+                                                int64_t monotonicNowMs = -1);
+
     // ── State accessors (read-only, post-evaluate) ────────────────
     int32_t getAcceptedCount() const;
     int32_t getMaxCount() const;