react-native-image-stitcher 0.15.2 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/CHANGELOG.md +171 -1
  2. package/README.md +131 -5
  3. package/android/src/main/cpp/image_stitcher_jni.cpp +154 -13
  4. package/android/src/main/cpp/keyframe_gate_jni.cpp +15 -8
  5. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +223 -1
  6. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +220 -87
  7. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +1 -1
  8. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +7 -36
  9. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +14 -8
  10. package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +39 -1
  11. package/cpp/crop_quad.cpp +162 -0
  12. package/cpp/crop_quad.hpp +163 -0
  13. package/cpp/keyframe_gate.cpp +54 -15
  14. package/cpp/keyframe_gate.hpp +33 -0
  15. package/cpp/stitcher.cpp +1122 -132
  16. package/cpp/stitcher.hpp +62 -0
  17. package/cpp/warp_guard.hpp +212 -0
  18. package/dist/camera/Camera.d.ts +209 -12
  19. package/dist/camera/Camera.js +575 -36
  20. package/dist/camera/CameraView.js +35 -16
  21. package/dist/camera/CaptureCountdownOverlay.d.ts +70 -0
  22. package/dist/camera/CaptureCountdownOverlay.js +239 -0
  23. package/dist/camera/CaptureFrameCounterOverlay.d.ts +58 -0
  24. package/dist/camera/CaptureFrameCounterOverlay.js +153 -0
  25. package/dist/camera/CaptureMemoryPill.d.ts +24 -8
  26. package/dist/camera/CaptureMemoryPill.js +37 -12
  27. package/dist/camera/CapturePreview.js +2 -1
  28. package/dist/camera/CaptureStatusOverlay.d.ts +11 -4
  29. package/dist/camera/CaptureStatusOverlay.js +22 -5
  30. package/dist/camera/CaptureThumbnailStrip.js +2 -1
  31. package/dist/camera/LateralMotionModal.d.ts +85 -0
  32. package/dist/camera/LateralMotionModal.js +134 -0
  33. package/dist/camera/PanHowToOverlay.d.ts +76 -0
  34. package/dist/camera/PanHowToOverlay.js +222 -0
  35. package/dist/camera/PanoramaBandOverlay.d.ts +2 -1
  36. package/dist/camera/PanoramaBandOverlay.js +9 -3
  37. package/dist/camera/PanoramaSettings.d.ts +8 -6
  38. package/dist/camera/PanoramaSettings.js +19 -1
  39. package/dist/camera/PanoramaSettingsModal.js +4 -4
  40. package/dist/camera/RectCropPreview.d.ts +135 -0
  41. package/dist/camera/RectCropPreview.js +370 -0
  42. package/dist/camera/RotateToLandscapePrompt.d.ts +87 -0
  43. package/dist/camera/RotateToLandscapePrompt.js +138 -0
  44. package/dist/camera/buildPanoramaInitialSettings.d.ts +19 -2
  45. package/dist/camera/buildPanoramaInitialSettings.js +9 -0
  46. package/dist/camera/cameraErrorMessages.d.ts +30 -1
  47. package/dist/camera/cameraErrorMessages.js +26 -10
  48. package/dist/camera/cameraGuidanceCopy.d.ts +87 -0
  49. package/dist/camera/cameraGuidanceCopy.js +80 -0
  50. package/dist/camera/captureCountdown.d.ts +52 -0
  51. package/dist/camera/captureCountdown.js +76 -0
  52. package/dist/camera/captureWarnings.d.ts +90 -0
  53. package/dist/camera/captureWarnings.js +108 -0
  54. package/dist/camera/classifyStitchError.d.ts +30 -0
  55. package/dist/camera/classifyStitchError.js +42 -0
  56. package/dist/camera/cropGeometry.d.ts +136 -0
  57. package/dist/camera/cropGeometry.js +223 -0
  58. package/dist/camera/displayDecodeImageProps.d.ts +25 -0
  59. package/dist/camera/displayDecodeImageProps.js +29 -0
  60. package/dist/camera/guidanceGraphics.d.ts +58 -0
  61. package/dist/camera/guidanceGraphics.js +280 -0
  62. package/dist/camera/guidanceTokens.d.ts +54 -0
  63. package/dist/camera/guidanceTokens.js +58 -0
  64. package/dist/camera/panModeGate.d.ts +54 -0
  65. package/dist/camera/panModeGate.js +62 -0
  66. package/dist/camera/pickCaptureFormat.d.ts +71 -0
  67. package/dist/camera/pickCaptureFormat.js +85 -0
  68. package/dist/camera/stitchDebugInfo.d.ts +27 -0
  69. package/dist/camera/stitchDebugInfo.js +55 -0
  70. package/dist/camera/usePanMotion.d.ts +250 -0
  71. package/dist/camera/usePanMotion.js +451 -0
  72. package/dist/index.d.ts +24 -3
  73. package/dist/index.js +33 -2
  74. package/dist/stitching/computeInscribedRect.d.ts +40 -0
  75. package/dist/stitching/computeInscribedRect.js +55 -0
  76. package/dist/stitching/cropQuad.d.ts +78 -0
  77. package/dist/stitching/cropQuad.js +116 -0
  78. package/dist/stitching/incremental.d.ts +74 -0
  79. package/dist/stitching/useIncrementalStitcher.d.ts +7 -1
  80. package/dist/stitching/useIncrementalStitcher.js +7 -1
  81. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +154 -29
  82. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +4 -0
  83. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +15 -0
  84. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +2 -2
  85. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +48 -5
  86. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +27 -0
  87. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +211 -7
  88. package/ios/Sources/RNImageStitcher/RNSARSession.swift +25 -1
  89. package/ios/Sources/RNImageStitcher/Stitcher.swift +34 -1
  90. package/ios/Sources/RNImageStitcher/StitcherBridge.m +5 -0
  91. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +56 -0
  92. package/package.json +5 -1
  93. package/src/camera/Camera.tsx +945 -47
  94. package/src/camera/CameraView.tsx +48 -16
  95. package/src/camera/CaptureCountdownOverlay.tsx +272 -0
  96. package/src/camera/CaptureFrameCounterOverlay.tsx +197 -0
  97. package/src/camera/CaptureMemoryPill.tsx +50 -12
  98. package/src/camera/CapturePreview.tsx +5 -0
  99. package/src/camera/CaptureStatusOverlay.tsx +35 -7
  100. package/src/camera/CaptureThumbnailStrip.tsx +4 -0
  101. package/src/camera/LateralMotionModal.tsx +199 -0
  102. package/src/camera/PanHowToOverlay.tsx +246 -0
  103. package/src/camera/PanoramaBandOverlay.tsx +9 -1
  104. package/src/camera/PanoramaSettings.ts +27 -7
  105. package/src/camera/PanoramaSettingsModal.tsx +4 -4
  106. package/src/camera/RectCropPreview.tsx +638 -0
  107. package/src/camera/RotateToLandscapePrompt.tsx +188 -0
  108. package/src/camera/buildPanoramaInitialSettings.ts +30 -1
  109. package/src/camera/cameraErrorMessages.ts +39 -2
  110. package/src/camera/cameraGuidanceCopy.ts +145 -0
  111. package/src/camera/captureCountdown.ts +83 -0
  112. package/src/camera/captureWarnings.ts +190 -0
  113. package/src/camera/classifyStitchError.ts +68 -0
  114. package/src/camera/cropGeometry.ts +268 -0
  115. package/src/camera/displayDecodeImageProps.ts +25 -0
  116. package/src/camera/guidanceGraphics.tsx +347 -0
  117. package/src/camera/guidanceTokens.ts +57 -0
  118. package/src/camera/panModeGate.ts +81 -0
  119. package/src/camera/pickCaptureFormat.ts +130 -0
  120. package/src/camera/stitchDebugInfo.ts +71 -0
  121. package/src/camera/usePanMotion.ts +667 -0
  122. package/src/index.ts +66 -3
  123. package/src/stitching/computeInscribedRect.ts +81 -0
  124. package/src/stitching/cropQuad.ts +167 -0
  125. package/src/stitching/incremental.ts +74 -0
  126. package/src/stitching/useIncrementalStitcher.ts +13 -0
  127. package/android/src/main/java/io/imagestitcher/rn/TransferredNV21.kt +0 -100
  128. package/cpp/tests/CMakeLists.txt +0 -104
  129. package/cpp/tests/README.md +0 -86
  130. package/cpp/tests/keyframe_timebudget_test.cpp +0 -65
  131. package/cpp/tests/pose_test.cpp +0 -74
  132. package/cpp/tests/stitcher_frame_data_test.cpp +0 -132
  133. package/cpp/tests/stubs/jsi/jsi.h +0 -33
  134. package/cpp/tests/stubs/react-native-worklets-core/WKTJsiWorklet.h +0 -34
  135. package/cpp/tests/warp_guard_test.cpp +0 -48
  136. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +0 -190
  137. package/src/camera/__tests__/bandThumbRotation.test.ts +0 -120
  138. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +0 -160
  139. package/src/camera/__tests__/cameraErrorMessages.test.ts +0 -76
  140. package/src/camera/__tests__/homeIndicatorEdge.test.ts +0 -116
  141. package/src/camera/__tests__/lowMemDevice.test.ts +0 -52
  142. package/src/camera/__tests__/selectCaptureDevice.test.ts +0 -210
  143. package/src/camera/__tests__/useContentRotation.test.ts +0 -89
  144. package/src/camera/__tests__/useOrientationDrift.test.ts +0 -169
  145. package/src/stitching/__tests__/subscribeIncrementalState.refine.test.ts +0 -276
  146. package/src/stitching/__tests__/useStitcherWorklet.test.ts +0 -202
@@ -56,6 +56,10 @@ RCT_EXTERN_METHOD(markNextFrameAsLastKeyframe:(RCTPromiseResolveBlock)resolver
56
56
  RCT_EXTERN_METHOD(getMemoryFootprintMB:(RCTPromiseResolveBlock)resolver
57
57
  rejecter:(RCTPromiseRejectBlock)rejecter)
58
58
 
59
+ // 2026-06-16 — total physical RAM (MB) for the pill's RAM-aware pressure bands.
60
+ RCT_EXTERN_METHOD(getDeviceTotalRamMB:(RCTPromiseResolveBlock)resolver
61
+ rejecter:(RCTPromiseRejectBlock)rejecter)
62
+
59
63
  // 2026-05-16 — realtime+batch fusion (Option A "Replace on completion").
60
64
  // Run the shared C++ stitcher over a caller-supplied list of keyframe
61
65
  // JPEG paths and write a refined panorama to `outputPath`. See JS
@@ -217,6 +217,10 @@ public final class IncrementalStitcherBridge: RCTEventEmitter {
217
217
  // and to PANORAMA when both are 0).
218
218
  let imuT = (options["imuTranslationMetres"] as? Double) ?? 0.0
219
219
  IncrementalStitcher.shared.updateImuTranslationMetres(imuT)
220
+ // 2026-06-16 — the EXPLICIT lens the user selected ('1x'|'0.5x'): the
221
+ // reliable zoom signal for the high-level warper tree (0.5x → spherical).
222
+ let lens = (options["lens"] as? String) ?? "1x"
223
+ IncrementalStitcher.shared.updateLens(lens)
220
224
  IncrementalStitcher.shared.finalize(
221
225
  toPath: outputPath,
222
226
  jpegQuality: quality
@@ -325,6 +329,17 @@ public final class IncrementalStitcherBridge: RCTEventEmitter {
325
329
  resolver(mb)
326
330
  }
327
331
 
332
+ /// Total physical RAM in MB. Lets the DEV memory pill derive RAM-aware
333
+ /// pressure bands (iOS jetsam scales with device RAM) instead of fixed
334
+ /// thresholds. NSProcessInfo.physicalMemory is exact + cheap.
335
+ @objc(getDeviceTotalRamMB:rejecter:)
336
+ public func getDeviceTotalRamMB(
337
+ resolver: @escaping RCTPromiseResolveBlock,
338
+ rejecter: @escaping RCTPromiseRejectBlock
339
+ ) {
340
+ resolver(Double(ProcessInfo.processInfo.physicalMemory) / (1024.0 * 1024.0))
341
+ }
342
+
328
343
  /// 2026-05-16 — realtime+batch fusion (Option A) bridge. Marshal
329
344
  /// the options dictionary into the engine layer, dispatch the
330
345
  /// refinement off the bridge thread so the JS Promise doesn't block
@@ -151,9 +151,9 @@ final class KeyframeGate {
151
151
  /// overlapThreshold. Unlike `flowMaxTranslationCm` this applies to
152
152
  /// BOTH the Pose and Flow strategies, and is passed STRAIGHT
153
153
  /// THROUGH to the bridge (the unit is already what C++ expects — no
154
- /// cm→m style conversion). Default 2000 ms; 0 = disabled. The C++
154
+ /// cm→m style conversion). Default 1500 ms; 0 = disabled. The C++
155
155
  /// setter clamps to ≥ 0.
156
- var maxKeyframeIntervalMs: Double = 2000.0 {
156
+ var maxKeyframeIntervalMs: Double = 1500.0 {
157
157
  didSet {
158
158
  bridge.setMaxKeyframeIntervalMs(maxKeyframeIntervalMs)
159
159
  }
@@ -12,6 +12,16 @@
12
12
  #include <opencv2/imgcodecs.hpp>
13
13
  #pragma pop_macro("NO")
14
14
 
15
+ // v0.16 — keyframe long-edge clamp (px) applied before the JPEG is written.
16
+ // The stitcher composites at ~1 MP (COMPOSE_MP) and `compose_scale` never
17
+ // upscales, so a keyframe larger than ~1.2 MP only inflates the held-set RAM
18
+ // (N × decoded frame) without sharpening the panorama — the 0.5× ultra-wide
19
+ // otherwise lands ~8 MP/frame here. 1280 px sits just above the compose
20
+ // target, so it reclaims ~6× of that RAM with zero quality loss. (Android's
21
+ // equivalent clamp is 640 px — a tighter low-RAM budget for A35-class
22
+ // devices; iOS can afford the full compose resolution.)
23
+ static const int kKeyframeMaxLongEdge = 1280;
24
+
15
25
  // V16 Phase 1.fix2 — write a JPEG with an EXIF Orientation tag so
16
26
  // iOS image renderers display the saved frame correctly while
17
27
  // cv::imread (with IMREAD_IGNORE_ORIENTATION) gets raw landscape
@@ -119,17 +129,26 @@ static BOOL WriteJPEGWithEXIF(const cv::Mat &bgr,
119
129
 
120
130
  - (nullable instancetype)initWithError:(NSError **)error {
121
131
  if ((self = [super init])) {
122
- NSURL *appSupport = [[NSFileManager defaultManager]
123
- URLForDirectory:NSApplicationSupportDirectory
132
+ // DEBUG builds write keyframes under Documents so they are inspectable in
133
+ // the Files app (gated by the example's Info.plist UIFileSharingEnabled +
134
+ // LSSupportsOpeningDocumentsInPlace). RELEASE keeps them in the private,
135
+ // auto-cleaned ApplicationSupport dir. See `cleanup` (retains in DEBUG).
136
+ #if DEBUG
137
+ NSSearchPathDirectory baseDirType = NSDocumentDirectory;
138
+ #else
139
+ NSSearchPathDirectory baseDirType = NSApplicationSupportDirectory;
140
+ #endif
141
+ NSURL *baseDir = [[NSFileManager defaultManager]
142
+ URLForDirectory:baseDirType
124
143
  inDomain:NSUserDomainMask
125
144
  appropriateForURL:nil
126
145
  create:YES
127
146
  error:error];
128
- if (!appSupport) return nil;
147
+ if (!baseDir) return nil;
129
148
  NSString *captureUUID = [[NSUUID UUID] UUIDString];
130
149
  NSString *sessionPath =
131
- [[appSupport.path stringByAppendingPathComponent:@"Captures"]
132
- stringByAppendingPathComponent:captureUUID];
150
+ [[baseDir.path stringByAppendingPathComponent:@"Captures"]
151
+ stringByAppendingPathComponent:captureUUID];
133
152
  BOOL ok = [[NSFileManager defaultManager]
134
153
  createDirectoryAtPath:sessionPath
135
154
  withIntermediateDirectories:YES
@@ -190,6 +209,22 @@ static BOOL WriteJPEGWithEXIF(const cv::Mat &bgr,
190
209
  rotated = bgr;
191
210
  }
192
211
 
212
+ // Clamp the keyframe's long edge (see kKeyframeMaxLongEdge). Uniform
213
+ // downscale — same factor on both axes — so it preserves aspect ratio AND
214
+ // orientation (no transpose/flip); the rotate above and the EXIF tag below
215
+ // are unaffected, only the pixel count shrinks. INTER_AREA is the correct
216
+ // filter for downsampling.
217
+ {
218
+ const int longEdge =
219
+ rotated.cols > rotated.rows ? rotated.cols : rotated.rows;
220
+ if (longEdge > kKeyframeMaxLongEdge) {
221
+ const double s = (double)kKeyframeMaxLongEdge / (double)longEdge;
222
+ cv::Mat scaled;
223
+ cv::resize(rotated, scaled, cv::Size(), s, s, cv::INTER_AREA);
224
+ rotated = scaled;
225
+ }
226
+ }
227
+
193
228
  NSInteger idx = self.acceptedCount;
194
229
  NSString *filename =
195
230
  [NSString stringWithFormat:@"keyframe-%03ld.jpg", (long)idx];
@@ -230,8 +265,16 @@ static BOOL WriteJPEGWithEXIF(const cv::Mat &bgr,
230
265
 
231
266
  - (void)cleanup {
232
267
  if (self.sessionDir.length == 0) return;
268
+ #if DEBUG
269
+ // DEBUG: keep the session's keyframes on disk so they can be inspected in
270
+ // the Files app (Documents/Captures/<uuid>/keyframe-NNN.jpg). Each capture
271
+ // is a fresh UUID folder; delete old ones via Files when done.
272
+ NSLog(@"[KeyframeCollector] DEBUG — retaining keyframes for inspection: %@",
273
+ self.sessionDir);
274
+ #else
233
275
  [[NSFileManager defaultManager] removeItemAtPath:self.sessionDir
234
276
  error:nil];
277
+ #endif
235
278
  }
236
279
 
237
280
  // ── CVPixelBuffer → cv::Mat (BGR) ──────────────────────────────────
@@ -44,6 +44,10 @@ extern NSString *const RNImageStitcherErrorDomain;
44
44
  @property (nonatomic, assign, readonly) NSInteger framesRequested;
45
45
  @property (nonatomic, assign, readonly) NSInteger framesIncluded;
46
46
  @property (nonatomic, assign, readonly) double finalConfidenceThresh;
47
+ /// 2026-06-14 (DEV overlay) — semicolon-separated `key=value` trace of the
48
+ /// stitcher's runtime choices for this output (pipeline/warper/route/seam/
49
+ /// blend), surfaced on the preview in __DEV__. Empty string when unavailable.
50
+ @property (nonatomic, copy, readonly) NSString *debugSummary;
47
51
  - (instancetype)initWithOutputPath:(NSString *)outputPath
48
52
  width:(NSInteger)width
49
53
  height:(NSInteger)height
@@ -108,6 +112,10 @@ extern NSString *const RNImageStitcherErrorDomain;
108
112
  /// them. With `useInscribedRectCrop:YES` we find the largest
109
113
  /// axis-aligned rectangle entirely inside the non-zero region
110
114
  /// and crop to that — clean output with no black corners.
115
+ /// `useManualPipeline`: YES → the manual cv::detail pipeline (graphcut +
116
+ /// multiband, with the full memory-guard machinery); NO → stock high-level
117
+ /// cv::Stitcher. The batch capture passes YES (the default output); the
118
+ /// on-demand high-level tab re-stitches the same keyframes with NO.
111
119
  + (nullable RNStitchResult *)stitchFramePaths:(NSArray<NSString *> *)framePaths
112
120
  outputPath:(NSString *)outputPath
113
121
  jpegQuality:(NSInteger)quality
@@ -117,6 +125,7 @@ extern NSString *const RNImageStitcherErrorDomain;
117
125
  captureOrientation:(nullable NSString *)captureOrientation
118
126
  useInscribedRectCrop:(BOOL)useInscribedRectCrop
119
127
  stitchMode:(nullable NSString *)stitchMode
128
+ useManualPipeline:(BOOL)useManualPipeline
120
129
  error:(NSError **)error;
121
130
 
122
131
  /// Extract `maxFrames` evenly-spaced frames from the video at
@@ -194,6 +203,24 @@ extern NSString *const RNImageStitcherErrorDomain;
194
203
  quality:(NSInteger)quality
195
204
  error:(NSError **)error;
196
205
 
206
+ /// item-7 — free-quad perspective crop. Takes 4 IMAGE-PIXEL corners
207
+ /// (ordered TL, TR, BR, BL) and rectifies the enclosed quadrilateral to
208
+ /// an upright rectangle (cv::getPerspectiveTransform + warpPerspective),
209
+ /// re-encodes at `quality`, overwrites in place. Returns the rectified
210
+ /// `{ width, height }`. Rejects a degenerate / non-convex / out-of-bounds
211
+ /// quad, and guards the output canvas with the shared canvasExceedsGuard.
212
+ + (nullable NSDictionary<NSString *, NSNumber *> *)cropToQuadAtPath:(NSString *)imagePath
213
+ tlX:(double)tlX
214
+ tlY:(double)tlY
215
+ trX:(double)trX
216
+ trY:(double)trY
217
+ brX:(double)brX
218
+ brY:(double)brY
219
+ blX:(double)blX
220
+ blY:(double)blY
221
+ quality:(NSInteger)quality
222
+ error:(NSError **)error;
223
+
197
224
  /// v0.15 debug — write a red-tinted overlay JPEG (excluded / sub-threshold
198
225
  /// pixels rendered red) next to `imagePath` (suffix ".mask.jpg") so the
199
226
  /// harness can show WHY the inscribed rect lands where it does. Returns
@@ -58,6 +58,11 @@
58
58
  // The header lives in the SDK's `cpp/` dir and is on the pod's
59
59
  // HEADER_SEARCH_PATHS (see RNImageStitcher.podspec).
60
60
  #import "stitcher.hpp"
61
+ // item-7 cropToQuad: the OpenCV-free quad geometry (quadDstRect /
62
+ // isQuadAcceptable) + the shared canvas OOM guard. Same `cpp/`
63
+ // HEADER_SEARCH_PATHS as stitcher.hpp.
64
+ #import "crop_quad.hpp"
65
+ #import "warp_guard.hpp"
61
66
  #import <UIKit/UIKit.h>
62
67
  #import <AVFoundation/AVFoundation.h>
63
68
  #import <os/log.h>
@@ -262,6 +267,12 @@ NSString *const RNImageStitcherErrorDomain = @"RNImageStitcherErrorDomain";
262
267
  // RNStitchResult
263
268
  // ─────────────────────────────────────────────────────────────────────
264
269
 
270
+ // Redeclare debugSummary as readwrite internally so it can be set after the
271
+ // designated initializer (keeps the init signature unchanged).
272
+ @interface RNStitchResult ()
273
+ @property (nonatomic, copy, readwrite) NSString *debugSummary;
274
+ @end
275
+
265
276
  @implementation RNStitchResult
266
277
 
267
278
  - (instancetype)initWithOutputPath:(NSString *)outputPath
@@ -280,6 +291,7 @@ NSString *const RNImageStitcherErrorDomain = @"RNImageStitcherErrorDomain";
280
291
  _framesRequested = framesRequested;
281
292
  _framesIncluded = framesIncluded;
282
293
  _finalConfidenceThresh = finalConfidenceThresh;
294
+ _debugSummary = @"";
283
295
  }
284
296
  return self;
285
297
  }
@@ -411,6 +423,7 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
411
423
  captureOrientation:(NSString *)captureOrientation
412
424
  useInscribedRectCrop:(BOOL)useInscribedRectCrop
413
425
  stitchMode:(NSString *)stitchMode
426
+ useManualPipeline:(BOOL)useManualPipeline
414
427
  error:(NSError **)error {
415
428
  // ── Phase 2 (2026-05-16): delegated to shared C++ ───────────────────
416
429
  //
@@ -438,7 +451,7 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
438
451
  // working until they are updated. The shared C++ has its own
439
452
  // defaults but we want the wrapper to be tolerant of nil inputs
440
453
  // from Swift / Obj-C callers that grew up against the legacy API.
441
- if (warperType == nil || warperType.length == 0) warperType = @"plane";
454
+ if (warperType == nil || warperType.length == 0) warperType = @"spherical";
442
455
  if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
443
456
  if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
444
457
  if (captureOrientation == nil || captureOrientation.length == 0) captureOrientation = @"portrait";
@@ -479,12 +492,33 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
479
492
  cfg.availableRamMB =
480
493
  (double)NSProcessInfo.processInfo.physicalMemory
481
494
  / (1024.0 * 1024.0);
482
- // Route to the manual cv::detail::* pipeline; the high-level
483
- // cv::Stitcher::create path (Android's default) is unsuitable for
484
- // iOS's shelf-pan capture shape (compose-MP defaults, graphcut at
485
- // compose-MP, BA convergence params see stitcher.hpp comment
486
- // block).
487
- cfg.useManualPipeline = true;
495
+ // 2026-06-15 — DEFAULT to the MANUAL cv::detail pipeline. ALL the memory/OOM
496
+ // hardening lives on the manual path (PreStitchMemoryAbort, RAM-aware
497
+ // canvas-budget downscale, STREAM/BATCH held-set routing, the black-canvas
498
+ // utilization guard); the high-level cv::Stitcher path calls NONE of it. So
499
+ // manual is both the user's preferred output AND the memory-safe one.
500
+ //
501
+ // WARPER: NOT hardcoded — cfg.warperType carries the caller's choice (set from
502
+ // the JS `warperType`, which defaults to "spherical" and is settable via the
503
+ // ⚙️ panel / the host's `defaultWarper` prop). The JS default is the single
504
+ // source of truth now. Choosing "plane" re-arms the dynamic plane→spherical
505
+ // fallback + divergence switch in the manual pipeline (they only fire when
506
+ // warperType != "spherical").
507
+ //
508
+ // The pipeline is caller-driven: batch capture passes YES (manual, default
509
+ // output); the on-demand high-level tab re-stitches with NO.
510
+ cfg.useManualPipeline = useManualPipeline;
511
+
512
+ // 2026-06-16 — iOS resident-memory probe. iOS has no /proc/self/statm, so the
513
+ // shared rss_mb() returned -1 — which (a) blinded the per-stitch profiling and
514
+ // (b) silently DISABLED the runtime-pressure half of the manual pipeline's OOM
515
+ // router (the lowBatchHeadroom STREAM trigger), on the very platform (jetsam)
516
+ // it protects. Plug task_info(TASK_VM_INFO).phys_footprint (the metric jetsam
517
+ // evaluates) as the probe. Set UNCONDITIONALLY — the OOM guards must work in
518
+ // release too; only the sampler + per-stitch record are gated by the compile
519
+ // flag (debug-on, release-off).
520
+ cfg.memProbeFn = []() -> double { return StitcherResidentMB(); };
521
+ cfg.enableMemoryProfiling = (RNIS_MEMORY_PROFILING != 0);
488
522
 
489
523
  // Marshal NSArray<NSString*> → std::vector<std::string>. Strip the
490
524
  // `file://` scheme that some callers attach so the shared C++ can
@@ -566,6 +600,27 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
566
600
  framesRequested:framesRequested
567
601
  framesIncluded:(NSInteger)r.framesIncluded
568
602
  finalConfidenceThresh:r.finalConfidenceThresh];
603
+ if (!r.debugSummary.empty()) {
604
+ std::string dbg = r.debugSummary;
605
+ // iOS has no mallopt purge; the post-stitch settle read IS the leak
606
+ // floor (memFloor). Append it so it rides debugSummary to JS like
607
+ // Android's post-purge value (gated; debug-only).
608
+ if (RNIS_MEMORY_PROFILING != 0) {
609
+ char fbuf[40];
610
+ snprintf(fbuf, sizeof(fbuf), ";memFloor=%.1f", StitcherResidentMB());
611
+ dbg += fbuf;
612
+ }
613
+ result.debugSummary =
614
+ [NSString stringWithUTF8String:dbg.c_str()];
615
+ }
616
+ // 2026-06-15 — the eager A/B harness that ALSO stitched the high-level
617
+ // alt on EVERY capture has been REMOVED. Manual is now the default (this
618
+ // method), so computing high-level eagerly was pure wasted work —
619
+ // especially while profiling memory/perf — when the user isn't viewing
620
+ // it. The keyframe JPEGs are retained on disk so high-level can be
621
+ // produced ON DEMAND (follow-up: a `useManualPipeline` param on this
622
+ // method lets `refinePanorama` re-stitch them via the high-level path
623
+ // when the user switches to the high-level tab).
569
624
  } else {
570
625
  // Map StitchErrorCode → NSError.code. Preserves the existing
571
626
  // 9001/9002/9003/1001/9007 sentinels the JS UX layer already
@@ -863,6 +918,7 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
863
918
  captureOrientation:nil
864
919
  useInscribedRectCrop:NO
865
920
  stitchMode:nil
921
+ useManualPipeline:NO // legacy video path keeps high-level cv::Stitcher
866
922
  error:&stitchErr];
867
923
 
868
924
  // Always tear down the tmp dir, success or fail — leaving
@@ -1063,6 +1119,154 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
1063
1119
  };
1064
1120
  }
1065
1121
 
1122
+ // item-7 — free-quad perspective crop. Mirrors cropToRectAtPath, but
1123
+ // instead of an axis-aligned sub-rectangle it takes 4 user-dragged
1124
+ // corners in IMAGE-PIXEL space (ordered TL, TR, BR, BL by the JS editor's
1125
+ // orderQuadCorners) and rectifies them to an upright rectangle via
1126
+ // cv::getPerspectiveTransform + cv::warpPerspective. The destination
1127
+ // size + the convex/min-area/in-bounds gate come from the shared OpenCV-
1128
+ // free cpp/crop_quad.hpp so iOS / Android / JS agree bit-for-bit; the
1129
+ // output canvas is GUARDED with the same canvasExceedsGuard the stitch
1130
+ // pipeline uses so a near-collinear quad can't OOM a multi-MP panorama.
1131
+ + (NSDictionary<NSString *, NSNumber *> *)cropToQuadAtPath:(NSString *)imagePath
1132
+ tlX:(double)tlX
1133
+ tlY:(double)tlY
1134
+ trX:(double)trX
1135
+ trY:(double)trY
1136
+ brX:(double)brX
1137
+ brY:(double)brY
1138
+ blX:(double)blX
1139
+ blY:(double)blY
1140
+ quality:(NSInteger)quality
1141
+ error:(NSError **)error {
1142
+ NSString *cleaned = normalizeImagePath(imagePath);
1143
+ if (![[NSFileManager defaultManager] fileExistsAtPath:cleaned]) {
1144
+ if (error) {
1145
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1146
+ code:1020
1147
+ userInfo:@{
1148
+ NSLocalizedDescriptionKey:
1149
+ [NSString stringWithFormat:@"Image not found: %@", imagePath],
1150
+ }];
1151
+ }
1152
+ return nil;
1153
+ }
1154
+
1155
+ std::string nativePath(cleaned.UTF8String);
1156
+ cv::Mat img = cv::imread(nativePath, cv::IMREAD_COLOR);
1157
+ if (img.empty()) {
1158
+ if (error) {
1159
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1160
+ code:1021
1161
+ userInfo:@{
1162
+ NSLocalizedDescriptionKey:
1163
+ [NSString stringWithFormat:@"Could not decode image at %@", imagePath],
1164
+ }];
1165
+ }
1166
+ return nil;
1167
+ }
1168
+
1169
+ retailens::CropQuad quad;
1170
+ quad.tl = {tlX, tlY};
1171
+ quad.tr = {trX, trY};
1172
+ quad.br = {brX, brY};
1173
+ quad.bl = {blX, blY};
1174
+
1175
+ // Geometry gate — convex, non-degenerate, inside the decoded image.
1176
+ if (!retailens::isQuadAcceptable(quad, (double)img.cols, (double)img.rows)) {
1177
+ if (error) {
1178
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1179
+ code:1023
1180
+ userInfo:@{
1181
+ NSLocalizedDescriptionKey:
1182
+ @"Crop quad is degenerate (non-convex, zero-area, or out of bounds)",
1183
+ }];
1184
+ }
1185
+ return nil;
1186
+ }
1187
+
1188
+ const retailens::QuadDstSize dst = retailens::quadDstRect(quad);
1189
+ // Output-canvas OOM net — the same guard the stitch pipeline uses.
1190
+ if (dst.width <= 0 || dst.height <= 0 ||
1191
+ retailens::canvasExceedsGuard(dst.width, dst.height)) {
1192
+ if (error) {
1193
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1194
+ code:1024
1195
+ userInfo:@{
1196
+ NSLocalizedDescriptionKey:
1197
+ [NSString stringWithFormat:
1198
+ @"Crop quad output canvas is degenerate or exceeds the size guard (%dx%d)",
1199
+ dst.width, dst.height],
1200
+ }];
1201
+ }
1202
+ return nil;
1203
+ }
1204
+
1205
+ const cv::Point2f src[4] = {
1206
+ cv::Point2f((float)tlX, (float)tlY),
1207
+ cv::Point2f((float)trX, (float)trY),
1208
+ cv::Point2f((float)brX, (float)brY),
1209
+ cv::Point2f((float)blX, (float)blY),
1210
+ };
1211
+ const cv::Point2f dstPts[4] = {
1212
+ cv::Point2f(0.0f, 0.0f),
1213
+ cv::Point2f((float)dst.width, 0.0f),
1214
+ cv::Point2f((float)dst.width, (float)dst.height),
1215
+ cv::Point2f(0.0f, (float)dst.height),
1216
+ };
1217
+
1218
+ cv::Mat warped;
1219
+ // OpenCV throws cv::Exception (a C++ exception) — catch with a C++
1220
+ // try/catch, NOT @try/@catch (which only traps NSException).
1221
+ try {
1222
+ cv::Mat transform = cv::getPerspectiveTransform(src, dstPts);
1223
+ cv::warpPerspective(img, warped, transform,
1224
+ cv::Size(dst.width, dst.height), cv::INTER_LINEAR);
1225
+ } catch (const cv::Exception &e) {
1226
+ if (error) {
1227
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1228
+ code:1025
1229
+ userInfo:@{
1230
+ NSLocalizedDescriptionKey:
1231
+ [NSString stringWithFormat:@"Perspective warp failed: %s", e.what()],
1232
+ }];
1233
+ }
1234
+ return nil;
1235
+ }
1236
+ if (warped.empty()) {
1237
+ if (error) {
1238
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1239
+ code:1025
1240
+ userInfo:@{
1241
+ NSLocalizedDescriptionKey: @"Perspective warp produced an empty image",
1242
+ }];
1243
+ }
1244
+ return nil;
1245
+ }
1246
+
1247
+ int q = (int)quality;
1248
+ if (q < 1) { q = 1; }
1249
+ if (q > 100) { q = 100; }
1250
+ std::vector<int> writeParams = { cv::IMWRITE_JPEG_QUALITY, q };
1251
+ bool ok = cv::imwrite(nativePath, warped, writeParams);
1252
+ if (!ok) {
1253
+ if (error) {
1254
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1255
+ code:1022
1256
+ userInfo:@{
1257
+ NSLocalizedDescriptionKey:
1258
+ [NSString stringWithFormat:@"Could not rewrite image at %@", imagePath],
1259
+ }];
1260
+ }
1261
+ return nil;
1262
+ }
1263
+
1264
+ return @{
1265
+ @"width": @((NSInteger)warped.cols),
1266
+ @"height": @((NSInteger)warped.rows),
1267
+ };
1268
+ }
1269
+
1066
1270
  + (NSDictionary *)debugMaskOverlayAtPath:(NSString *)imagePath
1067
1271
  threshold:(NSInteger)threshold
1068
1272
  error:(NSError **)error {
@@ -154,6 +154,10 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
154
154
  attributes: .concurrent
155
155
  )
156
156
  private static let MAX_POSE_LOG = 600 // ~10 s @ 60Hz
157
+ /// AR keyframe long-edge budget (px) — downscale every device's frame to
158
+ /// this before encoding so stitch memory is consistent cross-device.
159
+ /// Mirrors Android's AR_KEYFRAME_MAX_LONG_EDGE.
160
+ private static let arKeyframeMaxLongEdge: CGFloat = 640
157
161
 
158
162
  /// Latest tracking state. Read by JS for UI feedback.
159
163
  @objc public private(set) var currentTrackingState: RNSARTrackingState = .notAvailable
@@ -472,6 +476,18 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
472
476
  // small text and packaging detail.
473
477
  config.isAutoFocusEnabled = true
474
478
 
479
+ // Option B — prefer the 4:3 videoFormat for full sensor FOV; the
480
+ // keyframe is downscaled to arKeyframeMaxLongEdge below so memory
481
+ // stays consistent across devices regardless of the format res.
482
+ if let fmt = ARWorldTrackingConfiguration.supportedVideoFormats.min(by: { a, b in
483
+ let da = abs(a.imageResolution.width / a.imageResolution.height - 4.0 / 3.0)
484
+ let db = abs(b.imageResolution.width / b.imageResolution.height - 4.0 / 3.0)
485
+ if abs(da - db) > 0.001 { return da < db }
486
+ return a.imageResolution.width * a.imageResolution.height
487
+ < b.imageResolution.width * b.imageResolution.height
488
+ }) {
489
+ config.videoFormat = fmt
490
+ }
475
491
  arSession.run(config, options: [.resetTracking, .removeExistingAnchors])
476
492
  // V16-diag — log the chosen video format so we can correlate
477
493
  // batch-keyframe memory with ARFrame resolution. iPhone Pro
@@ -855,8 +871,16 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
855
871
  case "portrait-upside-down": exifOrientation = .left
856
872
  default: exifOrientation = .right // portrait + unknown
857
873
  }
858
- let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
874
+ var ciImage = CIImage(cvPixelBuffer: pixelBuffer)
859
875
  .oriented(exifOrientation)
876
+ // AR keyframe downscale guard — normalise long edge to the budget so
877
+ // every device produces a ~0.3 MP keyframe (cross-device-consistent
878
+ // stitch memory). Mirrors Android's downscale in YuvImageConverter.
879
+ let kfLongEdge = max(ciImage.extent.width, ciImage.extent.height)
880
+ if kfLongEdge > Self.arKeyframeMaxLongEdge {
881
+ let kfScale = Self.arKeyframeMaxLongEdge / kfLongEdge
882
+ ciImage = ciImage.transformed(by: CGAffineTransform(scaleX: kfScale, y: kfScale))
883
+ }
860
884
  let context = CIContext(options: nil)
861
885
  guard let cgImage = context.createCGImage(
862
886
  ciImage,
@@ -161,7 +161,11 @@ public enum Stitcher {
161
161
  // expose stitchMode in its options dict yet. nil falls
162
162
  // through to Panorama in OpenCVStitcher.mm (preserves
163
163
  // historical behaviour).
164
- stitchMode: nil
164
+ stitchMode: nil,
165
+ // Generic one-shot API keeps the high-level cv::Stitcher pipeline
166
+ // (its historical behaviour); the batch capture is what defaults to
167
+ // manual. warperType "plane" above only matters on the manual path.
168
+ useManualPipeline: false
165
169
  )
166
170
  return StitchResult(
167
171
  outputPath: result.outputPath,
@@ -239,6 +243,35 @@ public enum Stitcher {
239
243
  }
240
244
  }
241
245
 
246
+ /// item-7 — perspective-rectify the quadrilateral with corners
247
+ /// `(tlX,tlY) (trX,trY) (brX,brY) (blX,blY)` (IMAGE-PIXEL space, ordered
248
+ /// TL→TR→BR→BL) out of `imagePath` into an upright rectangle, overwrite
249
+ /// in place, re-encode at `quality`. Pairs with the axis-aligned
250
+ /// `cropToRect`; the JS editor picks this when the dragged quad isn't
251
+ /// ~rectangular. Returns the rectified `{ width, height }`.
252
+ public static func cropToQuad(
253
+ imagePath: String,
254
+ tlX: Double, tlY: Double,
255
+ trX: Double, trY: Double,
256
+ brX: Double, brY: Double,
257
+ blX: Double, blY: Double,
258
+ quality: Int
259
+ ) throws -> (width: Int, height: Int) {
260
+ do {
261
+ let d = try OpenCVStitcher.cropToQuad(
262
+ atPath: imagePath,
263
+ tlX: tlX, tlY: tlY,
264
+ trX: trX, trY: trY,
265
+ brX: brX, brY: brY,
266
+ blX: blX, blY: blY,
267
+ quality: quality
268
+ )
269
+ return (width: d["width"]?.intValue ?? 0, height: d["height"]?.intValue ?? 0)
270
+ } catch let nsError as NSError {
271
+ throw StitcherError.fromNSError(nsError)
272
+ }
273
+ }
274
+
242
275
  /// v0.15 debug — write a red-tinted mask overlay (excluded pixels =
243
276
  /// red) next to the image and report what fraction the brightness mask
244
277
  /// drops. Pairs with the inscribed-rect debug harness.
@@ -34,6 +34,11 @@ RCT_EXTERN_METHOD(cropToRect:(NSDictionary *)options
34
34
  resolver:(RCTPromiseResolveBlock)resolver
35
35
  rejecter:(RCTPromiseRejectBlock)rejecter)
36
36
 
37
+ // item-7 perspective crop (free-quad rectify; pairs with cropToRect).
38
+ RCT_EXTERN_METHOD(cropToQuad:(NSDictionary *)options
39
+ resolver:(RCTPromiseResolveBlock)resolver
40
+ rejecter:(RCTPromiseRejectBlock)rejecter)
41
+
37
42
  RCT_EXTERN_METHOD(debugMaskOverlay:(NSDictionary *)options
38
43
  resolver:(RCTPromiseResolveBlock)resolver
39
44
  rejecter:(RCTPromiseRejectBlock)rejecter)
@@ -265,6 +265,62 @@ public class StitcherBridge: NSObject {
265
265
  }
266
266
  }
267
267
 
268
+ /// item-7 — perspective-rectify a user-dragged quad. `options` carries
269
+ /// `imagePath` + the 4 IMAGE-PIXEL corners as a flat `quad` array of 8
270
+ /// numbers `[tlX, tlY, trX, trY, brX, brY, blX, blY]` (ordered
271
+ /// TL→TR→BR→BL by the JS editor) + optional `quality` (default 90).
272
+ /// Resolves the rectified `{ width, height }`.
273
+ @objc(cropToQuad:resolver:rejecter:)
274
+ public func cropToQuad(
275
+ options: NSDictionary,
276
+ resolver: @escaping RCTPromiseResolveBlock,
277
+ rejecter: @escaping RCTPromiseRejectBlock
278
+ ) {
279
+ guard let imagePath = options["imagePath"] as? String else {
280
+ rejecter("invalid-options", "imagePath must be a string", nil)
281
+ return
282
+ }
283
+ guard let quad = options["quad"] as? [NSNumber], quad.count == 8 else {
284
+ rejecter(
285
+ "invalid-options",
286
+ "quad must be an array of 8 numbers [tlX,tlY,trX,trY,brX,brY,blX,blY]",
287
+ nil
288
+ )
289
+ return
290
+ }
291
+ let p = quad.map { $0.doubleValue }
292
+ let quality = (options["quality"] as? NSNumber)?.intValue ?? 90
293
+ DispatchQueue.global(qos: .userInitiated).async {
294
+ do {
295
+ let dims = try Stitcher.cropToQuad(
296
+ imagePath: imagePath,
297
+ tlX: p[0], tlY: p[1],
298
+ trX: p[2], trY: p[3],
299
+ brX: p[4], brY: p[5],
300
+ blX: p[6], blY: p[7],
301
+ quality: quality
302
+ )
303
+ resolver([
304
+ "width": dims.width,
305
+ "height": dims.height,
306
+ ])
307
+ } catch let err as StitcherError {
308
+ switch err {
309
+ case .insufficientFrames(let count):
310
+ rejecter("insufficient-frames", "(unexpected for cropToQuad) frames=\(count)", err)
311
+ case .readFailed(let path):
312
+ rejecter("read-failed", "Could not read image: \(path)", err)
313
+ case .writeFailed(let path):
314
+ rejecter("write-failed", "Could not write image: \(path)", err)
315
+ case .opencvFailed(let code, let message):
316
+ rejecter("opencv-failed-\(code)", message, err)
317
+ }
318
+ } catch {
319
+ rejecter("unknown", "Unexpected cropToQuad failure: \(error)", error)
320
+ }
321
+ }
322
+ }
323
+
268
324
  /// v0.15 debug — write a red-tinted mask overlay for `imagePath`
269
325
  /// (excluded pixels red). `threshold` optional (default 1, matching the
270
326
  /// inscribed-rect mask). Resolves