react-native-image-stitcher 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/CHANGELOG.md +96 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +21 -0
  4. package/README.md +189 -0
  5. package/RNImageStitcher.podspec +76 -0
  6. package/android/build.gradle +224 -0
  7. package/android/src/main/AndroidManifest.xml +3 -0
  8. package/android/src/main/cpp/CMakeLists.txt +124 -0
  9. package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
  10. package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
  11. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
  12. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
  13. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
  14. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
  15. package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
  16. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
  17. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
  18. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
  19. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
  20. package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
  21. package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
  22. package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
  23. package/cpp/ar_frame_pose.h +63 -0
  24. package/cpp/keyframe_gate.cpp +927 -0
  25. package/cpp/keyframe_gate.hpp +240 -0
  26. package/cpp/stitcher.cpp +2207 -0
  27. package/cpp/stitcher.hpp +275 -0
  28. package/dist/ar/useARSession.d.ts +102 -0
  29. package/dist/ar/useARSession.js +133 -0
  30. package/dist/camera/ARCameraView.d.ts +93 -0
  31. package/dist/camera/ARCameraView.js +170 -0
  32. package/dist/camera/Camera.d.ts +134 -0
  33. package/dist/camera/Camera.js +688 -0
  34. package/dist/camera/CameraShutter.d.ts +80 -0
  35. package/dist/camera/CameraShutter.js +237 -0
  36. package/dist/camera/CameraView.d.ts +65 -0
  37. package/dist/camera/CameraView.js +117 -0
  38. package/dist/camera/CaptureControlsBar.d.ts +87 -0
  39. package/dist/camera/CaptureControlsBar.js +82 -0
  40. package/dist/camera/CaptureHeader.d.ts +62 -0
  41. package/dist/camera/CaptureHeader.js +81 -0
  42. package/dist/camera/CapturePreview.d.ts +70 -0
  43. package/dist/camera/CapturePreview.js +188 -0
  44. package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
  45. package/dist/camera/CaptureStatusOverlay.js +326 -0
  46. package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
  47. package/dist/camera/CaptureThumbnailStrip.js +177 -0
  48. package/dist/camera/IncrementalPanGuide.d.ts +83 -0
  49. package/dist/camera/IncrementalPanGuide.js +267 -0
  50. package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
  51. package/dist/camera/PanoramaBandOverlay.js +399 -0
  52. package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
  53. package/dist/camera/PanoramaConfirmModal.js +128 -0
  54. package/dist/camera/PanoramaGuidance.d.ts +79 -0
  55. package/dist/camera/PanoramaGuidance.js +246 -0
  56. package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
  57. package/dist/camera/PanoramaSettingsModal.js +611 -0
  58. package/dist/camera/ViewportCropOverlay.d.ts +46 -0
  59. package/dist/camera/ViewportCropOverlay.js +67 -0
  60. package/dist/camera/useCapture.d.ts +111 -0
  61. package/dist/camera/useCapture.js +160 -0
  62. package/dist/camera/useDeviceOrientation.d.ts +48 -0
  63. package/dist/camera/useDeviceOrientation.js +131 -0
  64. package/dist/camera/useVideoCapture.d.ts +79 -0
  65. package/dist/camera/useVideoCapture.js +151 -0
  66. package/dist/index.d.ts +26 -0
  67. package/dist/index.js +39 -0
  68. package/dist/quality/normaliseOrientation.d.ts +36 -0
  69. package/dist/quality/normaliseOrientation.js +62 -0
  70. package/dist/quality/runQualityCheck.d.ts +41 -0
  71. package/dist/quality/runQualityCheck.js +98 -0
  72. package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
  73. package/dist/sensors/useIMUTranslationGate.js +235 -0
  74. package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
  75. package/dist/stitching/IncrementalStitcherView.js +157 -0
  76. package/dist/stitching/incremental.d.ts +930 -0
  77. package/dist/stitching/incremental.js +133 -0
  78. package/dist/stitching/stitchFrames.d.ts +55 -0
  79. package/dist/stitching/stitchFrames.js +56 -0
  80. package/dist/stitching/stitchVideo.d.ts +119 -0
  81. package/dist/stitching/stitchVideo.js +57 -0
  82. package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
  83. package/dist/stitching/useIncrementalJSDriver.js +199 -0
  84. package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
  85. package/dist/stitching/useIncrementalStitcher.js +172 -0
  86. package/dist/types.d.ts +58 -0
  87. package/dist/types.js +15 -0
  88. package/ios/Package.swift +72 -0
  89. package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
  90. package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
  91. package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
  92. package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
  93. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
  94. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
  95. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
  96. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
  97. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
  98. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
  99. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
  100. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
  101. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
  102. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
  104. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
  105. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
  106. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
  107. package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
  108. package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
  109. package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
  110. package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
  111. package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
  112. package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
  113. package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
  114. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
  115. package/package.json +73 -0
  116. package/react-native.config.js +34 -0
  117. package/scripts/opencv-version.txt +1 -0
  118. package/scripts/postinstall-fetch-binaries.js +286 -0
  119. package/src/ar/useARSession.ts +210 -0
  120. package/src/camera/.gitkeep +0 -0
  121. package/src/camera/ARCameraView.tsx +256 -0
  122. package/src/camera/Camera.tsx +1053 -0
  123. package/src/camera/CameraShutter.tsx +292 -0
  124. package/src/camera/CameraView.tsx +157 -0
  125. package/src/camera/CaptureControlsBar.tsx +204 -0
  126. package/src/camera/CaptureHeader.tsx +184 -0
  127. package/src/camera/CapturePreview.tsx +318 -0
  128. package/src/camera/CaptureStatusOverlay.tsx +391 -0
  129. package/src/camera/CaptureThumbnailStrip.tsx +277 -0
  130. package/src/camera/IncrementalPanGuide.tsx +328 -0
  131. package/src/camera/PanoramaBandOverlay.tsx +498 -0
  132. package/src/camera/PanoramaConfirmModal.tsx +206 -0
  133. package/src/camera/PanoramaGuidance.tsx +327 -0
  134. package/src/camera/PanoramaSettingsModal.tsx +1357 -0
  135. package/src/camera/ViewportCropOverlay.tsx +81 -0
  136. package/src/camera/useCapture.ts +279 -0
  137. package/src/camera/useDeviceOrientation.ts +140 -0
  138. package/src/camera/useVideoCapture.ts +236 -0
  139. package/src/index.ts +53 -0
  140. package/src/quality/.gitkeep +0 -0
  141. package/src/quality/normaliseOrientation.ts +79 -0
  142. package/src/quality/runQualityCheck.ts +131 -0
  143. package/src/sensors/useIMUTranslationGate.ts +347 -0
  144. package/src/stitching/.gitkeep +0 -0
  145. package/src/stitching/IncrementalStitcherView.tsx +198 -0
  146. package/src/stitching/incremental.ts +1021 -0
  147. package/src/stitching/stitchFrames.ts +88 -0
  148. package/src/stitching/stitchVideo.ts +153 -0
  149. package/src/stitching/useIncrementalJSDriver.ts +273 -0
  150. package/src/stitching/useIncrementalStitcher.ts +252 -0
  151. package/src/types.ts +78 -0
@@ -0,0 +1,1880 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ //
3
+ // OpenCVStitcher.mm
4
+ //
5
+ // Objective-C++ implementation that wraps cv::Stitcher. This is the
6
+ // only file in the SDK that includes <opencv2/...> — everything else
7
+ // sees the slim `OpenCVStitcher.h` interface above and stays in
8
+ // Swift / Objective-C.
9
+ //
10
+ // History note (V12 → V16 → 2026-05-16 shared-C++ port):
11
+ // V12-era this file used `cv::Stitcher::SCANS` mode (translational,
12
+ // plane warp, ORB). V16 fix-11 reverted that to PANORAMA after
13
+ // discovering the AffineBestOf2NearestMatcher swap broke the
14
+ // warper/blender pipeline coherence (see learning doc
15
+ // `stitcher-pipeline-coherence`). 2026-05-15 reintroduced SCANS
16
+ // as one of two modes selected per-capture via the shared
17
+ // `StitchMode` enum in `cpp/stitcher.hpp` (translation-heavy
18
+ // captures → SCANS; rotation-heavy → PANORAMA; auto-resolved by
19
+ // the KeyframeGate's accumulated motion totals).
20
+ // 2026-05-16 commit 98b1a60 swapped this method body to delegate
21
+ // to the shared C++ at `cpp/stitcher.cpp` — both modes now live
22
+ // there, this file is just the Obj-C++ marshalling shim.
23
+ //
24
+ // References:
25
+ // * OpenCV docs: https://docs.opencv.org/4.x/d2/d8d/classcv_1_1Stitcher.html
26
+ // * Mode-selection design: docs/site-content/design/2026-05-13-stitch-pipeline-mode-selection.md
27
+ // * Pipeline coherence learning: docs/site-content/learnings/2026-05-13-stitcher-pipeline-coherence.md
28
+
29
+ // OpenCV's stitching headers contain `enum { NO, ... }` and `enum { YES, ... }`
30
+ // definitions. Objective-C's `<objc/objc.h>` (transitively imported by every
31
+ // Cocoapods prefix.pch) #defines `NO` and `YES` as macros for the boolean
32
+ // constants — by the time OpenCV's enum is parsed, the preprocessor has
33
+ // already eaten those identifiers and the build dies with "expected
34
+ // identifier". Undef both BEFORE importing opencv2/*. This is the
35
+ // standard pattern used by every ObjC++ ↔ OpenCV bridge.
36
+ #ifdef NO
37
+ #undef NO
38
+ #endif
39
+ #ifdef YES
40
+ #undef YES
41
+ #endif
42
+
43
+ #import <opencv2/opencv.hpp>
44
+ #import <opencv2/stitching.hpp>
45
+ #import <opencv2/imgcodecs.hpp>
46
+ #import <chrono>
47
+ #import <vector>
48
+ #import <string>
49
+
50
+ // Now that OpenCV is parsed, restore the ObjC macros + import the
51
+ // Foundation/UIKit deps the rest of this file uses.
52
+ #define NO ((BOOL)0)
53
+ #define YES ((BOOL)1)
54
+
55
+ #import "OpenCVStitcher.h"
56
+ // Phase 2 shared-stitcher port (2026-05-16): stitchFramePaths now
57
+ // delegates to the cross-platform C++ pipeline in cpp/stitcher.cpp.
58
+ // The header lives in the SDK's `cpp/` dir and is on the pod's
59
+ // HEADER_SEARCH_PATHS (see RNImageStitcher.podspec).
60
+ #import "stitcher.hpp"
61
+ #import <UIKit/UIKit.h>
62
+ #import <AVFoundation/AVFoundation.h>
63
+ #import <os/log.h>
64
+ // V16 Phase 1b.fix3 — ImageIO for EXIF Orientation tag on output
65
+ // panorama JPEG.
66
+ #import <ImageIO/ImageIO.h>
67
+ #import <mach/mach.h>
68
+ #import <mach/task.h>
69
+ #import <mach/task_info.h>
70
+
71
+ // V12.14.2 — dedicated os_log subsystem for the stitcher. os_log
72
+ // with OS_LOG_TYPE_FAULT survives Console.app's rate-limit cap that
73
+ // drops bursts of NSLog calls — Ram's V12.14 trace had Run 2's
74
+ // extractFrames + loadFrames + step1 entirely missing, only the
75
+ // step2-5 enter cluster surviving. We use FAULT level for SENTINEL
76
+ // breadcrumbs that MUST be visible (start of stitch, BA call site,
77
+ // any catch-all for the BA crash).
78
+ static os_log_t StitcherDiagLog(void) {
79
+ static os_log_t log = NULL;
80
+ static dispatch_once_t once;
81
+ dispatch_once(&once, ^{
82
+ log = os_log_create("com.tiger.retailens.sdk", "stitch");
83
+ });
84
+ return log;
85
+ }
86
+
87
+ // V12.14.7 — resident memory probe for jetsam diagnosis. Returns
88
+ // the process' resident_size in MB. When stitch fails with cv::Exception
89
+ // AND the app subsequently dies (V12.14.6 trace pattern: throw caught
90
+ // → app quits), iOS jetsam OOM-kill is the prime suspect. Logging
91
+ // resident_size before/after each pipeline stage lets us correlate
92
+ // the kill with a memory growth pattern across successive captures.
93
+ static double StitcherResidentMB(void) {
94
+ task_vm_info_data_t info;
95
+ mach_msg_type_number_t count = TASK_VM_INFO_COUNT;
96
+ kern_return_t kr = task_info(mach_task_self(), TASK_VM_INFO,
97
+ (task_info_t)&info, &count);
98
+ if (kr != KERN_SUCCESS) return -1.0;
99
+ // phys_footprint is what jetsam evaluates against; resident_size
100
+ // is what `ps`/Xcode shows. We log both via the same helper for
101
+ // correlation — phys_footprint is the one that matters for survival.
102
+ return (double)info.phys_footprint / (1024.0 * 1024.0);
103
+ }
104
+
105
+ // V16 Phase 1b.fix3 — find the largest axis-aligned rectangle that
106
+ // fits ENTIRELY inside the non-zero region of `mask` (CV_8UC1).
107
+ // Used to crop the post-stitch panorama tightly: the regular
108
+ // boundingRect of non-zero pixels still includes the black corners
109
+ // where the projection didn't fill; the max-inscribed rectangle
110
+ // excludes those entirely.
111
+ //
112
+ // Algorithm: maximum-rectangle-in-histogram swept row by row.
113
+ // O(W * H). For a 4-6 MP panorama on iPhone 16 Pro, completes in
114
+ // 30-60 ms.
115
+ //
116
+ // Returns cv::Rect(0,0,0,0) if `mask` is empty or fully zero.
117
+ static cv::Rect MaxInscribedRectFromMask(const cv::Mat &mask) {
118
+ if (mask.empty() || mask.type() != CV_8UC1) {
119
+ return cv::Rect();
120
+ }
121
+ const int H = mask.rows;
122
+ const int W = mask.cols;
123
+
124
+ // Per-column running heights of consecutive non-zero pixels
125
+ // ending at the current row.
126
+ std::vector<int> heights((size_t)W, 0);
127
+ cv::Rect bestRect(0, 0, 0, 0);
128
+ long long bestArea = 0;
129
+
130
+ // Reusable monotonic stack for the row's largest-rectangle-in-
131
+ // histogram subroutine.
132
+ std::vector<int> stack;
133
+ stack.reserve((size_t)W + 1);
134
+
135
+ for (int row = 0; row < H; ++row) {
136
+ const uchar *m = mask.ptr<uchar>(row);
137
+ for (int col = 0; col < W; ++col) {
138
+ heights[(size_t)col] =
139
+ (m[col] != 0) ? heights[(size_t)col] + 1 : 0;
140
+ }
141
+
142
+ // Largest rectangle in the histogram for this row.
143
+ stack.clear();
144
+ for (int col = 0; col <= W; ++col) {
145
+ const int h = (col == W) ? 0 : heights[(size_t)col];
146
+ while (!stack.empty()
147
+ && heights[(size_t)stack.back()] > h) {
148
+ const int topIdx = stack.back();
149
+ stack.pop_back();
150
+ const int leftIdx =
151
+ stack.empty() ? -1 : stack.back();
152
+ const int width = col - leftIdx - 1;
153
+ const long long area =
154
+ (long long)heights[(size_t)topIdx]
155
+ * (long long)width;
156
+ if (area > bestArea) {
157
+ bestArea = area;
158
+ bestRect = cv::Rect(
159
+ leftIdx + 1,
160
+ row - heights[(size_t)topIdx] + 1,
161
+ width,
162
+ heights[(size_t)topIdx]
163
+ );
164
+ }
165
+ }
166
+ stack.push_back(col);
167
+ }
168
+ }
169
+ return bestRect;
170
+ }
171
+
172
+
173
+ // V16 Phase 1b.fix3 — write a cv::Mat (BGR) as a JPEG with an EXIF
174
+ // Orientation tag, via ImageIO. iOS image renderers (UIImage,
175
+ // RN's <Image>, Files.app, Photos) honour the tag; cv::imread with
176
+ // IMREAD_IGNORE_ORIENTATION returns raw landscape pixels. Mirrors
177
+ // the helper of the same name in OpenCVKeyframeCollector.mm — kept
178
+ // duplicated rather than refactored to a shared header per the
179
+ // codebase convention ("duplicate stage code, DRY when proven").
180
+ static BOOL WriteJPEGWithEXIFTag(const cv::Mat &bgr,
181
+ NSString *path,
182
+ NSInteger exifOrientation,
183
+ NSInteger quality) {
184
+ if (bgr.empty()) return NO;
185
+
186
+ cv::Mat rgba;
187
+ cv::cvtColor(bgr, rgba, cv::COLOR_BGR2RGBA);
188
+
189
+ CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
190
+ CGBitmapInfo bitmapInfo =
191
+ kCGBitmapByteOrderDefault | kCGImageAlphaNoneSkipLast;
192
+ CGContextRef ctx = CGBitmapContextCreate(
193
+ rgba.data,
194
+ (size_t)rgba.cols,
195
+ (size_t)rgba.rows,
196
+ 8,
197
+ (size_t)rgba.step,
198
+ colorSpace,
199
+ bitmapInfo);
200
+ if (!ctx) {
201
+ CGColorSpaceRelease(colorSpace);
202
+ return NO;
203
+ }
204
+ CGImageRef cgImage = CGBitmapContextCreateImage(ctx);
205
+ CGContextRelease(ctx);
206
+ CGColorSpaceRelease(colorSpace);
207
+ if (!cgImage) return NO;
208
+
209
+ NSURL *url = [NSURL fileURLWithPath:path];
210
+ CGImageDestinationRef dst = CGImageDestinationCreateWithURL(
211
+ (__bridge CFURLRef)url,
212
+ CFSTR("public.jpeg"),
213
+ 1,
214
+ NULL);
215
+ if (!dst) {
216
+ CGImageRelease(cgImage);
217
+ return NO;
218
+ }
219
+
220
+ NSInteger q = MAX(0, MIN(100, quality));
221
+ NSInteger exif = (exifOrientation >= 1 && exifOrientation <= 8)
222
+ ? exifOrientation : 1;
223
+ NSDictionary *props = @{
224
+ (id)kCGImageDestinationLossyCompressionQuality:
225
+ @((double)q / 100.0),
226
+ (id)kCGImagePropertyOrientation: @(exif),
227
+ };
228
+ CGImageDestinationAddImage(
229
+ dst, cgImage, (__bridge CFDictionaryRef)props);
230
+ BOOL ok = CGImageDestinationFinalize(dst);
231
+ CFRelease(dst);
232
+ CGImageRelease(cgImage);
233
+ return ok;
234
+ }
235
+
236
+
237
+ NSString *const RNImageStitcherErrorDomain = @"RNImageStitcherErrorDomain";
238
+
239
+ // ─────────────────────────────────────────────────────────────────────
240
+ // RNStitchResult
241
+ // ─────────────────────────────────────────────────────────────────────
242
+
243
+ @implementation RNStitchResult
244
+
245
+ - (instancetype)initWithOutputPath:(NSString *)outputPath
246
+ width:(NSInteger)width
247
+ height:(NSInteger)height
248
+ durationMs:(double)durationMs
249
+ framesRequested:(NSInteger)framesRequested
250
+ framesIncluded:(NSInteger)framesIncluded
251
+ finalConfidenceThresh:(double)finalConfidenceThresh {
252
+ self = [super init];
253
+ if (self) {
254
+ _outputPath = [outputPath copy];
255
+ _width = width;
256
+ _height = height;
257
+ _durationMs = durationMs;
258
+ _framesRequested = framesRequested;
259
+ _framesIncluded = framesIncluded;
260
+ _finalConfidenceThresh = finalConfidenceThresh;
261
+ }
262
+ return self;
263
+ }
264
+
265
+ - (instancetype)initWithOutputPath:(NSString *)outputPath
266
+ width:(NSInteger)width
267
+ height:(NSInteger)height
268
+ durationMs:(double)durationMs {
269
+ return [self initWithOutputPath:outputPath
270
+ width:width
271
+ height:height
272
+ durationMs:durationMs
273
+ framesRequested:-1
274
+ framesIncluded:-1
275
+ finalConfidenceThresh:-1.0];
276
+ }
277
+
278
+ @end
279
+
280
+
281
+ // ─────────────────────────────────────────────────────────────────────
282
+ // Helpers
283
+ // ─────────────────────────────────────────────────────────────────────
284
+
285
+ namespace {
286
+
287
+ // Strip the `file://` scheme some callers attach so cv::imread can
288
+ // open the path (cv::imread takes a filesystem path, not a URL).
289
+ NSString *normalizeImagePath(NSString *path) {
290
+ if ([path hasPrefix:@"file://"]) {
291
+ return [path substringFromIndex:[@"file://" length]];
292
+ }
293
+ return path;
294
+ }
295
+
296
+ // 2026-05-16 (post-Phase-2 cleanup): `loadFramesOrFail()` and
297
+ // `errorForStitchStatus()` removed from this file. Both were
298
+ // called only by the prior `stitchFramePaths:` method body that
299
+ // was replaced by the shared-C++ delegating wrapper in commit
300
+ // 98b1a60. Equivalents now live at:
301
+ // - frame loading: cpp/stitcher.cpp anonymous-namespace
302
+ // `loadAllFrames()` (called by both the high-level and manual
303
+ // entries)
304
+ // - error mapping: the explicit StitchErrorCode → NSError.code
305
+ // switch in this file at the new wrapper (lines ~528-595)
306
+ // Removed to keep the anonymous namespace tight; sibling methods
307
+ // (stitchKeyframePaths, stitchVideoAtPath) don't need them.
308
+
309
+ // Phase 5: build a cv::detail::CameraParams from an ARKit pose.
310
+ //
311
+ // ARKit's camera-to-world transform uses a right-handed system
312
+ // with +X right, +Y up, -Z forward (out of the screen). OpenCV
313
+ // uses +X right, +Y down, +Z forward (into the scene). Conversion
314
+ // is:
315
+ //
316
+ // M = diag(1, -1, -1) // axis-flip from ARKit → OpenCV
317
+ // R_ar_to_world = quaternion → 3x3 rotation matrix
318
+ // R_world_to_cv = M * R_ar_to_world.transpose()
319
+ //
320
+ // The transpose is what changes from camera-to-world (what ARKit
321
+ // gives us) to world-to-camera (what cv::detail::CameraParams.R
322
+ // expects). We don't set CameraParams.t — for panoramic stitching,
323
+ // translation is largely irrelevant (warpers project rays, not
324
+ // world points), and ARKit's metric translations would otherwise
325
+ // throw off cv::detail::SphericalWarper's scale heuristics.
326
+ //
327
+ // Intrinsics come straight from ARFrame.camera.intrinsics —
328
+ // focal length and principal point in pixels at the ARFrame's
329
+ // native resolution.
330
+ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
331
+ cv::detail::CameraParams cam;
332
+
333
+ double qx = [pose[@"qx"] doubleValue];
334
+ double qy = [pose[@"qy"] doubleValue];
335
+ double qz = [pose[@"qz"] doubleValue];
336
+ double qw = [pose[@"qw"] doubleValue];
337
+
338
+ // Quaternion → 3x3 rotation matrix (camera-to-world in ARKit).
339
+ // Standard formula; assumes the quaternion is unit-length
340
+ // (ARKit guarantees this).
341
+ cv::Mat R_ar = (cv::Mat_<double>(3, 3) <<
342
+ 1 - 2*(qy*qy + qz*qz), 2*(qx*qy - qw*qz), 2*(qx*qz + qw*qy),
343
+ 2*(qx*qy + qw*qz), 1 - 2*(qx*qx + qz*qz), 2*(qy*qz - qw*qx),
344
+ 2*(qx*qz - qw*qy), 2*(qy*qz + qw*qx), 1 - 2*(qx*qx + qy*qy)
345
+ );
346
+
347
+ // Axis-flip matrix: ARKit Y-up → OpenCV Y-down, ARKit -Z forward
348
+ // → OpenCV +Z forward.
349
+ cv::Mat M = (cv::Mat_<double>(3, 3) <<
350
+ 1, 0, 0,
351
+ 0, -1, 0,
352
+ 0, 0, -1
353
+ );
354
+
355
+ // R_world_to_cv = M * R_ar_to_world.T
356
+ cv::Mat R_world_to_cv = M * R_ar.t();
357
+ cv::Mat R_float;
358
+ R_world_to_cv.convertTo(R_float, CV_32F);
359
+ cam.R = R_float;
360
+ cam.t = cv::Mat::zeros(3, 1, CV_32F);
361
+
362
+ // Intrinsics — at the pose's native image resolution. The
363
+ // compose-rescale step below will adjust these to compose scale.
364
+ double fx = [pose[@"fx"] doubleValue];
365
+ double fy = [pose[@"fy"] doubleValue];
366
+ cam.focal = (fx + fy) / 2.0;
367
+ cam.aspect = (fx > 0.0) ? (fy / fx) : 1.0;
368
+ cam.ppx = [pose[@"cx"] doubleValue];
369
+ cam.ppy = [pose[@"cy"] doubleValue];
370
+
371
+ return cam;
372
+ }
373
+
374
+ } // namespace
375
+
376
+
377
+ // ─────────────────────────────────────────────────────────────────────
378
+ // OpenCVStitcher (public)
379
+ // ─────────────────────────────────────────────────────────────────────
380
+
381
+ @implementation OpenCVStitcher
382
+
383
+ + (nullable RNStitchResult *)stitchFramePaths:(NSArray<NSString *> *)framePaths
384
+ outputPath:(NSString *)outputPath
385
+ jpegQuality:(NSInteger)quality
386
+ warperType:(NSString *)warperType
387
+ blenderType:(NSString *)blenderType
388
+ seamFinderType:(NSString *)seamFinderType
389
+ captureOrientation:(NSString *)captureOrientation
390
+ useInscribedRectCrop:(BOOL)useInscribedRectCrop
391
+ error:(NSError **)error {
392
+ // ── Phase 2 (2026-05-16): delegated to shared C++ ───────────────────
393
+ //
394
+ // The hand-rolled cv::detail::* pipeline that used to live here
395
+ // (~1500 lines from the original implementation, covering frames-
396
+ // load → ORB features → BestOf2Nearest matching → leaveBiggest
397
+ // Component → HomographyBasedEstimator → BundleAdjusterRay → wave
398
+ // correct → median-focal warper-scale → seam find → multi-band
399
+ // blend → max-inscribed-rect crop → bake-rotate → JPEG write) was
400
+ // ported verbatim to `retailens::stitchFramePathsManual()` in
401
+ // cpp/stitcher.cpp during Phase 1 (commit 02534ac). Android already
402
+ // routes through the same file via the high-level pipeline; iOS
403
+ // now routes through it via `useManualPipeline=true`.
404
+ //
405
+ // Git blame on commit 02534ac (and its parent) captures the full
406
+ // algorithm history with the original step-by-step comments. The
407
+ // shared C++ file carries forward equivalent comments at each step.
408
+ //
409
+ // This wrapper's only job: marshal Obj-C args into the shared
410
+ // StitchConfig + std::vector<std::string>, route logs to os_log,
411
+ // map StitchErrorCode → NSError.code so the JS-side UX taxonomy
412
+ // (9001 / 9002 / … / 9007) is preserved.
413
+
414
+ // Defaults if caller passed nil — keeps the older 3-arg call-sites
415
+ // working until they are updated. The shared C++ has its own
416
+ // defaults but we want the wrapper to be tolerant of nil inputs
417
+ // from Swift / Obj-C callers that grew up against the legacy API.
418
+ if (warperType == nil || warperType.length == 0) warperType = @"plane";
419
+ if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
420
+ if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
421
+ if (captureOrientation == nil || captureOrientation.length == 0) captureOrientation = @"portrait";
422
+
423
+ // Build the shared-C++ config. Sentinel resolution budgets (-1.0)
424
+ // let the manual entry point pick its own defaults (registration
425
+ // 0.6 MP / seam 0.1 MP / compose 0.6 MP per Phase 1 fixes).
426
+ retailens::StitchConfig cfg;
427
+ cfg.warperType = warperType.UTF8String;
428
+ cfg.blenderType = blenderType.UTF8String;
429
+ cfg.seamFinderType = seamFinderType.UTF8String;
430
+ cfg.captureOrientation = captureOrientation.UTF8String;
431
+ cfg.useInscribedRectCrop = (useInscribedRectCrop != NO);
432
+ cfg.jpegQuality = (int)quality;
433
+ // The iOS API doesn't expose stitchMode yet; defaulting to Panorama
434
+ // matches the prior hand-rolled pipeline's BestOf2NearestMatcher +
435
+ // BundleAdjusterRay configuration (rotation-only end-to-end).
436
+ cfg.stitchMode = retailens::StitchMode::Panorama;
437
+ // Pre-stitch memory-abort threshold inside the manual pipeline keys
438
+ // off this value. Plumb the device's physical RAM through so the
439
+ // heuristic scales correctly across the iPhone fleet (~2 GB legacy
440
+ // → ~8 GB iPhone 16 Pro).
441
+ cfg.availableRamMB =
442
+ (double)NSProcessInfo.processInfo.physicalMemory
443
+ / (1024.0 * 1024.0);
444
+ // Route to the manual cv::detail::* pipeline; the high-level
445
+ // cv::Stitcher::create path (Android's default) is unsuitable for
446
+ // iOS's shelf-pan capture shape (compose-MP defaults, graphcut at
447
+ // compose-MP, BA convergence params — see stitcher.hpp comment
448
+ // block).
449
+ cfg.useManualPipeline = true;
450
+
451
+ // Marshal NSArray<NSString*> → std::vector<std::string>. Strip the
452
+ // `file://` scheme that some callers attach so the shared C++ can
453
+ // cv::imread the raw filesystem path.
454
+ std::vector<std::string> paths;
455
+ paths.reserve(framePaths.count);
456
+ for (NSString *p in framePaths) {
457
+ NSString *cleaned = p;
458
+ if ([cleaned hasPrefix:@"file://"]) {
459
+ cleaned = [cleaned substringFromIndex:[@"file://" length]];
460
+ }
461
+ paths.emplace_back(cleaned.UTF8String);
462
+ }
463
+ NSString *cleanedOutputPath = outputPath;
464
+ if ([cleanedOutputPath hasPrefix:@"file://"]) {
465
+ cleanedOutputPath = [cleanedOutputPath substringFromIndex:[@"file://" length]];
466
+ }
467
+
468
+ // Logging callback: route shared-C++ logs to the same os_log
469
+ // subsystem the rest of this file uses, so Console.app shows them
470
+ // alongside the existing breadcrumbs. Level mapping mirrors what
471
+ // the shared C++ already documents (0=info, 1=warn, 2=error).
472
+ retailens::LogFn logFn = [](int level, const char *tag, const char *msg) {
473
+ os_log_type_t logType;
474
+ switch (level) {
475
+ case 0: logType = OS_LOG_TYPE_INFO; break;
476
+ case 1: logType = OS_LOG_TYPE_DEFAULT; break;
477
+ case 2: logType = OS_LOG_TYPE_FAULT; break;
478
+ default: logType = OS_LOG_TYPE_DEFAULT; break;
479
+ }
480
+ os_log_with_type(StitcherDiagLog(), logType, "%{public}s %{public}s",
481
+ tag ? tag : "[stitch]", msg ? msg : "");
482
+ };
483
+
484
+ // ── Run the stitch under an @autoreleasepool ─────────────────────
485
+ //
486
+ // fix-10 pattern (see line 599-ish of the prior file revision for
487
+ // the canonical comment block — preserved by git blame on the
488
+ // pre-Phase-2 commit): any NSError/NSString created INSIDE the
489
+ // pool would otherwise be autoreleased into the pool and freed at
490
+ // the closing brace BEFORE Swift's `objc_retainAutoreleasedReturn-
491
+ // Value` could retain it, producing the EXC_BAD_ACCESS the old
492
+ // implementation chased through fix-1 through fix-9.
493
+ //
494
+ // For this wrapper the C++ call doesn't autorelease anything by
495
+ // itself, but ANY `[NSString stringWithUTF8String:]` or
496
+ // `[NSError errorWithDomain:…]` we build from the result IS
497
+ // autoreleased. So we run the C++ call + the NSError build inside
498
+ // the pool, but capture the NSError into a STRONG LOCAL declared
499
+ // ABOVE the pool. The pool drains; the strong local survives
500
+ // (ARC retain on the alloc, NOT autoreleased); after the pool we
501
+ // either return the success result, write `*error` from the strong
502
+ // local, or fall through.
503
+ //
504
+ // See docs/site-content/learnings/react-native.md#autoreleasepool-return-uaf
505
+ RNStitchResult *result = nil;
506
+ NSError *capturedError = nil;
507
+ @autoreleasepool {
508
+ retailens::StitchResult r = retailens::stitchFramePaths(
509
+ paths,
510
+ cleanedOutputPath.UTF8String,
511
+ cfg,
512
+ logFn);
513
+
514
+ if (r.success) {
515
+ const int64_t durationMs = r.durationMs;
516
+ // 2026-05-16 (Issue 5) — pass C+D retry telemetry up to Swift so
517
+ // the JS finalize dict can carry it. framesRequested defaults
518
+ // to the input count when the cpp path didn't fill it (e.g. an
519
+ // early-return success path that bypassed the retry loop).
520
+ const NSInteger framesRequested =
521
+ r.framesRequested > 0 ? (NSInteger)r.framesRequested
522
+ : (NSInteger)paths.size();
523
+ result = [[RNStitchResult alloc]
524
+ initWithOutputPath:outputPath
525
+ width:(NSInteger)r.width
526
+ height:(NSInteger)r.height
527
+ durationMs:(double)durationMs
528
+ framesRequested:framesRequested
529
+ framesIncluded:(NSInteger)r.framesIncluded
530
+ finalConfidenceThresh:r.finalConfidenceThresh];
531
+ } else {
532
+ // Map StitchErrorCode → NSError.code. Preserves the existing
533
+ // 9001/9002/9003/1001/9007 sentinels the JS UX layer already
534
+ // branches on; adds new codes 9100-9103 for manual-pipeline-
535
+ // specific failure modes that previously collapsed into
536
+ // 9007 / generic crashes.
537
+ NSInteger nsCode = 9999;
538
+ switch (r.errorCode) {
539
+ case retailens::StitchErrorCode::NeedMoreImages:
540
+ nsCode = 9001;
541
+ break;
542
+ case retailens::StitchErrorCode::HomographyEstimationFailed:
543
+ nsCode = 9002;
544
+ break;
545
+ case retailens::StitchErrorCode::CameraParamsAdjustFailed:
546
+ nsCode = 9003;
547
+ break;
548
+ case retailens::StitchErrorCode::ImageReadFailed:
549
+ nsCode = 1001;
550
+ break;
551
+ case retailens::StitchErrorCode::AllFramesDroppedByConfidence:
552
+ // 9007 preserves the existing sentinel the JS-side surfaces
553
+ // as "could not stitch — try recapturing with more overlap";
554
+ // changing this would silently flip the operator-facing
555
+ // copy across the app.
556
+ nsCode = 9007;
557
+ break;
558
+ case retailens::StitchErrorCode::PreStitchMemoryAbort:
559
+ nsCode = 9100;
560
+ break;
561
+ case retailens::StitchErrorCode::ComposeResizeFailed:
562
+ nsCode = 9101;
563
+ break;
564
+ case retailens::StitchErrorCode::WarpFailed:
565
+ nsCode = 9102;
566
+ break;
567
+ case retailens::StitchErrorCode::EmptyPanorama:
568
+ nsCode = 9103;
569
+ break;
570
+ case retailens::StitchErrorCode::InvalidArgument:
571
+ nsCode = 9000;
572
+ break;
573
+ default:
574
+ nsCode = 9999;
575
+ break;
576
+ }
577
+ NSString *msg =
578
+ [NSString stringWithUTF8String:r.errorMessage.c_str()] ?: @"Stitch failed";
579
+ capturedError = [NSError errorWithDomain:RNImageStitcherErrorDomain
580
+ code:nsCode
581
+ userInfo:@{NSLocalizedDescriptionKey: msg}];
582
+ }
583
+ } // end @autoreleasepool — drains shared-C++ temporary NSStrings
584
+ // that we built from r.errorMessage / tag strings. result and
585
+ // capturedError survive the drain because they were assigned
586
+ // to strong locals declared ABOVE the pool.
587
+
588
+ // Failure path: outparameter assignment happens AFTER the pool
589
+ // drains so the NSError lives in the OUTER pool (drained by the
590
+ // GCD work item / Swift autoreleasing boundary).
591
+ if (capturedError != nil) {
592
+ if (error) {
593
+ *error = capturedError;
594
+ }
595
+ return nil;
596
+ }
597
+ return result;
598
+ }
599
+
600
+
601
+ // ─────────────────────────────────────────────────────────────────────
602
+ // Video → frames (AVFoundation, no OpenCV)
603
+ // ─────────────────────────────────────────────────────────────────────
604
+
605
+ + (nullable NSArray<NSString *> *)extractFramesFromVideoAtPath:(NSString *)videoPath
606
+ outputDir:(NSString *)outputDir
607
+ maxFrames:(NSInteger)maxFrames
608
+ jpegQuality:(NSInteger)quality
609
+ error:(NSError **)error {
610
+ if (maxFrames < 2) {
611
+ if (error) {
612
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
613
+ code:1010
614
+ userInfo:@{
615
+ NSLocalizedDescriptionKey:
616
+ @"maxFrames must be ≥ 2 for the stitcher to have something to align.",
617
+ }];
618
+ }
619
+ return nil;
620
+ }
621
+
622
+ NSString *cleanedVideoPath = normalizeImagePath(videoPath);
623
+ NSURL *videoURL = [NSURL fileURLWithPath:cleanedVideoPath];
624
+ if (![[NSFileManager defaultManager] fileExistsAtPath:cleanedVideoPath]) {
625
+ if (error) {
626
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
627
+ code:1011
628
+ userInfo:@{
629
+ NSLocalizedDescriptionKey:
630
+ [NSString stringWithFormat:@"Video file not found: %@", videoPath],
631
+ }];
632
+ }
633
+ return nil;
634
+ }
635
+
636
+ // Make sure outputDir exists; the SDK call creates it but be
637
+ // defensive in case the host wrote a literal path that doesn't.
638
+ [[NSFileManager defaultManager] createDirectoryAtPath:normalizeImagePath(outputDir)
639
+ withIntermediateDirectories:YES
640
+ attributes:nil
641
+ error:nil];
642
+
643
+ AVURLAsset *asset = [AVURLAsset assetWithURL:videoURL];
644
+ CMTime duration = asset.duration;
645
+ Float64 totalSeconds = CMTimeGetSeconds(duration);
646
+ if (!isfinite(totalSeconds) || totalSeconds <= 0) {
647
+ if (error) {
648
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
649
+ code:1012
650
+ userInfo:@{
651
+ NSLocalizedDescriptionKey:
652
+ @"Could not read video duration — file may be corrupt or still being written.",
653
+ }];
654
+ }
655
+ return nil;
656
+ }
657
+
658
+ AVAssetImageGenerator *generator =
659
+ [AVAssetImageGenerator assetImageGeneratorWithAsset:asset];
660
+ // Honour the camera's recorded orientation — without this, all
661
+ // frames come out unrotated and stitch into a sideways panorama.
662
+ generator.appliesPreferredTrackTransform = YES;
663
+ // Tight tolerances → AVFoundation seeks to the requested timestamp
664
+ // exactly rather than the nearest keyframe. Cost: slower extract.
665
+ // Worth it; nearest-keyframe sampling can give near-duplicate frames
666
+ // when the keyframe interval lines up with our sample rate.
667
+ generator.requestedTimeToleranceBefore = kCMTimeZero;
668
+ generator.requestedTimeToleranceAfter = kCMTimeZero;
669
+
670
+ NSInteger clampedQuality = MAX(0, MIN(100, quality));
671
+ CGFloat compressionQuality = clampedQuality / 100.0;
672
+
673
+ NSMutableArray<NSString *> *paths =
674
+ [NSMutableArray arrayWithCapacity:(NSUInteger)maxFrames];
675
+ NSString *cleanedOutputDir = normalizeImagePath(outputDir);
676
+
677
+ // V12.13 — diagnostic for the landscape-only `EXC_BAD_ACCESS` crash
678
+ // Ram caught. Track per-frame extract progress + dimensions so the
679
+ // log breadcrumb pinpoints which stage and frame triggers the
680
+ // memory error if it recurs. Also log the asset's video track size
681
+ // + preferred transform up front so we know what AVFoundation is
682
+ // about to hand us before the loop runs.
683
+ AVAssetTrack *videoTrack = nil;
684
+ NSArray *videoTracks = [asset tracksWithMediaType:AVMediaTypeVideo];
685
+ if (videoTracks.count > 0) {
686
+ videoTrack = videoTracks.firstObject;
687
+ }
688
+ CGSize naturalSize = videoTrack ? videoTrack.naturalSize : CGSizeZero;
689
+ CGAffineTransform xform = videoTrack ? videoTrack.preferredTransform
690
+ : CGAffineTransformIdentity;
691
+ NSLog(@"[stitch-bc] extractFrames start: maxFrames=%ld duration=%.2fs "
692
+ @"track.naturalSize=%.0fx%.0f preferredTransform=[a=%.2f b=%.2f c=%.2f d=%.2f tx=%.2f ty=%.2f]",
693
+ (long)maxFrames, totalSeconds,
694
+ naturalSize.width, naturalSize.height,
695
+ xform.a, xform.b, xform.c, xform.d, xform.tx, xform.ty);
696
+
697
+ for (NSInteger i = 0; i < maxFrames; i++) {
698
+ // V12.13 — wrap each iteration in its own @autoreleasepool so
699
+ // UIImage / NSData / NSString temporaries get drained per-frame
700
+ // instead of accumulating to function exit. Without this, a
701
+ // 30-frame extract from a landscape video can hold ~100+ MB of
702
+ // autoreleased temporaries — combined with the video extractor's
703
+ // own caches this has historically triggered jetsam +
704
+ // EXC_BAD_ACCESS-during-tear-down (see the COMPOSE_MP comment
705
+ // around line 313 of stitchFramePaths for the same pattern).
706
+ @autoreleasepool {
707
+ // Even time spacing across [0, duration]. Dividing by
708
+ // (maxFrames - 1) gives endpoints at exactly 0 and `duration`,
709
+ // capturing the first and last useful moments.
710
+ Float64 fraction = (Float64)i / (Float64)(maxFrames - 1);
711
+ Float64 timeSeconds = fraction * totalSeconds;
712
+ CMTime cmTime = CMTimeMakeWithSeconds(timeSeconds, 600);
713
+
714
+ NSError *frameErr = nil;
715
+ CGImageRef cgImage =
716
+ [generator copyCGImageAtTime:cmTime actualTime:NULL error:&frameErr];
717
+ if (cgImage == NULL) {
718
+ NSLog(@"[stitch-bc] frame %ld/%ld: copyCGImageAtTime returned NULL "
719
+ @"(t=%.2fs, err=%@)",
720
+ (long)i, (long)maxFrames, timeSeconds,
721
+ frameErr.localizedDescription ?: @"nil");
722
+ // Skip an unreadable frame rather than aborting — sometimes
723
+ // the very-last-millisecond seek fails on short videos. The
724
+ // stitcher just gets one fewer frame.
725
+ continue;
726
+ }
727
+
728
+ size_t cgW = CGImageGetWidth(cgImage);
729
+ size_t cgH = CGImageGetHeight(cgImage);
730
+
731
+ UIImage *uiImage = [UIImage imageWithCGImage:cgImage];
732
+ CGImageRelease(cgImage);
733
+
734
+ NSData *jpegData = UIImageJPEGRepresentation(uiImage, compressionQuality);
735
+ if (jpegData == nil) {
736
+ NSLog(@"[stitch-bc] frame %ld/%ld: UIImageJPEGRepresentation returned nil",
737
+ (long)i, (long)maxFrames);
738
+ continue;
739
+ }
740
+
741
+ NSString *framePath =
742
+ [cleanedOutputDir stringByAppendingPathComponent:
743
+ [NSString stringWithFormat:@"frame_%03ld.jpg", (long)i]];
744
+ BOOL wrote = [jpegData writeToFile:framePath atomically:YES];
745
+ NSLog(@"[stitch-bc] frame %ld/%ld: cgImage=%zux%zu jpeg=%lu bytes wrote=%d",
746
+ (long)i, (long)maxFrames, cgW, cgH,
747
+ (unsigned long)jpegData.length, (int)wrote);
748
+ if (wrote) {
749
+ [paths addObject:framePath];
750
+ }
751
+ }
752
+ }
753
+ NSLog(@"[stitch-bc] extractFrames done: produced %lu frames",
754
+ (unsigned long)paths.count);
755
+
756
+ if (paths.count < 2) {
757
+ if (error) {
758
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
759
+ code:1013
760
+ userInfo:@{
761
+ NSLocalizedDescriptionKey:
762
+ [NSString stringWithFormat:
763
+ @"Extracted only %lu frames from video — need ≥ 2. "
764
+ "The video may be too short or the file unreadable.",
765
+ (unsigned long)paths.count],
766
+ }];
767
+ }
768
+ return nil;
769
+ }
770
+
771
+ return paths;
772
+ }
773
+
774
+
775
+ // ─────────────────────────────────────────────────────────────────────
776
+ // Combined pipeline: video → stitched panorama
777
+ // ─────────────────────────────────────────────────────────────────────
778
+
779
+ + (nullable RNStitchResult *)stitchVideoAtPath:(NSString *)videoPath
780
+ outputPath:(NSString *)outputPath
781
+ maxFrames:(NSInteger)maxFrames
782
+ jpegQuality:(NSInteger)quality
783
+ warperType:(NSString *)warperType
784
+ blenderType:(NSString *)blenderType
785
+ seamFinderType:(NSString *)seamFinderType
786
+ error:(NSError **)error {
787
+ // Tmp dir for extracted frames — UUID'd so concurrent stitches
788
+ // can't clobber each other's working state.
789
+ NSString *tmpDir =
790
+ [NSTemporaryDirectory() stringByAppendingPathComponent:
791
+ [NSString stringWithFormat:@"RNImageStitcherStitch-%@",
792
+ [[NSUUID UUID] UUIDString]]];
793
+
794
+ NSError *extractErr = nil;
795
+ NSArray<NSString *> *frames =
796
+ [self extractFramesFromVideoAtPath:videoPath
797
+ outputDir:tmpDir
798
+ maxFrames:maxFrames
799
+ jpegQuality:quality
800
+ error:&extractErr];
801
+ if (!frames) {
802
+ // Best-effort cleanup — the dir may not exist if extract bailed
803
+ // before creating it. Ignore the error.
804
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
805
+ if (error) *error = extractErr;
806
+ return nil;
807
+ }
808
+
809
+ NSError *stitchErr = nil;
810
+ // AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
811
+ // Legacy video-driven path: no AR-frame orientation context, so
812
+ // we pass nil for captureOrientation → the .mm side treats nil as
813
+ // "portrait" → no bake-rotation. Callers wanting rotation should
814
+ // use the keyframe-driven Swift path which carries the orientation
815
+ // from the JS accelerometer hook through IncrementalStitcher.
816
+ RNStitchResult *result =
817
+ [self stitchFramePaths:frames
818
+ outputPath:outputPath
819
+ jpegQuality:quality
820
+ warperType:warperType
821
+ blenderType:blenderType
822
+ seamFinderType:seamFinderType
823
+ captureOrientation:nil
824
+ useInscribedRectCrop:NO
825
+ error:&stitchErr];
826
+
827
+ // Always tear down the tmp dir, success or fail — leaving
828
+ // hundreds of MB of frame JPEGs in /tmp would balloon the app's
829
+ // working set across panoramas.
830
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
831
+
832
+ if (!result && error) *error = stitchErr;
833
+ return result;
834
+ }
835
+
836
+
837
+ // ─────────────────────────────────────────────────────────────────────
838
+ // Phase 5: pose-driven video → panorama (ARKit/ARCore)
839
+ // ─────────────────────────────────────────────────────────────────────
840
+ //
841
+ // Same end-to-end shape as `stitchVideoAtPath` but consumes
842
+ // pre-computed camera poses (from ARKit/ARCore via the host's
843
+ // RNSARSession) and skips the brittle features → matching
844
+ // → BundleAdjuster steps that the feature-matched path runs.
845
+ // The compose stage (warp + seam + blend + crop) is duplicated
846
+ // from `stitchFramePaths` rather than refactored — keeps the
847
+ // hard-won existing pipeline untouched while we field-test the
848
+ // pose path; both paths can be DRY'd into a shared helper once
849
+ // the new code is proven on real shelf captures.
850
+
851
+ + (nullable RNStitchResult *)stitchVideoAtPath:(NSString *)videoPath
852
+ outputPath:(NSString *)outputPath
853
+ maxFrames:(NSInteger)maxFrames
854
+ jpegQuality:(NSInteger)quality
855
+ warperType:(NSString *)warperType
856
+ blenderType:(NSString *)blenderType
857
+ seamFinderType:(NSString *)seamFinderType
858
+ poses:(NSArray<NSDictionary *> *)poses
859
+ error:(NSError **)error {
860
+ if (warperType == nil || warperType.length == 0) warperType = @"plane";
861
+ if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
862
+ if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
863
+ if (poses.count < 2) {
864
+ if (error) {
865
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
866
+ code:1030
867
+ userInfo:@{
868
+ NSLocalizedDescriptionKey:
869
+ @"Pose-driven stitch needs at least 2 poses; got fewer.",
870
+ }];
871
+ }
872
+ return nil;
873
+ }
874
+
875
+ NSString *tmpDir =
876
+ [NSTemporaryDirectory() stringByAppendingPathComponent:
877
+ [NSString stringWithFormat:@"RNImageStitcherStitchAR-%@",
878
+ [[NSUUID UUID] UUIDString]]];
879
+
880
+ // Extract evenly-spaced frames from the video (same helper the
881
+ // feature-matched path uses). Returns paths only; we'll compute
882
+ // each frame's timestamp ourselves to match against `poses`.
883
+ NSError *extractErr = nil;
884
+ NSArray<NSString *> *framePaths =
885
+ [self extractFramesFromVideoAtPath:videoPath
886
+ outputDir:tmpDir
887
+ maxFrames:maxFrames
888
+ jpegQuality:quality
889
+ error:&extractErr];
890
+ if (!framePaths) {
891
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
892
+ if (error) *error = extractErr;
893
+ return nil;
894
+ }
895
+
896
+ // Compute total video duration so frame timestamps match what
897
+ // the AR session captured. Pose timestamps are in absolute ms;
898
+ // we normalise against poses[0] so they align with the mp4
899
+ // timeline (which AVAssetWriter wrote starting at 0).
900
+ NSURL *videoURL = [NSURL fileURLWithPath:
901
+ ([videoPath hasPrefix:@"file://"]
902
+ ? [videoPath substringFromIndex:[@"file://" length]]
903
+ : videoPath)];
904
+ AVURLAsset *asset = [AVURLAsset assetWithURL:videoURL];
905
+ Float64 totalSeconds = CMTimeGetSeconds(asset.duration);
906
+ if (!isfinite(totalSeconds) || totalSeconds <= 0) {
907
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
908
+ if (error) {
909
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
910
+ code:1031
911
+ userInfo:@{
912
+ NSLocalizedDescriptionKey:
913
+ @"Could not read video duration for pose-time alignment.",
914
+ }];
915
+ }
916
+ return nil;
917
+ }
918
+ double baseMs = [poses[0][@"timestampMs"] doubleValue];
919
+
920
+ // Match each extracted frame to its closest pose by timestamp.
921
+ // Tolerance is 100 ms — at 60 Hz pose log + 30 fps frame extract,
922
+ // worst case is ~17 ms drift, plenty of headroom.
923
+ NSInteger N = (NSInteger)framePaths.count;
924
+ std::vector<cv::Mat> frames;
925
+ std::vector<cv::detail::CameraParams> cameras;
926
+ frames.reserve(N);
927
+ cameras.reserve(N);
928
+ int matched = 0, dropped = 0;
929
+ for (NSInteger i = 0; i < N; i++) {
930
+ Float64 fraction = (N == 1) ? 0.0 : ((Float64)i / (Float64)(N - 1));
931
+ Float64 frameTimeMs = fraction * totalSeconds * 1000.0;
932
+
933
+ NSDictionary *bestPose = nil;
934
+ double bestDelta = INFINITY;
935
+ for (NSDictionary *pose in poses) {
936
+ double poseMs = [pose[@"timestampMs"] doubleValue] - baseMs;
937
+ double delta = fabs(poseMs - frameTimeMs);
938
+ if (delta < bestDelta) {
939
+ bestDelta = delta;
940
+ bestPose = pose;
941
+ }
942
+ }
943
+ if (!bestPose || bestDelta > 100.0) {
944
+ dropped++;
945
+ continue;
946
+ }
947
+ // V16 Phase 1.fix3 — IMREAD_IGNORE_ORIENTATION parity with the
948
+ // batch-keyframe path. AVAssetImageGenerator writes JPEGs with
949
+ // EXIF Orientation tags; cv::imread defaults (OpenCV 4.5+) apply
950
+ // them, returning rotated pixels that don't match the pose's
951
+ // intrinsics (which describe the unrotated landscape sensor).
952
+ // Force raw landscape pixels for the stitcher.
953
+ cv::Mat img = cv::imread([framePaths[i] UTF8String],
954
+ cv::IMREAD_COLOR | cv::IMREAD_IGNORE_ORIENTATION);
955
+ if (img.empty()) {
956
+ dropped++;
957
+ continue;
958
+ }
959
+ frames.push_back(img);
960
+ cameras.push_back(cameraParamsFromPose(bestPose));
961
+ matched++;
962
+ }
963
+ NSLog(@"[BatchStitcher] pose-driven: matched=%d dropped=%d",
964
+ matched, dropped);
965
+
966
+ if (frames.size() < 2) {
967
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
968
+ if (error) {
969
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
970
+ code:1032
971
+ userInfo:@{
972
+ NSLocalizedDescriptionKey:
973
+ @"Fewer than 2 frames matched a pose within tolerance — "
974
+ "AR tracking may have been lost during the pan.",
975
+ }];
976
+ }
977
+ return nil;
978
+ }
979
+
980
+ auto t0 = std::chrono::steady_clock::now();
981
+ cv::Mat panorama;
982
+
983
+ @autoreleasepool {
984
+ try {
985
+ // Pose-driven path: cameras already populated. intrinsics are
986
+ // at the source frame's native resolution, so work_scale = 1.0.
987
+ int origCols = frames[0].cols;
988
+ int origRows = frames[0].rows;
989
+ double origMp = (double)origCols * origRows / 1e6;
990
+ constexpr double COMPOSE_MP = 1.0;
991
+ double compose_scale = (origMp > COMPOSE_MP)
992
+ ? std::sqrt(COMPOSE_MP / origMp)
993
+ : 1.0;
994
+ double compose_work_aspect = compose_scale; // work_scale == 1
995
+
996
+ // No camera-0 normalisation in the pose-driven path.
997
+ //
998
+ // I added one previously thinking it matched cv::Stitcher's BA
999
+ // convention. In fact it BROKE the natural orientation: BA
1000
+ // normalises into a frame where camera 0's "up" is the panorama
1001
+ // up; for pose-driven, the cameras already live in ARKit's
1002
+ // gravity-aligned world (Y-up = scene up regardless of phone
1003
+ // orientation), so passing R values in ARKit's world frame is
1004
+ // exactly what cv::detail::SphericalWarper wants — it unwraps
1005
+ // the sphere with world's +Y as up, giving correct orientation
1006
+ // for any phone pose + any pan direction. Normalising rotated
1007
+ // the panorama 90° (the user's left-to-right pan in portrait
1008
+ // came out with natural-up on the side).
1009
+ //
1010
+ // waveCorrect below provides the per-camera fine alignment that
1011
+ // BA would have done in the feature-matched path.
1012
+
1013
+ // Optional waveCorrect — uses HORIZ to match the feature-
1014
+ // matched path. Operators may pan in any direction; HORIZ
1015
+ // aligns each camera's "up" to the world Y axis (gravity),
1016
+ // which is what we want for both portrait+horizontal and
1017
+ // landscape+vertical pans (assuming the user keeps the phone
1018
+ // oriented to gravity, which is the typical handheld case).
1019
+ std::vector<cv::Mat> rmats;
1020
+ rmats.reserve(cameras.size());
1021
+ for (const auto &cam : cameras) rmats.push_back(cam.R.clone());
1022
+ try {
1023
+ cv::detail::waveCorrect(rmats, cv::detail::WAVE_CORRECT_HORIZ);
1024
+ for (size_t i = 0; i < cameras.size(); i++) {
1025
+ cameras[i].R = rmats[i];
1026
+ }
1027
+ } catch (const cv::Exception &e) {
1028
+ NSLog(@"[BatchStitcher] pose: wave correction skipped: %s", e.what());
1029
+ }
1030
+
1031
+ // Rescale intrinsics for compose-scale warping.
1032
+ for (auto &cam : cameras) {
1033
+ cam.focal *= compose_work_aspect;
1034
+ cam.ppx *= compose_work_aspect;
1035
+ cam.ppy *= compose_work_aspect;
1036
+ }
1037
+
1038
+ std::vector<double> focals;
1039
+ for (const auto &cam : cameras) focals.push_back(cam.focal);
1040
+ std::sort(focals.begin(), focals.end());
1041
+ float warpedScale = focals.empty() ? 1.0f
1042
+ : (float)focals[focals.size() / 2];
1043
+
1044
+ cv::Ptr<cv::WarperCreator> warperCreator;
1045
+ if ([warperType isEqualToString:@"cylindrical"]) {
1046
+ warperCreator = cv::makePtr<cv::CylindricalWarper>();
1047
+ } else if ([warperType isEqualToString:@"spherical"]) {
1048
+ warperCreator = cv::makePtr<cv::SphericalWarper>();
1049
+ } else {
1050
+ warperCreator = cv::makePtr<cv::PlaneWarper>();
1051
+ }
1052
+ cv::Ptr<cv::detail::RotationWarper> warper =
1053
+ warperCreator->create(warpedScale);
1054
+
1055
+ // Build composeFrames at COMPOSE_MP from full-res input.
1056
+ std::vector<cv::Mat> composeFrames;
1057
+ composeFrames.reserve(frames.size());
1058
+ for (const auto &f : frames) {
1059
+ cv::Mat scaled;
1060
+ if (std::abs(compose_scale - 1.0) > 1e-3) {
1061
+ cv::resize(f, scaled, cv::Size(), compose_scale, compose_scale,
1062
+ cv::INTER_AREA);
1063
+ } else {
1064
+ scaled = f.clone();
1065
+ }
1066
+ composeFrames.push_back(scaled);
1067
+ }
1068
+ for (auto &f : frames) f.release();
1069
+ frames.clear();
1070
+
1071
+ // Build the blender (same selection logic as the feature-matched
1072
+ // path). The "u != 0" UMat assertion the original feature-matched
1073
+ // builds hit was OOM-induced; with the per-frame Mat releases
1074
+ // and @autoreleasepool from that path's stabilisation, MultiBand
1075
+ // + GraphCut are safe here too.
1076
+ BOOL useSeam = [seamFinderType isEqualToString:@"graphcut"];
1077
+ cv::Ptr<cv::detail::Blender> blender;
1078
+ if ([blenderType isEqualToString:@"feather"]) {
1079
+ blender = cv::detail::Blender::createDefault(
1080
+ cv::detail::Blender::FEATHER, false);
1081
+ auto fb = blender.dynamicCast<cv::detail::FeatherBlender>();
1082
+ if (fb) fb->setSharpness(0.02f);
1083
+ } else {
1084
+ blender = cv::detail::Blender::createDefault(
1085
+ cv::detail::Blender::MULTI_BAND, false);
1086
+ auto mbb = blender.dynamicCast<cv::detail::MultiBandBlender>();
1087
+ if (mbb) mbb->setNumBands(5);
1088
+ }
1089
+
1090
+ if (useSeam) {
1091
+ const size_t M = composeFrames.size();
1092
+ std::vector<cv::Point> corners(M);
1093
+ std::vector<cv::Mat> imagesWarped(M);
1094
+ std::vector<cv::Mat> masksWarped(M);
1095
+ std::vector<cv::Size> sizes(M);
1096
+ for (size_t i = 0; i < M; i++) {
1097
+ cv::Mat K;
1098
+ cameras[i].K().convertTo(K, CV_32F);
1099
+ cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
1100
+ corners[i] = warper->warp(
1101
+ composeFrames[i], K, cameras[i].R, cv::INTER_LINEAR,
1102
+ cv::BORDER_CONSTANT, imagesWarped[i]);
1103
+ warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
1104
+ cv::BORDER_CONSTANT, masksWarped[i]);
1105
+ sizes[i] = imagesWarped[i].size();
1106
+ }
1107
+ for (auto &cf : composeFrames) cf.release();
1108
+ composeFrames.clear();
1109
+
1110
+ // Seam finder at SEAM_MP scale (same downscale-find-upscale
1111
+ // pattern as the feature-matched path).
1112
+ const double SEAM_MP = 0.1;
1113
+ double seam_scale = std::min(1.0, std::sqrt(SEAM_MP / origMp));
1114
+ double seam_compose_aspect = seam_scale / compose_scale;
1115
+ std::vector<cv::UMat> imagesWarpedF_seam(M);
1116
+ std::vector<cv::UMat> masksWarpedU_seam(M);
1117
+ std::vector<cv::Point> corners_seam(M);
1118
+ for (size_t i = 0; i < M; i++) {
1119
+ cv::Mat seamImage, seamMask;
1120
+ cv::resize(imagesWarped[i], seamImage, cv::Size(),
1121
+ seam_compose_aspect, seam_compose_aspect,
1122
+ cv::INTER_LINEAR);
1123
+ cv::resize(masksWarped[i], seamMask, cv::Size(),
1124
+ seam_compose_aspect, seam_compose_aspect,
1125
+ cv::INTER_NEAREST);
1126
+ seamImage.convertTo(imagesWarpedF_seam[i], CV_32F);
1127
+ seamMask.copyTo(masksWarpedU_seam[i]);
1128
+ corners_seam[i] = cv::Point(
1129
+ cvRound(corners[i].x * seam_compose_aspect),
1130
+ cvRound(corners[i].y * seam_compose_aspect));
1131
+ }
1132
+ cv::Ptr<cv::detail::SeamFinder> seamFinder =
1133
+ cv::makePtr<cv::detail::GraphCutSeamFinder>(
1134
+ cv::detail::GraphCutSeamFinder::COST_COLOR);
1135
+ seamFinder->find(imagesWarpedF_seam, corners_seam, masksWarpedU_seam);
1136
+ imagesWarpedF_seam.clear();
1137
+ for (size_t i = 0; i < M; i++) {
1138
+ cv::Mat seamMaskCpu, seamMaskDilated, seamMaskFull;
1139
+ masksWarpedU_seam[i].copyTo(seamMaskCpu);
1140
+ cv::dilate(seamMaskCpu, seamMaskDilated, cv::Mat());
1141
+ cv::resize(seamMaskDilated, seamMaskFull,
1142
+ masksWarped[i].size(), 0, 0, cv::INTER_LINEAR);
1143
+ cv::bitwise_and(seamMaskFull, masksWarped[i], masksWarped[i]);
1144
+ }
1145
+ masksWarpedU_seam.clear();
1146
+
1147
+ blender->prepare(corners, sizes);
1148
+ for (size_t i = 0; i < M; i++) {
1149
+ cv::Mat imgS;
1150
+ imagesWarped[i].convertTo(imgS, CV_16S);
1151
+ blender->feed(imgS, masksWarped[i], corners[i]);
1152
+ imagesWarped[i].release();
1153
+ masksWarped[i].release();
1154
+ imgS.release();
1155
+ }
1156
+ imagesWarped.clear();
1157
+ masksWarped.clear();
1158
+ } else {
1159
+ // STREAM path
1160
+ const size_t M = composeFrames.size();
1161
+ std::vector<cv::Point> corners(M);
1162
+ std::vector<cv::Size> sizes(M);
1163
+ for (size_t i = 0; i < M; i++) {
1164
+ cv::Mat K;
1165
+ cameras[i].K().convertTo(K, CV_32F);
1166
+ cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
1167
+ cv::Mat tmpMaskWarped;
1168
+ corners[i] = warper->warp(
1169
+ mask, K, cameras[i].R, cv::INTER_NEAREST,
1170
+ cv::BORDER_CONSTANT, tmpMaskWarped);
1171
+ sizes[i] = tmpMaskWarped.size();
1172
+ }
1173
+ blender->prepare(corners, sizes);
1174
+ for (size_t i = 0; i < M; i++) {
1175
+ cv::Mat K;
1176
+ cameras[i].K().convertTo(K, CV_32F);
1177
+ cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
1178
+ cv::Mat imgWarped, maskWarped;
1179
+ warper->warp(composeFrames[i], K, cameras[i].R,
1180
+ cv::INTER_LINEAR, cv::BORDER_CONSTANT, imgWarped);
1181
+ warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
1182
+ cv::BORDER_CONSTANT, maskWarped);
1183
+ cv::Mat imgS;
1184
+ imgWarped.convertTo(imgS, CV_16S);
1185
+ blender->feed(imgS, maskWarped, corners[i]);
1186
+ composeFrames[i].release();
1187
+ }
1188
+ composeFrames.clear();
1189
+ }
1190
+
1191
+ cv::Mat panoramaS, panoramaMask;
1192
+ blender->blend(panoramaS, panoramaMask);
1193
+ panoramaS.convertTo(panorama, CV_8U);
1194
+ } catch (const cv::Exception &e) {
1195
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
1196
+ if (error) {
1197
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1198
+ code:1100
1199
+ userInfo:@{
1200
+ NSLocalizedDescriptionKey:
1201
+ [NSString stringWithFormat:
1202
+ @"OpenCV exception during pose-driven stitch: %s", e.what()],
1203
+ }];
1204
+ }
1205
+ return nil;
1206
+ } catch (...) {
1207
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
1208
+ if (error) {
1209
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1210
+ code:1102
1211
+ userInfo:@{
1212
+ NSLocalizedDescriptionKey:
1213
+ @"Unknown exception during pose-driven stitch.",
1214
+ }];
1215
+ }
1216
+ return nil;
1217
+ }
1218
+ } // end @autoreleasepool
1219
+
1220
+ if (panorama.empty()) {
1221
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
1222
+ if (error) {
1223
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1224
+ code:1003
1225
+ userInfo:@{
1226
+ NSLocalizedDescriptionKey:
1227
+ @"Pose-driven stitch produced an empty panorama.",
1228
+ }];
1229
+ }
1230
+ return nil;
1231
+ }
1232
+
1233
+ // Crop to bounding box (skip the column-projection rect crop —
1234
+ // pose-driven stitches don't have the hourglass shape that
1235
+ // plane-warper feature-matched panoramas produce).
1236
+ cv::Mat finalImage = panorama;
1237
+ try {
1238
+ cv::Mat gray;
1239
+ cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
1240
+ cv::Mat mask;
1241
+ cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
1242
+ cv::Rect bbox = cv::boundingRect(mask);
1243
+ if (bbox.width > 0 && bbox.height > 0
1244
+ && bbox.width <= panorama.cols && bbox.height <= panorama.rows) {
1245
+ finalImage = panorama(bbox).clone();
1246
+ }
1247
+ } catch (...) {
1248
+ finalImage = panorama;
1249
+ }
1250
+
1251
+ auto t1 = std::chrono::steady_clock::now();
1252
+ double durationMs =
1253
+ std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
1254
+
1255
+ NSInteger clampedQuality = MAX(0, MIN(100, quality));
1256
+ std::vector<int> params = {
1257
+ cv::IMWRITE_JPEG_QUALITY, static_cast<int>(clampedQuality),
1258
+ };
1259
+ NSString *cleanedOutPath = ([outputPath hasPrefix:@"file://"]
1260
+ ? [outputPath substringFromIndex:[@"file://" length]]
1261
+ : outputPath);
1262
+ bool wrote = cv::imwrite([cleanedOutPath UTF8String], finalImage, params);
1263
+
1264
+ // Cleanup the tmp dir always.
1265
+ [[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
1266
+
1267
+ if (!wrote) {
1268
+ if (error) {
1269
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1270
+ code:1002
1271
+ userInfo:@{
1272
+ NSLocalizedDescriptionKey:
1273
+ [NSString stringWithFormat:
1274
+ @"Pose-driven stitch succeeded but could not write JPEG to %@",
1275
+ outputPath],
1276
+ }];
1277
+ }
1278
+ return nil;
1279
+ }
1280
+
1281
+ return [[RNStitchResult alloc]
1282
+ initWithOutputPath:outputPath
1283
+ width:(NSInteger)finalImage.cols
1284
+ height:(NSInteger)finalImage.rows
1285
+ durationMs:durationMs];
1286
+ }
1287
+
1288
+
1289
+ // ─────────────────────────────────────────────────────────────────────
1290
+ // V16 Phase 1: pose-driven stitch over explicit keyframe paths
1291
+ // ─────────────────────────────────────────────────────────────────────
1292
+ //
1293
+ // Same compose stage as the video-driven pose path above, minus the
1294
+ // AVAssetImageGenerator extract + timestamp-matching step. Frames
1295
+ // arrive as already-on-disk JPEGs from the AR-keyframe capture flow;
1296
+ // poses are 1:1 with frames (KeyframeGate saved both as the user
1297
+ // panned). Compose code is duplicated per the convention noted
1298
+ // above ("DRY when the new path is proven on real shelf captures").
1299
+ //
1300
+ // AUDIT NOTE (2026-05-15, sibling @autoreleasepool-return audit)
1301
+ // ──────────────────────────────────────────────────────────────
1302
+ //
1303
+ // This method (and the pose-driven `stitchVideoAtPath:withPoses:`
1304
+ // variant earlier in this file at ~line 2162) BOTH have the same
1305
+ // @autoreleasepool-return-UAF pattern that V16 fix-10 closed in
1306
+ // `stitchFramePaths:` at line 597 — autoreleased NSError* assigned
1307
+ // to the `error` outparameter from inside an @autoreleasepool, then
1308
+ // the function returns, the pool drains, the NSError dangles, the
1309
+ // caller crashes dereferencing. See:
1310
+ // docs/site-content/design/2026-05-12-finalize-crash-investigation.md
1311
+ //
1312
+ // CURRENT REACHABILITY: BOTH methods are dead code as of 2026-05-15.
1313
+ // Confirmed by grep — only referenced in dSYM debug symbols + comments,
1314
+ // never actually called from Swift/Obj-C/Kotlin source paths. V16
1315
+ // batch-keyframe uses `stitchFramePaths:` exclusively; this method
1316
+ // was the earlier per-keyframe-with-pose design that was superseded.
1317
+ //
1318
+ // IF/WHEN RE-ENABLED, apply fix-10's pattern (also in this file
1319
+ // around `stitchFramePaths:` lines 562-571 + 1519-1527):
1320
+ //
1321
+ // NSError *capturedError = nil;
1322
+ // RNStitchResult *result = nil;
1323
+ // @autoreleasepool {
1324
+ // do {
1325
+ // try { ... ; result = [[RNStitchResult alloc] init...]; break; }
1326
+ // catch (cv::Exception &e) { capturedError = [NSError ...]; break; }
1327
+ // catch (...) { capturedError = [NSError ...]; break; }
1328
+ // } while (0);
1329
+ // }
1330
+ // if (capturedError) { if (error) *error = capturedError; return nil; }
1331
+ // return result;
1332
+ //
1333
+ // Strong locals (`capturedError`, `result`) are declared OUTSIDE the
1334
+ // @autoreleasepool so their refcount survives the pool drain. Both
1335
+ // success + failure paths exit the pool via `break` rather than
1336
+ // `return nil;` so the pool drains cleanly before the function
1337
+ // returns.
1338
+ //
1339
+ // Not applied now because the methods aren't called; risk is latent
1340
+ // not active. Refactoring dead code carries its own risk (subtle
1341
+ // behaviour changes) without active testing.
1342
+
1343
+ + (nullable RNStitchResult *)stitchKeyframePaths:(NSArray<NSString *> *)framePaths
1344
+ outputPath:(NSString *)outputPath
1345
+ jpegQuality:(NSInteger)quality
1346
+ warperType:(NSString *)warperType
1347
+ blenderType:(NSString *)blenderType
1348
+ seamFinderType:(NSString *)seamFinderType
1349
+ poses:(NSArray<NSDictionary *> *)poses
1350
+ error:(NSError **)error {
1351
+ if (warperType == nil || warperType.length == 0) warperType = @"plane";
1352
+ if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
1353
+ if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
1354
+ if (framePaths.count < 2) {
1355
+ if (error) {
1356
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1357
+ code:1030
1358
+ userInfo:@{
1359
+ NSLocalizedDescriptionKey:
1360
+ @"Keyframe stitch needs at least 2 frames; got fewer.",
1361
+ }];
1362
+ }
1363
+ return nil;
1364
+ }
1365
+ if (framePaths.count != poses.count) {
1366
+ if (error) {
1367
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1368
+ code:1033
1369
+ userInfo:@{
1370
+ NSLocalizedDescriptionKey:
1371
+ [NSString stringWithFormat:
1372
+ @"Keyframe stitch requires 1:1 paths/poses; "
1373
+ "got %lu paths, %lu poses.",
1374
+ (unsigned long)framePaths.count,
1375
+ (unsigned long)poses.count],
1376
+ }];
1377
+ }
1378
+ return nil;
1379
+ }
1380
+
1381
+ // V16 Phase 1 — memory diagnostic instrumentation. Each stage
1382
+ // logs phys_footprint (the metric jetsam evaluates) so we can
1383
+ // bisect the stage that pushed us into OS-watchdog termination.
1384
+ // FAULT level so iOS doesn't drop logs under burst.
1385
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1386
+ "[V16-stitch-mem] ENTER framePaths=%d posesCount=%d phys=%.1fMB",
1387
+ (int)framePaths.count, (int)poses.count, StitcherResidentMB());
1388
+
1389
+ // Load each path → cv::Mat + cameraParams. Drop any that fail
1390
+ // to load (corrupt JPEG, missing file) — but require ≥2 to
1391
+ // succeed for a panorama to be possible.
1392
+ //
1393
+ // V16 Phase 1.fix2 — IMREAD_IGNORE_ORIENTATION: collector saves
1394
+ // JPEGs with an EXIF Orientation tag so iOS Image renderers (e.g.
1395
+ // LiveFrameStrip) display correctly. cv::imread defaults (since
1396
+ // OpenCV 4.5+) APPLY the EXIF rotation; that would re-introduce
1397
+ // the image-vs-intrinsics mismatch fix1 was meant to remove. Pass
1398
+ // IMREAD_IGNORE_ORIENTATION explicitly to get raw landscape sensor
1399
+ // pixels for the stitcher.
1400
+ std::vector<cv::Mat> frames;
1401
+ std::vector<cv::detail::CameraParams> cameras;
1402
+ frames.reserve(framePaths.count);
1403
+ cameras.reserve(framePaths.count);
1404
+ int loaded = 0, dropped = 0;
1405
+ for (NSInteger i = 0; i < (NSInteger)framePaths.count; i++) {
1406
+ NSString *path = framePaths[i];
1407
+ NSString *cleaned = ([path hasPrefix:@"file://"]
1408
+ ? [path substringFromIndex:[@"file://" length]]
1409
+ : path);
1410
+ cv::Mat img = cv::imread([cleaned UTF8String],
1411
+ cv::IMREAD_COLOR | cv::IMREAD_IGNORE_ORIENTATION);
1412
+ if (img.empty()) {
1413
+ dropped++;
1414
+ continue;
1415
+ }
1416
+ frames.push_back(img);
1417
+ cameras.push_back(cameraParamsFromPose(poses[i]));
1418
+ loaded++;
1419
+ }
1420
+ NSLog(@"[BatchStitcher] keyframe-stitch: loaded=%d dropped=%d",
1421
+ loaded, dropped);
1422
+ if (!frames.empty()) {
1423
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1424
+ "[V16-stitch-mem] AFTER imread N=%d size=%dx%d totalMB=%.1f phys=%.1fMB",
1425
+ (int)frames.size(),
1426
+ frames[0].cols, frames[0].rows,
1427
+ (double)frames.size() * frames[0].cols * frames[0].rows * 3
1428
+ / (1024.0 * 1024.0),
1429
+ StitcherResidentMB());
1430
+ }
1431
+
1432
+ if (frames.size() < 2) {
1433
+ if (error) {
1434
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1435
+ code:1032
1436
+ userInfo:@{
1437
+ NSLocalizedDescriptionKey:
1438
+ @"Fewer than 2 keyframes loaded successfully — JPEGs may "
1439
+ "have been corrupted or removed before stitch ran.",
1440
+ }];
1441
+ }
1442
+ return nil;
1443
+ }
1444
+
1445
+ auto t0 = std::chrono::steady_clock::now();
1446
+ cv::Mat panorama;
1447
+
1448
+ @autoreleasepool {
1449
+ try {
1450
+ int origCols = frames[0].cols;
1451
+ int origRows = frames[0].rows;
1452
+ double origMp = (double)origCols * origRows / 1e6;
1453
+ constexpr double COMPOSE_MP = 1.0;
1454
+ double compose_scale = (origMp > COMPOSE_MP)
1455
+ ? std::sqrt(COMPOSE_MP / origMp)
1456
+ : 1.0;
1457
+ double compose_work_aspect = compose_scale; // work_scale == 1
1458
+
1459
+ // V16 Phase 1.fix2 — auto-detect pan axis from camera rotation
1460
+ // spread. Compute the std-dev of camera "forward" vectors
1461
+ // projected onto each world axis; the axis with the smallest
1462
+ // spread is the pan-rotation axis (i.e. rotation about that
1463
+ // axis is what differs across frames most). HORIZ_PAN means
1464
+ // rotation about world Y (yaw): use WAVE_CORRECT_HORIZ.
1465
+ // VERT_PAN means rotation about world X (pitch): use WAVE_CORRECT_VERT.
1466
+ //
1467
+ // Earlier hardcoded HORIZ produced misaligned panoramas for
1468
+ // Ram's top-to-bottom landscape pan (no yaw spread; pitch
1469
+ // spread). Picking the right axis lets waveCorrect actually
1470
+ // help instead of being a no-op (or flipping the panorama).
1471
+ cv::detail::WaveCorrectKind waveKind = cv::detail::WAVE_CORRECT_HORIZ;
1472
+ if (cameras.size() >= 2) {
1473
+ // forward[i] = -3rd-column of R (camera looks along -Z in cv)
1474
+ double minF[3] = { 1e9, 1e9, 1e9};
1475
+ double maxF[3] = {-1e9,-1e9,-1e9};
1476
+ for (const auto &cam : cameras) {
1477
+ for (int axis = 0; axis < 3; axis++) {
1478
+ double v = -cam.R.at<float>(2, axis);
1479
+ if (v < minF[axis]) minF[axis] = v;
1480
+ if (v > maxF[axis]) maxF[axis] = v;
1481
+ }
1482
+ }
1483
+ double rangeX = maxF[0] - minF[0];
1484
+ double rangeY = maxF[1] - minF[1];
1485
+ // Larger Y-range of forward => more vertical (pitch) variation
1486
+ // => vertical pan => WAVE_CORRECT_VERT.
1487
+ if (rangeY > rangeX) {
1488
+ waveKind = cv::detail::WAVE_CORRECT_VERT;
1489
+ }
1490
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1491
+ "[V16-stitch-mem] waveKind=%{public}s "
1492
+ "rangeForwardX=%.3f rangeForwardY=%.3f",
1493
+ (waveKind == cv::detail::WAVE_CORRECT_VERT)
1494
+ ? "VERT (vertical pan)"
1495
+ : "HORIZ (horizontal pan)",
1496
+ rangeX, rangeY);
1497
+ }
1498
+ std::vector<cv::Mat> rmats;
1499
+ rmats.reserve(cameras.size());
1500
+ for (const auto &cam : cameras) rmats.push_back(cam.R.clone());
1501
+ try {
1502
+ cv::detail::waveCorrect(rmats, waveKind);
1503
+ for (size_t i = 0; i < cameras.size(); i++) {
1504
+ cameras[i].R = rmats[i];
1505
+ }
1506
+ } catch (const cv::Exception &e) {
1507
+ NSLog(@"[BatchStitcher] keyframe: wave correction skipped: %s",
1508
+ e.what());
1509
+ }
1510
+
1511
+ // Rescale intrinsics for compose-scale warping.
1512
+ for (auto &cam : cameras) {
1513
+ cam.focal *= compose_work_aspect;
1514
+ cam.ppx *= compose_work_aspect;
1515
+ cam.ppy *= compose_work_aspect;
1516
+ }
1517
+
1518
+ std::vector<double> focals;
1519
+ for (const auto &cam : cameras) focals.push_back(cam.focal);
1520
+ std::sort(focals.begin(), focals.end());
1521
+ float warpedScale = focals.empty() ? 1.0f
1522
+ : (float)focals[focals.size() / 2];
1523
+
1524
+ cv::Ptr<cv::WarperCreator> warperCreator;
1525
+ if ([warperType isEqualToString:@"cylindrical"]) {
1526
+ warperCreator = cv::makePtr<cv::CylindricalWarper>();
1527
+ } else if ([warperType isEqualToString:@"spherical"]) {
1528
+ warperCreator = cv::makePtr<cv::SphericalWarper>();
1529
+ } else {
1530
+ warperCreator = cv::makePtr<cv::PlaneWarper>();
1531
+ }
1532
+ cv::Ptr<cv::detail::RotationWarper> warper =
1533
+ warperCreator->create(warpedScale);
1534
+
1535
+ // Build composeFrames at COMPOSE_MP from full-res input.
1536
+ std::vector<cv::Mat> composeFrames;
1537
+ composeFrames.reserve(frames.size());
1538
+ for (const auto &f : frames) {
1539
+ cv::Mat scaled;
1540
+ if (std::abs(compose_scale - 1.0) > 1e-3) {
1541
+ cv::resize(f, scaled, cv::Size(), compose_scale, compose_scale,
1542
+ cv::INTER_AREA);
1543
+ } else {
1544
+ scaled = f.clone();
1545
+ }
1546
+ composeFrames.push_back(scaled);
1547
+ }
1548
+ for (auto &f : frames) f.release();
1549
+ frames.clear();
1550
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1551
+ "[V16-stitch-mem] AFTER composeFrames built+frames cleared "
1552
+ "compose_scale=%.3f compose_size=%dx%d phys=%.1fMB",
1553
+ compose_scale,
1554
+ composeFrames.empty() ? 0 : composeFrames[0].cols,
1555
+ composeFrames.empty() ? 0 : composeFrames[0].rows,
1556
+ StitcherResidentMB());
1557
+
1558
+ BOOL useSeam = [seamFinderType isEqualToString:@"graphcut"];
1559
+ cv::Ptr<cv::detail::Blender> blender;
1560
+ if ([blenderType isEqualToString:@"feather"]) {
1561
+ blender = cv::detail::Blender::createDefault(
1562
+ cv::detail::Blender::FEATHER, false);
1563
+ auto fb = blender.dynamicCast<cv::detail::FeatherBlender>();
1564
+ if (fb) fb->setSharpness(0.02f);
1565
+ } else {
1566
+ blender = cv::detail::Blender::createDefault(
1567
+ cv::detail::Blender::MULTI_BAND, false);
1568
+ auto mbb = blender.dynamicCast<cv::detail::MultiBandBlender>();
1569
+ if (mbb) mbb->setNumBands(5);
1570
+ }
1571
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1572
+ "[V16-stitch-mem] config blender=%{public}@ seam=%{public}@ warper=%{public}@ phys=%.1fMB",
1573
+ blenderType, seamFinderType, warperType, StitcherResidentMB());
1574
+
1575
+ if (useSeam) {
1576
+ const size_t M = composeFrames.size();
1577
+ std::vector<cv::Point> corners(M);
1578
+ std::vector<cv::Mat> imagesWarped(M);
1579
+ std::vector<cv::Mat> masksWarped(M);
1580
+ std::vector<cv::Size> sizes(M);
1581
+ for (size_t i = 0; i < M; i++) {
1582
+ cv::Mat K;
1583
+ cameras[i].K().convertTo(K, CV_32F);
1584
+ cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
1585
+ corners[i] = warper->warp(
1586
+ composeFrames[i], K, cameras[i].R, cv::INTER_LINEAR,
1587
+ cv::BORDER_CONSTANT, imagesWarped[i]);
1588
+ warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
1589
+ cv::BORDER_CONSTANT, masksWarped[i]);
1590
+ sizes[i] = imagesWarped[i].size();
1591
+ }
1592
+ // Compute panorama bbox so we can see if the warped span is
1593
+ // unexpectedly large (drives MultiBand pyramid memory).
1594
+ int minX = INT_MAX, minY = INT_MAX, maxX = INT_MIN, maxY = INT_MIN;
1595
+ for (size_t i = 0; i < M; i++) {
1596
+ minX = std::min(minX, corners[i].x);
1597
+ minY = std::min(minY, corners[i].y);
1598
+ maxX = std::max(maxX, corners[i].x + (int)sizes[i].width);
1599
+ maxY = std::max(maxY, corners[i].y + (int)sizes[i].height);
1600
+ }
1601
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1602
+ "[V16-stitch-mem] AFTER warps M=%d bbox=%dx%d "
1603
+ "warpedTotalMB=%.1f phys=%.1fMB",
1604
+ (int)M,
1605
+ (maxX > minX ? maxX - minX : 0),
1606
+ (maxY > minY ? maxY - minY : 0),
1607
+ (double)M * (M ? sizes[0].width : 0)
1608
+ * (M ? sizes[0].height : 0) * 3 / (1024.0 * 1024.0),
1609
+ StitcherResidentMB());
1610
+ const int panBboxW = (maxX > minX ? maxX - minX : 0);
1611
+ const int panBboxH = (maxY > minY ? maxY - minY : 0);
1612
+ // Quiet `unused variable` warnings if the inner os_log calls
1613
+ // are stripped by the compiler in release builds.
1614
+ (void)panBboxW; (void)panBboxH;
1615
+ for (auto &cf : composeFrames) cf.release();
1616
+ composeFrames.clear();
1617
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1618
+ "[V16-stitch-mem] AFTER composeFrames cleared (warps held) phys=%.1fMB",
1619
+ StitcherResidentMB());
1620
+
1621
+ const double SEAM_MP = 0.1;
1622
+ double seam_scale = std::min(1.0, std::sqrt(SEAM_MP / origMp));
1623
+ double seam_compose_aspect = seam_scale / compose_scale;
1624
+ std::vector<cv::UMat> imagesWarpedF_seam(M);
1625
+ std::vector<cv::UMat> masksWarpedU_seam(M);
1626
+ std::vector<cv::Point> corners_seam(M);
1627
+ for (size_t i = 0; i < M; i++) {
1628
+ cv::Mat seamImage, seamMask;
1629
+ cv::resize(imagesWarped[i], seamImage, cv::Size(),
1630
+ seam_compose_aspect, seam_compose_aspect,
1631
+ cv::INTER_LINEAR);
1632
+ cv::resize(masksWarped[i], seamMask, cv::Size(),
1633
+ seam_compose_aspect, seam_compose_aspect,
1634
+ cv::INTER_NEAREST);
1635
+ seamImage.convertTo(imagesWarpedF_seam[i], CV_32F);
1636
+ seamMask.copyTo(masksWarpedU_seam[i]);
1637
+ corners_seam[i] = cv::Point(
1638
+ cvRound(corners[i].x * seam_compose_aspect),
1639
+ cvRound(corners[i].y * seam_compose_aspect));
1640
+ }
1641
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1642
+ "[V16-stitch-mem] BEFORE GraphCutSeamFinder seam_scale=%.3f phys=%.1fMB",
1643
+ seam_scale, StitcherResidentMB());
1644
+ cv::Ptr<cv::detail::SeamFinder> seamFinder =
1645
+ cv::makePtr<cv::detail::GraphCutSeamFinder>(
1646
+ cv::detail::GraphCutSeamFinder::COST_COLOR);
1647
+ seamFinder->find(imagesWarpedF_seam, corners_seam, masksWarpedU_seam);
1648
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1649
+ "[V16-stitch-mem] AFTER GraphCutSeamFinder phys=%.1fMB",
1650
+ StitcherResidentMB());
1651
+ imagesWarpedF_seam.clear();
1652
+ for (size_t i = 0; i < M; i++) {
1653
+ cv::Mat seamMaskCpu, seamMaskDilated, seamMaskFull;
1654
+ masksWarpedU_seam[i].copyTo(seamMaskCpu);
1655
+ cv::dilate(seamMaskCpu, seamMaskDilated, cv::Mat());
1656
+ cv::resize(seamMaskDilated, seamMaskFull,
1657
+ masksWarped[i].size(), 0, 0, cv::INTER_LINEAR);
1658
+ cv::bitwise_and(seamMaskFull, masksWarped[i], masksWarped[i]);
1659
+ }
1660
+ masksWarpedU_seam.clear();
1661
+
1662
+ blender->prepare(corners, sizes);
1663
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1664
+ "[V16-stitch-mem] AFTER blender->prepare() phys=%.1fMB",
1665
+ StitcherResidentMB());
1666
+ for (size_t i = 0; i < M; i++) {
1667
+ cv::Mat imgS;
1668
+ imagesWarped[i].convertTo(imgS, CV_16S);
1669
+ blender->feed(imgS, masksWarped[i], corners[i]);
1670
+ imagesWarped[i].release();
1671
+ masksWarped[i].release();
1672
+ imgS.release();
1673
+ }
1674
+ imagesWarped.clear();
1675
+ masksWarped.clear();
1676
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1677
+ "[V16-stitch-mem] AFTER blender->feed() loop (graphcut) phys=%.1fMB",
1678
+ StitcherResidentMB());
1679
+ } else {
1680
+ // STREAM path
1681
+ const size_t M = composeFrames.size();
1682
+ std::vector<cv::Point> corners(M);
1683
+ std::vector<cv::Size> sizes(M);
1684
+ for (size_t i = 0; i < M; i++) {
1685
+ cv::Mat K;
1686
+ cameras[i].K().convertTo(K, CV_32F);
1687
+ cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
1688
+ cv::Mat tmpMaskWarped;
1689
+ corners[i] = warper->warp(
1690
+ mask, K, cameras[i].R, cv::INTER_NEAREST,
1691
+ cv::BORDER_CONSTANT, tmpMaskWarped);
1692
+ sizes[i] = tmpMaskWarped.size();
1693
+ }
1694
+ blender->prepare(corners, sizes);
1695
+ for (size_t i = 0; i < M; i++) {
1696
+ cv::Mat K;
1697
+ cameras[i].K().convertTo(K, CV_32F);
1698
+ cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
1699
+ cv::Mat imgWarped, maskWarped;
1700
+ warper->warp(composeFrames[i], K, cameras[i].R,
1701
+ cv::INTER_LINEAR, cv::BORDER_CONSTANT, imgWarped);
1702
+ warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
1703
+ cv::BORDER_CONSTANT, maskWarped);
1704
+ cv::Mat imgS;
1705
+ imgWarped.convertTo(imgS, CV_16S);
1706
+ blender->feed(imgS, maskWarped, corners[i]);
1707
+ composeFrames[i].release();
1708
+ }
1709
+ composeFrames.clear();
1710
+ }
1711
+
1712
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1713
+ "[V16-stitch-mem] BEFORE blender->blend() phys=%.1fMB",
1714
+ StitcherResidentMB());
1715
+ cv::Mat panoramaS, panoramaMask;
1716
+ blender->blend(panoramaS, panoramaMask);
1717
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1718
+ "[V16-stitch-mem] AFTER blender->blend() panorama=%dx%d phys=%.1fMB",
1719
+ panoramaS.cols, panoramaS.rows, StitcherResidentMB());
1720
+ panoramaS.convertTo(panorama, CV_8U);
1721
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1722
+ "[V16-stitch-mem] AFTER 16S->8U convert phys=%.1fMB",
1723
+ StitcherResidentMB());
1724
+ } catch (const cv::Exception &e) {
1725
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1726
+ "[V16-stitch-mem] cv::Exception: %{public}s phys=%.1fMB",
1727
+ e.what(), StitcherResidentMB());
1728
+ if (error) {
1729
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1730
+ code:1100
1731
+ userInfo:@{
1732
+ NSLocalizedDescriptionKey:
1733
+ [NSString stringWithFormat:
1734
+ @"OpenCV exception during keyframe stitch: %s", e.what()],
1735
+ }];
1736
+ }
1737
+ return nil;
1738
+ } catch (...) {
1739
+ if (error) {
1740
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1741
+ code:1102
1742
+ userInfo:@{
1743
+ NSLocalizedDescriptionKey:
1744
+ @"Unknown exception during keyframe stitch.",
1745
+ }];
1746
+ }
1747
+ return nil;
1748
+ }
1749
+ } // end @autoreleasepool
1750
+
1751
+ if (panorama.empty()) {
1752
+ if (error) {
1753
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1754
+ code:1003
1755
+ userInfo:@{
1756
+ NSLocalizedDescriptionKey:
1757
+ @"Keyframe stitch produced an empty panorama.",
1758
+ }];
1759
+ }
1760
+ return nil;
1761
+ }
1762
+
1763
+ // Crop to bounding box.
1764
+ cv::Mat finalImage = panorama;
1765
+ try {
1766
+ cv::Mat gray;
1767
+ cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
1768
+ cv::Mat mask;
1769
+ cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
1770
+ cv::Rect bbox = cv::boundingRect(mask);
1771
+ if (bbox.width > 0 && bbox.height > 0
1772
+ && bbox.width <= panorama.cols && bbox.height <= panorama.rows) {
1773
+ finalImage = panorama(bbox).clone();
1774
+ }
1775
+ } catch (...) {
1776
+ finalImage = panorama;
1777
+ }
1778
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1779
+ "[V16-stitch-mem] AFTER crop final=%dx%d phys=%.1fMB",
1780
+ finalImage.cols, finalImage.rows, StitcherResidentMB());
1781
+
1782
+ auto t1 = std::chrono::steady_clock::now();
1783
+ double durationMs =
1784
+ std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
1785
+
1786
+ NSInteger clampedQuality = MAX(0, MIN(100, quality));
1787
+ std::vector<int> params = {
1788
+ cv::IMWRITE_JPEG_QUALITY, static_cast<int>(clampedQuality),
1789
+ };
1790
+ NSString *cleanedOutPath = ([outputPath hasPrefix:@"file://"]
1791
+ ? [outputPath substringFromIndex:[@"file://" length]]
1792
+ : outputPath);
1793
+ bool wrote = cv::imwrite([cleanedOutPath UTF8String], finalImage, params);
1794
+ os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
1795
+ "[V16-stitch-mem] AFTER cv::imwrite ok=%d total=%.0fms phys=%.1fMB",
1796
+ wrote ? 1 : 0, durationMs, StitcherResidentMB());
1797
+
1798
+ if (!wrote) {
1799
+ if (error) {
1800
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1801
+ code:1002
1802
+ userInfo:@{
1803
+ NSLocalizedDescriptionKey:
1804
+ [NSString stringWithFormat:
1805
+ @"Keyframe stitch succeeded but could not write JPEG to %@",
1806
+ outputPath],
1807
+ }];
1808
+ }
1809
+ return nil;
1810
+ }
1811
+
1812
+ return [[RNStitchResult alloc]
1813
+ initWithOutputPath:outputPath
1814
+ width:(NSInteger)finalImage.cols
1815
+ height:(NSInteger)finalImage.rows
1816
+ durationMs:durationMs];
1817
+ }
1818
+
1819
+
1820
+ // ─────────────────────────────────────────────────────────────────────
1821
+ // Photo orientation normalisation
1822
+ // ─────────────────────────────────────────────────────────────────────
1823
+ // Round-trip through cv::imread / cv::imwrite to bake the EXIF
1824
+ // rotation into the pixel buffer, then write a plain JPEG with no
1825
+ // orientation metadata. Cheap (~ms for a typical iPhone JPEG) and
1826
+ // idempotent on already-normalised files.
1827
+
1828
+ + (NSDictionary<NSString *, NSNumber *> *)normaliseImageAtPath:(NSString *)imagePath
1829
+ error:(NSError **)error {
1830
+ NSString *cleaned = normalizeImagePath(imagePath);
1831
+ if (![[NSFileManager defaultManager] fileExistsAtPath:cleaned]) {
1832
+ if (error) {
1833
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1834
+ code:1020
1835
+ userInfo:@{
1836
+ NSLocalizedDescriptionKey:
1837
+ [NSString stringWithFormat:@"Image not found: %@", imagePath],
1838
+ }];
1839
+ }
1840
+ return nil;
1841
+ }
1842
+
1843
+ std::string nativePath(cleaned.UTF8String);
1844
+ cv::Mat img = cv::imread(nativePath, cv::IMREAD_COLOR);
1845
+ if (img.empty()) {
1846
+ if (error) {
1847
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1848
+ code:1021
1849
+ userInfo:@{
1850
+ NSLocalizedDescriptionKey:
1851
+ [NSString stringWithFormat:@"Could not decode image at %@", imagePath],
1852
+ }];
1853
+ }
1854
+ return nil;
1855
+ }
1856
+
1857
+ std::vector<int> writeParams = {
1858
+ cv::IMWRITE_JPEG_QUALITY, 92,
1859
+ };
1860
+ bool ok = cv::imwrite(nativePath, img, writeParams);
1861
+ if (!ok) {
1862
+ if (error) {
1863
+ *error = [NSError errorWithDomain:RNImageStitcherErrorDomain
1864
+ code:1022
1865
+ userInfo:@{
1866
+ NSLocalizedDescriptionKey:
1867
+ [NSString stringWithFormat:
1868
+ @"Could not rewrite image at %@", imagePath],
1869
+ }];
1870
+ }
1871
+ return nil;
1872
+ }
1873
+
1874
+ return @{
1875
+ @"width": @((NSInteger)img.cols),
1876
+ @"height": @((NSInteger)img.rows),
1877
+ };
1878
+ }
1879
+
1880
+ @end