react-native-image-stitcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/LICENSE +201 -0
- package/NOTICE +21 -0
- package/README.md +189 -0
- package/RNImageStitcher.podspec +76 -0
- package/android/build.gradle +224 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/cpp/CMakeLists.txt +124 -0
- package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
- package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
- package/cpp/ar_frame_pose.h +63 -0
- package/cpp/keyframe_gate.cpp +927 -0
- package/cpp/keyframe_gate.hpp +240 -0
- package/cpp/stitcher.cpp +2207 -0
- package/cpp/stitcher.hpp +275 -0
- package/dist/ar/useARSession.d.ts +102 -0
- package/dist/ar/useARSession.js +133 -0
- package/dist/camera/ARCameraView.d.ts +93 -0
- package/dist/camera/ARCameraView.js +170 -0
- package/dist/camera/Camera.d.ts +134 -0
- package/dist/camera/Camera.js +688 -0
- package/dist/camera/CameraShutter.d.ts +80 -0
- package/dist/camera/CameraShutter.js +237 -0
- package/dist/camera/CameraView.d.ts +65 -0
- package/dist/camera/CameraView.js +117 -0
- package/dist/camera/CaptureControlsBar.d.ts +87 -0
- package/dist/camera/CaptureControlsBar.js +82 -0
- package/dist/camera/CaptureHeader.d.ts +62 -0
- package/dist/camera/CaptureHeader.js +81 -0
- package/dist/camera/CapturePreview.d.ts +70 -0
- package/dist/camera/CapturePreview.js +188 -0
- package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
- package/dist/camera/CaptureStatusOverlay.js +326 -0
- package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
- package/dist/camera/CaptureThumbnailStrip.js +177 -0
- package/dist/camera/IncrementalPanGuide.d.ts +83 -0
- package/dist/camera/IncrementalPanGuide.js +267 -0
- package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
- package/dist/camera/PanoramaBandOverlay.js +399 -0
- package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
- package/dist/camera/PanoramaConfirmModal.js +128 -0
- package/dist/camera/PanoramaGuidance.d.ts +79 -0
- package/dist/camera/PanoramaGuidance.js +246 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
- package/dist/camera/PanoramaSettingsModal.js +611 -0
- package/dist/camera/ViewportCropOverlay.d.ts +46 -0
- package/dist/camera/ViewportCropOverlay.js +67 -0
- package/dist/camera/useCapture.d.ts +111 -0
- package/dist/camera/useCapture.js +160 -0
- package/dist/camera/useDeviceOrientation.d.ts +48 -0
- package/dist/camera/useDeviceOrientation.js +131 -0
- package/dist/camera/useVideoCapture.d.ts +79 -0
- package/dist/camera/useVideoCapture.js +151 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.js +39 -0
- package/dist/quality/normaliseOrientation.d.ts +36 -0
- package/dist/quality/normaliseOrientation.js +62 -0
- package/dist/quality/runQualityCheck.d.ts +41 -0
- package/dist/quality/runQualityCheck.js +98 -0
- package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
- package/dist/sensors/useIMUTranslationGate.js +235 -0
- package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
- package/dist/stitching/IncrementalStitcherView.js +157 -0
- package/dist/stitching/incremental.d.ts +930 -0
- package/dist/stitching/incremental.js +133 -0
- package/dist/stitching/stitchFrames.d.ts +55 -0
- package/dist/stitching/stitchFrames.js +56 -0
- package/dist/stitching/stitchVideo.d.ts +119 -0
- package/dist/stitching/stitchVideo.js +57 -0
- package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
- package/dist/stitching/useIncrementalJSDriver.js +199 -0
- package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
- package/dist/stitching/useIncrementalStitcher.js +172 -0
- package/dist/types.d.ts +58 -0
- package/dist/types.js +15 -0
- package/ios/Package.swift +72 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
- package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
- package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
- package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
- package/package.json +73 -0
- package/react-native.config.js +34 -0
- package/scripts/opencv-version.txt +1 -0
- package/scripts/postinstall-fetch-binaries.js +286 -0
- package/src/ar/useARSession.ts +210 -0
- package/src/camera/.gitkeep +0 -0
- package/src/camera/ARCameraView.tsx +256 -0
- package/src/camera/Camera.tsx +1053 -0
- package/src/camera/CameraShutter.tsx +292 -0
- package/src/camera/CameraView.tsx +157 -0
- package/src/camera/CaptureControlsBar.tsx +204 -0
- package/src/camera/CaptureHeader.tsx +184 -0
- package/src/camera/CapturePreview.tsx +318 -0
- package/src/camera/CaptureStatusOverlay.tsx +391 -0
- package/src/camera/CaptureThumbnailStrip.tsx +277 -0
- package/src/camera/IncrementalPanGuide.tsx +328 -0
- package/src/camera/PanoramaBandOverlay.tsx +498 -0
- package/src/camera/PanoramaConfirmModal.tsx +206 -0
- package/src/camera/PanoramaGuidance.tsx +327 -0
- package/src/camera/PanoramaSettingsModal.tsx +1357 -0
- package/src/camera/ViewportCropOverlay.tsx +81 -0
- package/src/camera/useCapture.ts +279 -0
- package/src/camera/useDeviceOrientation.ts +140 -0
- package/src/camera/useVideoCapture.ts +236 -0
- package/src/index.ts +53 -0
- package/src/quality/.gitkeep +0 -0
- package/src/quality/normaliseOrientation.ts +79 -0
- package/src/quality/runQualityCheck.ts +131 -0
- package/src/sensors/useIMUTranslationGate.ts +347 -0
- package/src/stitching/.gitkeep +0 -0
- package/src/stitching/IncrementalStitcherView.tsx +198 -0
- package/src/stitching/incremental.ts +1021 -0
- package/src/stitching/stitchFrames.ts +88 -0
- package/src/stitching/stitchVideo.ts +153 -0
- package/src/stitching/useIncrementalJSDriver.ts +273 -0
- package/src/stitching/useIncrementalStitcher.ts +252 -0
- package/src/types.ts +78 -0
|
@@ -0,0 +1,1880 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// OpenCVStitcher.mm
|
|
4
|
+
//
|
|
5
|
+
// Objective-C++ implementation that wraps cv::Stitcher. This is the
|
|
6
|
+
// only file in the SDK that includes <opencv2/...> — everything else
|
|
7
|
+
// sees the slim `OpenCVStitcher.h` interface above and stays in
|
|
8
|
+
// Swift / Objective-C.
|
|
9
|
+
//
|
|
10
|
+
// History note (V12 → V16 → 2026-05-16 shared-C++ port):
|
|
11
|
+
// V12-era this file used `cv::Stitcher::SCANS` mode (translational,
|
|
12
|
+
// plane warp, ORB). V16 fix-11 reverted that to PANORAMA after
|
|
13
|
+
// discovering the AffineBestOf2NearestMatcher swap broke the
|
|
14
|
+
// warper/blender pipeline coherence (see learning doc
|
|
15
|
+
// `stitcher-pipeline-coherence`). 2026-05-15 reintroduced SCANS
|
|
16
|
+
// as one of two modes selected per-capture via the shared
|
|
17
|
+
// `StitchMode` enum in `cpp/stitcher.hpp` (translation-heavy
|
|
18
|
+
// captures → SCANS; rotation-heavy → PANORAMA; auto-resolved by
|
|
19
|
+
// the KeyframeGate's accumulated motion totals).
|
|
20
|
+
// 2026-05-16 commit 98b1a60 swapped this method body to delegate
|
|
21
|
+
// to the shared C++ at `cpp/stitcher.cpp` — both modes now live
|
|
22
|
+
// there, this file is just the Obj-C++ marshalling shim.
|
|
23
|
+
//
|
|
24
|
+
// References:
|
|
25
|
+
// * OpenCV docs: https://docs.opencv.org/4.x/d2/d8d/classcv_1_1Stitcher.html
|
|
26
|
+
// * Mode-selection design: docs/site-content/design/2026-05-13-stitch-pipeline-mode-selection.md
|
|
27
|
+
// * Pipeline coherence learning: docs/site-content/learnings/2026-05-13-stitcher-pipeline-coherence.md
|
|
28
|
+
|
|
29
|
+
// OpenCV's stitching headers contain `enum { NO, ... }` and `enum { YES, ... }`
|
|
30
|
+
// definitions. Objective-C's `<objc/objc.h>` (transitively imported by every
|
|
31
|
+
// Cocoapods prefix.pch) #defines `NO` and `YES` as macros for the boolean
|
|
32
|
+
// constants — by the time OpenCV's enum is parsed, the preprocessor has
|
|
33
|
+
// already eaten those identifiers and the build dies with "expected
|
|
34
|
+
// identifier". Undef both BEFORE importing opencv2/*. This is the
|
|
35
|
+
// standard pattern used by every ObjC++ ↔ OpenCV bridge.
|
|
36
|
+
#ifdef NO
|
|
37
|
+
#undef NO
|
|
38
|
+
#endif
|
|
39
|
+
#ifdef YES
|
|
40
|
+
#undef YES
|
|
41
|
+
#endif
|
|
42
|
+
|
|
43
|
+
#import <opencv2/opencv.hpp>
|
|
44
|
+
#import <opencv2/stitching.hpp>
|
|
45
|
+
#import <opencv2/imgcodecs.hpp>
|
|
46
|
+
#import <chrono>
|
|
47
|
+
#import <vector>
|
|
48
|
+
#import <string>
|
|
49
|
+
|
|
50
|
+
// Now that OpenCV is parsed, restore the ObjC macros + import the
|
|
51
|
+
// Foundation/UIKit deps the rest of this file uses.
|
|
52
|
+
#define NO ((BOOL)0)
|
|
53
|
+
#define YES ((BOOL)1)
|
|
54
|
+
|
|
55
|
+
#import "OpenCVStitcher.h"
|
|
56
|
+
// Phase 2 shared-stitcher port (2026-05-16): stitchFramePaths now
|
|
57
|
+
// delegates to the cross-platform C++ pipeline in cpp/stitcher.cpp.
|
|
58
|
+
// The header lives in the SDK's `cpp/` dir and is on the pod's
|
|
59
|
+
// HEADER_SEARCH_PATHS (see RNImageStitcher.podspec).
|
|
60
|
+
#import "stitcher.hpp"
|
|
61
|
+
#import <UIKit/UIKit.h>
|
|
62
|
+
#import <AVFoundation/AVFoundation.h>
|
|
63
|
+
#import <os/log.h>
|
|
64
|
+
// V16 Phase 1b.fix3 — ImageIO for EXIF Orientation tag on output
|
|
65
|
+
// panorama JPEG.
|
|
66
|
+
#import <ImageIO/ImageIO.h>
|
|
67
|
+
#import <mach/mach.h>
|
|
68
|
+
#import <mach/task.h>
|
|
69
|
+
#import <mach/task_info.h>
|
|
70
|
+
|
|
71
|
+
// V12.14.2 — dedicated os_log subsystem for the stitcher. os_log
|
|
72
|
+
// with OS_LOG_TYPE_FAULT survives Console.app's rate-limit cap that
|
|
73
|
+
// drops bursts of NSLog calls — Ram's V12.14 trace had Run 2's
|
|
74
|
+
// extractFrames + loadFrames + step1 entirely missing, only the
|
|
75
|
+
// step2-5 enter cluster surviving. We use FAULT level for SENTINEL
|
|
76
|
+
// breadcrumbs that MUST be visible (start of stitch, BA call site,
|
|
77
|
+
// any catch-all for the BA crash).
|
|
78
|
+
static os_log_t StitcherDiagLog(void) {
|
|
79
|
+
static os_log_t log = NULL;
|
|
80
|
+
static dispatch_once_t once;
|
|
81
|
+
dispatch_once(&once, ^{
|
|
82
|
+
log = os_log_create("com.tiger.retailens.sdk", "stitch");
|
|
83
|
+
});
|
|
84
|
+
return log;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// V12.14.7 — resident memory probe for jetsam diagnosis. Returns
|
|
88
|
+
// the process' resident_size in MB. When stitch fails with cv::Exception
|
|
89
|
+
// AND the app subsequently dies (V12.14.6 trace pattern: throw caught
|
|
90
|
+
// → app quits), iOS jetsam OOM-kill is the prime suspect. Logging
|
|
91
|
+
// resident_size before/after each pipeline stage lets us correlate
|
|
92
|
+
// the kill with a memory growth pattern across successive captures.
|
|
93
|
+
static double StitcherResidentMB(void) {
|
|
94
|
+
task_vm_info_data_t info;
|
|
95
|
+
mach_msg_type_number_t count = TASK_VM_INFO_COUNT;
|
|
96
|
+
kern_return_t kr = task_info(mach_task_self(), TASK_VM_INFO,
|
|
97
|
+
(task_info_t)&info, &count);
|
|
98
|
+
if (kr != KERN_SUCCESS) return -1.0;
|
|
99
|
+
// phys_footprint is what jetsam evaluates against; resident_size
|
|
100
|
+
// is what `ps`/Xcode shows. We log both via the same helper for
|
|
101
|
+
// correlation — phys_footprint is the one that matters for survival.
|
|
102
|
+
return (double)info.phys_footprint / (1024.0 * 1024.0);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// V16 Phase 1b.fix3 — find the largest axis-aligned rectangle that
|
|
106
|
+
// fits ENTIRELY inside the non-zero region of `mask` (CV_8UC1).
|
|
107
|
+
// Used to crop the post-stitch panorama tightly: the regular
|
|
108
|
+
// boundingRect of non-zero pixels still includes the black corners
|
|
109
|
+
// where the projection didn't fill; the max-inscribed rectangle
|
|
110
|
+
// excludes those entirely.
|
|
111
|
+
//
|
|
112
|
+
// Algorithm: maximum-rectangle-in-histogram swept row by row.
|
|
113
|
+
// O(W * H). For a 4-6 MP panorama on iPhone 16 Pro, completes in
|
|
114
|
+
// 30-60 ms.
|
|
115
|
+
//
|
|
116
|
+
// Returns cv::Rect(0,0,0,0) if `mask` is empty or fully zero.
|
|
117
|
+
static cv::Rect MaxInscribedRectFromMask(const cv::Mat &mask) {
|
|
118
|
+
if (mask.empty() || mask.type() != CV_8UC1) {
|
|
119
|
+
return cv::Rect();
|
|
120
|
+
}
|
|
121
|
+
const int H = mask.rows;
|
|
122
|
+
const int W = mask.cols;
|
|
123
|
+
|
|
124
|
+
// Per-column running heights of consecutive non-zero pixels
|
|
125
|
+
// ending at the current row.
|
|
126
|
+
std::vector<int> heights((size_t)W, 0);
|
|
127
|
+
cv::Rect bestRect(0, 0, 0, 0);
|
|
128
|
+
long long bestArea = 0;
|
|
129
|
+
|
|
130
|
+
// Reusable monotonic stack for the row's largest-rectangle-in-
|
|
131
|
+
// histogram subroutine.
|
|
132
|
+
std::vector<int> stack;
|
|
133
|
+
stack.reserve((size_t)W + 1);
|
|
134
|
+
|
|
135
|
+
for (int row = 0; row < H; ++row) {
|
|
136
|
+
const uchar *m = mask.ptr<uchar>(row);
|
|
137
|
+
for (int col = 0; col < W; ++col) {
|
|
138
|
+
heights[(size_t)col] =
|
|
139
|
+
(m[col] != 0) ? heights[(size_t)col] + 1 : 0;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Largest rectangle in the histogram for this row.
|
|
143
|
+
stack.clear();
|
|
144
|
+
for (int col = 0; col <= W; ++col) {
|
|
145
|
+
const int h = (col == W) ? 0 : heights[(size_t)col];
|
|
146
|
+
while (!stack.empty()
|
|
147
|
+
&& heights[(size_t)stack.back()] > h) {
|
|
148
|
+
const int topIdx = stack.back();
|
|
149
|
+
stack.pop_back();
|
|
150
|
+
const int leftIdx =
|
|
151
|
+
stack.empty() ? -1 : stack.back();
|
|
152
|
+
const int width = col - leftIdx - 1;
|
|
153
|
+
const long long area =
|
|
154
|
+
(long long)heights[(size_t)topIdx]
|
|
155
|
+
* (long long)width;
|
|
156
|
+
if (area > bestArea) {
|
|
157
|
+
bestArea = area;
|
|
158
|
+
bestRect = cv::Rect(
|
|
159
|
+
leftIdx + 1,
|
|
160
|
+
row - heights[(size_t)topIdx] + 1,
|
|
161
|
+
width,
|
|
162
|
+
heights[(size_t)topIdx]
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
stack.push_back(col);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return bestRect;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
// V16 Phase 1b.fix3 — write a cv::Mat (BGR) as a JPEG with an EXIF
|
|
174
|
+
// Orientation tag, via ImageIO. iOS image renderers (UIImage,
|
|
175
|
+
// RN's <Image>, Files.app, Photos) honour the tag; cv::imread with
|
|
176
|
+
// IMREAD_IGNORE_ORIENTATION returns raw landscape pixels. Mirrors
|
|
177
|
+
// the helper of the same name in OpenCVKeyframeCollector.mm — kept
|
|
178
|
+
// duplicated rather than refactored to a shared header per the
|
|
179
|
+
// codebase convention ("duplicate stage code, DRY when proven").
|
|
180
|
+
static BOOL WriteJPEGWithEXIFTag(const cv::Mat &bgr,
|
|
181
|
+
NSString *path,
|
|
182
|
+
NSInteger exifOrientation,
|
|
183
|
+
NSInteger quality) {
|
|
184
|
+
if (bgr.empty()) return NO;
|
|
185
|
+
|
|
186
|
+
cv::Mat rgba;
|
|
187
|
+
cv::cvtColor(bgr, rgba, cv::COLOR_BGR2RGBA);
|
|
188
|
+
|
|
189
|
+
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
|
|
190
|
+
CGBitmapInfo bitmapInfo =
|
|
191
|
+
kCGBitmapByteOrderDefault | kCGImageAlphaNoneSkipLast;
|
|
192
|
+
CGContextRef ctx = CGBitmapContextCreate(
|
|
193
|
+
rgba.data,
|
|
194
|
+
(size_t)rgba.cols,
|
|
195
|
+
(size_t)rgba.rows,
|
|
196
|
+
8,
|
|
197
|
+
(size_t)rgba.step,
|
|
198
|
+
colorSpace,
|
|
199
|
+
bitmapInfo);
|
|
200
|
+
if (!ctx) {
|
|
201
|
+
CGColorSpaceRelease(colorSpace);
|
|
202
|
+
return NO;
|
|
203
|
+
}
|
|
204
|
+
CGImageRef cgImage = CGBitmapContextCreateImage(ctx);
|
|
205
|
+
CGContextRelease(ctx);
|
|
206
|
+
CGColorSpaceRelease(colorSpace);
|
|
207
|
+
if (!cgImage) return NO;
|
|
208
|
+
|
|
209
|
+
NSURL *url = [NSURL fileURLWithPath:path];
|
|
210
|
+
CGImageDestinationRef dst = CGImageDestinationCreateWithURL(
|
|
211
|
+
(__bridge CFURLRef)url,
|
|
212
|
+
CFSTR("public.jpeg"),
|
|
213
|
+
1,
|
|
214
|
+
NULL);
|
|
215
|
+
if (!dst) {
|
|
216
|
+
CGImageRelease(cgImage);
|
|
217
|
+
return NO;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
NSInteger q = MAX(0, MIN(100, quality));
|
|
221
|
+
NSInteger exif = (exifOrientation >= 1 && exifOrientation <= 8)
|
|
222
|
+
? exifOrientation : 1;
|
|
223
|
+
NSDictionary *props = @{
|
|
224
|
+
(id)kCGImageDestinationLossyCompressionQuality:
|
|
225
|
+
@((double)q / 100.0),
|
|
226
|
+
(id)kCGImagePropertyOrientation: @(exif),
|
|
227
|
+
};
|
|
228
|
+
CGImageDestinationAddImage(
|
|
229
|
+
dst, cgImage, (__bridge CFDictionaryRef)props);
|
|
230
|
+
BOOL ok = CGImageDestinationFinalize(dst);
|
|
231
|
+
CFRelease(dst);
|
|
232
|
+
CGImageRelease(cgImage);
|
|
233
|
+
return ok;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
NSString *const RNImageStitcherErrorDomain = @"RNImageStitcherErrorDomain";
|
|
238
|
+
|
|
239
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
240
|
+
// RNStitchResult
|
|
241
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
242
|
+
|
|
243
|
+
@implementation RNStitchResult
|
|
244
|
+
|
|
245
|
+
- (instancetype)initWithOutputPath:(NSString *)outputPath
|
|
246
|
+
width:(NSInteger)width
|
|
247
|
+
height:(NSInteger)height
|
|
248
|
+
durationMs:(double)durationMs
|
|
249
|
+
framesRequested:(NSInteger)framesRequested
|
|
250
|
+
framesIncluded:(NSInteger)framesIncluded
|
|
251
|
+
finalConfidenceThresh:(double)finalConfidenceThresh {
|
|
252
|
+
self = [super init];
|
|
253
|
+
if (self) {
|
|
254
|
+
_outputPath = [outputPath copy];
|
|
255
|
+
_width = width;
|
|
256
|
+
_height = height;
|
|
257
|
+
_durationMs = durationMs;
|
|
258
|
+
_framesRequested = framesRequested;
|
|
259
|
+
_framesIncluded = framesIncluded;
|
|
260
|
+
_finalConfidenceThresh = finalConfidenceThresh;
|
|
261
|
+
}
|
|
262
|
+
return self;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
- (instancetype)initWithOutputPath:(NSString *)outputPath
|
|
266
|
+
width:(NSInteger)width
|
|
267
|
+
height:(NSInteger)height
|
|
268
|
+
durationMs:(double)durationMs {
|
|
269
|
+
return [self initWithOutputPath:outputPath
|
|
270
|
+
width:width
|
|
271
|
+
height:height
|
|
272
|
+
durationMs:durationMs
|
|
273
|
+
framesRequested:-1
|
|
274
|
+
framesIncluded:-1
|
|
275
|
+
finalConfidenceThresh:-1.0];
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
@end
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
282
|
+
// Helpers
|
|
283
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
284
|
+
|
|
285
|
+
namespace {
|
|
286
|
+
|
|
287
|
+
// Strip the `file://` scheme some callers attach so cv::imread can
|
|
288
|
+
// open the path (cv::imread takes a filesystem path, not a URL).
|
|
289
|
+
NSString *normalizeImagePath(NSString *path) {
|
|
290
|
+
if ([path hasPrefix:@"file://"]) {
|
|
291
|
+
return [path substringFromIndex:[@"file://" length]];
|
|
292
|
+
}
|
|
293
|
+
return path;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// 2026-05-16 (post-Phase-2 cleanup): `loadFramesOrFail()` and
|
|
297
|
+
// `errorForStitchStatus()` removed from this file. Both were
|
|
298
|
+
// called only by the prior `stitchFramePaths:` method body that
|
|
299
|
+
// was replaced by the shared-C++ delegating wrapper in commit
|
|
300
|
+
// 98b1a60. Equivalents now live at:
|
|
301
|
+
// - frame loading: cpp/stitcher.cpp anonymous-namespace
|
|
302
|
+
// `loadAllFrames()` (called by both the high-level and manual
|
|
303
|
+
// entries)
|
|
304
|
+
// - error mapping: the explicit StitchErrorCode → NSError.code
|
|
305
|
+
// switch in this file at the new wrapper (lines ~528-595)
|
|
306
|
+
// Removed to keep the anonymous namespace tight; sibling methods
|
|
307
|
+
// (stitchKeyframePaths, stitchVideoAtPath) don't need them.
|
|
308
|
+
|
|
309
|
+
// Phase 5: build a cv::detail::CameraParams from an ARKit pose.
|
|
310
|
+
//
|
|
311
|
+
// ARKit's camera-to-world transform uses a right-handed system
|
|
312
|
+
// with +X right, +Y up, -Z forward (out of the screen). OpenCV
|
|
313
|
+
// uses +X right, +Y down, +Z forward (into the scene). Conversion
|
|
314
|
+
// is:
|
|
315
|
+
//
|
|
316
|
+
// M = diag(1, -1, -1) // axis-flip from ARKit → OpenCV
|
|
317
|
+
// R_ar_to_world = quaternion → 3x3 rotation matrix
|
|
318
|
+
// R_world_to_cv = M * R_ar_to_world.transpose()
|
|
319
|
+
//
|
|
320
|
+
// The transpose is what changes from camera-to-world (what ARKit
|
|
321
|
+
// gives us) to world-to-camera (what cv::detail::CameraParams.R
|
|
322
|
+
// expects). We don't set CameraParams.t — for panoramic stitching,
|
|
323
|
+
// translation is largely irrelevant (warpers project rays, not
|
|
324
|
+
// world points), and ARKit's metric translations would otherwise
|
|
325
|
+
// throw off cv::detail::SphericalWarper's scale heuristics.
|
|
326
|
+
//
|
|
327
|
+
// Intrinsics come straight from ARFrame.camera.intrinsics —
|
|
328
|
+
// focal length and principal point in pixels at the ARFrame's
|
|
329
|
+
// native resolution.
|
|
330
|
+
cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
|
|
331
|
+
cv::detail::CameraParams cam;
|
|
332
|
+
|
|
333
|
+
double qx = [pose[@"qx"] doubleValue];
|
|
334
|
+
double qy = [pose[@"qy"] doubleValue];
|
|
335
|
+
double qz = [pose[@"qz"] doubleValue];
|
|
336
|
+
double qw = [pose[@"qw"] doubleValue];
|
|
337
|
+
|
|
338
|
+
// Quaternion → 3x3 rotation matrix (camera-to-world in ARKit).
|
|
339
|
+
// Standard formula; assumes the quaternion is unit-length
|
|
340
|
+
// (ARKit guarantees this).
|
|
341
|
+
cv::Mat R_ar = (cv::Mat_<double>(3, 3) <<
|
|
342
|
+
1 - 2*(qy*qy + qz*qz), 2*(qx*qy - qw*qz), 2*(qx*qz + qw*qy),
|
|
343
|
+
2*(qx*qy + qw*qz), 1 - 2*(qx*qx + qz*qz), 2*(qy*qz - qw*qx),
|
|
344
|
+
2*(qx*qz - qw*qy), 2*(qy*qz + qw*qx), 1 - 2*(qx*qx + qy*qy)
|
|
345
|
+
);
|
|
346
|
+
|
|
347
|
+
// Axis-flip matrix: ARKit Y-up → OpenCV Y-down, ARKit -Z forward
|
|
348
|
+
// → OpenCV +Z forward.
|
|
349
|
+
cv::Mat M = (cv::Mat_<double>(3, 3) <<
|
|
350
|
+
1, 0, 0,
|
|
351
|
+
0, -1, 0,
|
|
352
|
+
0, 0, -1
|
|
353
|
+
);
|
|
354
|
+
|
|
355
|
+
// R_world_to_cv = M * R_ar_to_world.T
|
|
356
|
+
cv::Mat R_world_to_cv = M * R_ar.t();
|
|
357
|
+
cv::Mat R_float;
|
|
358
|
+
R_world_to_cv.convertTo(R_float, CV_32F);
|
|
359
|
+
cam.R = R_float;
|
|
360
|
+
cam.t = cv::Mat::zeros(3, 1, CV_32F);
|
|
361
|
+
|
|
362
|
+
// Intrinsics — at the pose's native image resolution. The
|
|
363
|
+
// compose-rescale step below will adjust these to compose scale.
|
|
364
|
+
double fx = [pose[@"fx"] doubleValue];
|
|
365
|
+
double fy = [pose[@"fy"] doubleValue];
|
|
366
|
+
cam.focal = (fx + fy) / 2.0;
|
|
367
|
+
cam.aspect = (fx > 0.0) ? (fy / fx) : 1.0;
|
|
368
|
+
cam.ppx = [pose[@"cx"] doubleValue];
|
|
369
|
+
cam.ppy = [pose[@"cy"] doubleValue];
|
|
370
|
+
|
|
371
|
+
return cam;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
} // namespace
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
378
|
+
// OpenCVStitcher (public)
|
|
379
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
380
|
+
|
|
381
|
+
@implementation OpenCVStitcher
|
|
382
|
+
|
|
383
|
+
+ (nullable RNStitchResult *)stitchFramePaths:(NSArray<NSString *> *)framePaths
|
|
384
|
+
outputPath:(NSString *)outputPath
|
|
385
|
+
jpegQuality:(NSInteger)quality
|
|
386
|
+
warperType:(NSString *)warperType
|
|
387
|
+
blenderType:(NSString *)blenderType
|
|
388
|
+
seamFinderType:(NSString *)seamFinderType
|
|
389
|
+
captureOrientation:(NSString *)captureOrientation
|
|
390
|
+
useInscribedRectCrop:(BOOL)useInscribedRectCrop
|
|
391
|
+
error:(NSError **)error {
|
|
392
|
+
// ── Phase 2 (2026-05-16): delegated to shared C++ ───────────────────
|
|
393
|
+
//
|
|
394
|
+
// The hand-rolled cv::detail::* pipeline that used to live here
|
|
395
|
+
// (~1500 lines from the original implementation, covering frames-
|
|
396
|
+
// load → ORB features → BestOf2Nearest matching → leaveBiggest
|
|
397
|
+
// Component → HomographyBasedEstimator → BundleAdjusterRay → wave
|
|
398
|
+
// correct → median-focal warper-scale → seam find → multi-band
|
|
399
|
+
// blend → max-inscribed-rect crop → bake-rotate → JPEG write) was
|
|
400
|
+
// ported verbatim to `retailens::stitchFramePathsManual()` in
|
|
401
|
+
// cpp/stitcher.cpp during Phase 1 (commit 02534ac). Android already
|
|
402
|
+
// routes through the same file via the high-level pipeline; iOS
|
|
403
|
+
// now routes through it via `useManualPipeline=true`.
|
|
404
|
+
//
|
|
405
|
+
// Git blame on commit 02534ac (and its parent) captures the full
|
|
406
|
+
// algorithm history with the original step-by-step comments. The
|
|
407
|
+
// shared C++ file carries forward equivalent comments at each step.
|
|
408
|
+
//
|
|
409
|
+
// This wrapper's only job: marshal Obj-C args into the shared
|
|
410
|
+
// StitchConfig + std::vector<std::string>, route logs to os_log,
|
|
411
|
+
// map StitchErrorCode → NSError.code so the JS-side UX taxonomy
|
|
412
|
+
// (9001 / 9002 / … / 9007) is preserved.
|
|
413
|
+
|
|
414
|
+
// Defaults if caller passed nil — keeps the older 3-arg call-sites
|
|
415
|
+
// working until they are updated. The shared C++ has its own
|
|
416
|
+
// defaults but we want the wrapper to be tolerant of nil inputs
|
|
417
|
+
// from Swift / Obj-C callers that grew up against the legacy API.
|
|
418
|
+
if (warperType == nil || warperType.length == 0) warperType = @"plane";
|
|
419
|
+
if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
|
|
420
|
+
if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
|
|
421
|
+
if (captureOrientation == nil || captureOrientation.length == 0) captureOrientation = @"portrait";
|
|
422
|
+
|
|
423
|
+
// Build the shared-C++ config. Sentinel resolution budgets (-1.0)
|
|
424
|
+
// let the manual entry point pick its own defaults (registration
|
|
425
|
+
// 0.6 MP / seam 0.1 MP / compose 0.6 MP per Phase 1 fixes).
|
|
426
|
+
retailens::StitchConfig cfg;
|
|
427
|
+
cfg.warperType = warperType.UTF8String;
|
|
428
|
+
cfg.blenderType = blenderType.UTF8String;
|
|
429
|
+
cfg.seamFinderType = seamFinderType.UTF8String;
|
|
430
|
+
cfg.captureOrientation = captureOrientation.UTF8String;
|
|
431
|
+
cfg.useInscribedRectCrop = (useInscribedRectCrop != NO);
|
|
432
|
+
cfg.jpegQuality = (int)quality;
|
|
433
|
+
// The iOS API doesn't expose stitchMode yet; defaulting to Panorama
|
|
434
|
+
// matches the prior hand-rolled pipeline's BestOf2NearestMatcher +
|
|
435
|
+
// BundleAdjusterRay configuration (rotation-only end-to-end).
|
|
436
|
+
cfg.stitchMode = retailens::StitchMode::Panorama;
|
|
437
|
+
// Pre-stitch memory-abort threshold inside the manual pipeline keys
|
|
438
|
+
// off this value. Plumb the device's physical RAM through so the
|
|
439
|
+
// heuristic scales correctly across the iPhone fleet (~2 GB legacy
|
|
440
|
+
// → ~8 GB iPhone 16 Pro).
|
|
441
|
+
cfg.availableRamMB =
|
|
442
|
+
(double)NSProcessInfo.processInfo.physicalMemory
|
|
443
|
+
/ (1024.0 * 1024.0);
|
|
444
|
+
// Route to the manual cv::detail::* pipeline; the high-level
|
|
445
|
+
// cv::Stitcher::create path (Android's default) is unsuitable for
|
|
446
|
+
// iOS's shelf-pan capture shape (compose-MP defaults, graphcut at
|
|
447
|
+
// compose-MP, BA convergence params — see stitcher.hpp comment
|
|
448
|
+
// block).
|
|
449
|
+
cfg.useManualPipeline = true;
|
|
450
|
+
|
|
451
|
+
// Marshal NSArray<NSString*> → std::vector<std::string>. Strip the
|
|
452
|
+
// `file://` scheme that some callers attach so the shared C++ can
|
|
453
|
+
// cv::imread the raw filesystem path.
|
|
454
|
+
std::vector<std::string> paths;
|
|
455
|
+
paths.reserve(framePaths.count);
|
|
456
|
+
for (NSString *p in framePaths) {
|
|
457
|
+
NSString *cleaned = p;
|
|
458
|
+
if ([cleaned hasPrefix:@"file://"]) {
|
|
459
|
+
cleaned = [cleaned substringFromIndex:[@"file://" length]];
|
|
460
|
+
}
|
|
461
|
+
paths.emplace_back(cleaned.UTF8String);
|
|
462
|
+
}
|
|
463
|
+
NSString *cleanedOutputPath = outputPath;
|
|
464
|
+
if ([cleanedOutputPath hasPrefix:@"file://"]) {
|
|
465
|
+
cleanedOutputPath = [cleanedOutputPath substringFromIndex:[@"file://" length]];
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// Logging callback: route shared-C++ logs to the same os_log
|
|
469
|
+
// subsystem the rest of this file uses, so Console.app shows them
|
|
470
|
+
// alongside the existing breadcrumbs. Level mapping mirrors what
|
|
471
|
+
// the shared C++ already documents (0=info, 1=warn, 2=error).
|
|
472
|
+
retailens::LogFn logFn = [](int level, const char *tag, const char *msg) {
|
|
473
|
+
os_log_type_t logType;
|
|
474
|
+
switch (level) {
|
|
475
|
+
case 0: logType = OS_LOG_TYPE_INFO; break;
|
|
476
|
+
case 1: logType = OS_LOG_TYPE_DEFAULT; break;
|
|
477
|
+
case 2: logType = OS_LOG_TYPE_FAULT; break;
|
|
478
|
+
default: logType = OS_LOG_TYPE_DEFAULT; break;
|
|
479
|
+
}
|
|
480
|
+
os_log_with_type(StitcherDiagLog(), logType, "%{public}s %{public}s",
|
|
481
|
+
tag ? tag : "[stitch]", msg ? msg : "");
|
|
482
|
+
};
|
|
483
|
+
|
|
484
|
+
// ── Run the stitch under an @autoreleasepool ─────────────────────
|
|
485
|
+
//
|
|
486
|
+
// fix-10 pattern (see line 599-ish of the prior file revision for
|
|
487
|
+
// the canonical comment block — preserved by git blame on the
|
|
488
|
+
// pre-Phase-2 commit): any NSError/NSString created INSIDE the
|
|
489
|
+
// pool would otherwise be autoreleased into the pool and freed at
|
|
490
|
+
// the closing brace BEFORE Swift's `objc_retainAutoreleasedReturn-
|
|
491
|
+
// Value` could retain it, producing the EXC_BAD_ACCESS the old
|
|
492
|
+
// implementation chased through fix-1 through fix-9.
|
|
493
|
+
//
|
|
494
|
+
// For this wrapper the C++ call doesn't autorelease anything by
|
|
495
|
+
// itself, but ANY `[NSString stringWithUTF8String:]` or
|
|
496
|
+
// `[NSError errorWithDomain:…]` we build from the result IS
|
|
497
|
+
// autoreleased. So we run the C++ call + the NSError build inside
|
|
498
|
+
// the pool, but capture the NSError into a STRONG LOCAL declared
|
|
499
|
+
// ABOVE the pool. The pool drains; the strong local survives
|
|
500
|
+
// (ARC retain on the alloc, NOT autoreleased); after the pool we
|
|
501
|
+
// either return the success result, write `*error` from the strong
|
|
502
|
+
// local, or fall through.
|
|
503
|
+
//
|
|
504
|
+
// See docs/site-content/learnings/react-native.md#autoreleasepool-return-uaf
|
|
505
|
+
RNStitchResult *result = nil;
|
|
506
|
+
NSError *capturedError = nil;
|
|
507
|
+
@autoreleasepool {
|
|
508
|
+
retailens::StitchResult r = retailens::stitchFramePaths(
|
|
509
|
+
paths,
|
|
510
|
+
cleanedOutputPath.UTF8String,
|
|
511
|
+
cfg,
|
|
512
|
+
logFn);
|
|
513
|
+
|
|
514
|
+
if (r.success) {
|
|
515
|
+
const int64_t durationMs = r.durationMs;
|
|
516
|
+
// 2026-05-16 (Issue 5) — pass C+D retry telemetry up to Swift so
|
|
517
|
+
// the JS finalize dict can carry it. framesRequested defaults
|
|
518
|
+
// to the input count when the cpp path didn't fill it (e.g. an
|
|
519
|
+
// early-return success path that bypassed the retry loop).
|
|
520
|
+
const NSInteger framesRequested =
|
|
521
|
+
r.framesRequested > 0 ? (NSInteger)r.framesRequested
|
|
522
|
+
: (NSInteger)paths.size();
|
|
523
|
+
result = [[RNStitchResult alloc]
|
|
524
|
+
initWithOutputPath:outputPath
|
|
525
|
+
width:(NSInteger)r.width
|
|
526
|
+
height:(NSInteger)r.height
|
|
527
|
+
durationMs:(double)durationMs
|
|
528
|
+
framesRequested:framesRequested
|
|
529
|
+
framesIncluded:(NSInteger)r.framesIncluded
|
|
530
|
+
finalConfidenceThresh:r.finalConfidenceThresh];
|
|
531
|
+
} else {
|
|
532
|
+
// Map StitchErrorCode → NSError.code. Preserves the existing
|
|
533
|
+
// 9001/9002/9003/1001/9007 sentinels the JS UX layer already
|
|
534
|
+
// branches on; adds new codes 9100-9103 for manual-pipeline-
|
|
535
|
+
// specific failure modes that previously collapsed into
|
|
536
|
+
// 9007 / generic crashes.
|
|
537
|
+
NSInteger nsCode = 9999;
|
|
538
|
+
switch (r.errorCode) {
|
|
539
|
+
case retailens::StitchErrorCode::NeedMoreImages:
|
|
540
|
+
nsCode = 9001;
|
|
541
|
+
break;
|
|
542
|
+
case retailens::StitchErrorCode::HomographyEstimationFailed:
|
|
543
|
+
nsCode = 9002;
|
|
544
|
+
break;
|
|
545
|
+
case retailens::StitchErrorCode::CameraParamsAdjustFailed:
|
|
546
|
+
nsCode = 9003;
|
|
547
|
+
break;
|
|
548
|
+
case retailens::StitchErrorCode::ImageReadFailed:
|
|
549
|
+
nsCode = 1001;
|
|
550
|
+
break;
|
|
551
|
+
case retailens::StitchErrorCode::AllFramesDroppedByConfidence:
|
|
552
|
+
// 9007 preserves the existing sentinel the JS-side surfaces
|
|
553
|
+
// as "could not stitch — try recapturing with more overlap";
|
|
554
|
+
// changing this would silently flip the operator-facing
|
|
555
|
+
// copy across the app.
|
|
556
|
+
nsCode = 9007;
|
|
557
|
+
break;
|
|
558
|
+
case retailens::StitchErrorCode::PreStitchMemoryAbort:
|
|
559
|
+
nsCode = 9100;
|
|
560
|
+
break;
|
|
561
|
+
case retailens::StitchErrorCode::ComposeResizeFailed:
|
|
562
|
+
nsCode = 9101;
|
|
563
|
+
break;
|
|
564
|
+
case retailens::StitchErrorCode::WarpFailed:
|
|
565
|
+
nsCode = 9102;
|
|
566
|
+
break;
|
|
567
|
+
case retailens::StitchErrorCode::EmptyPanorama:
|
|
568
|
+
nsCode = 9103;
|
|
569
|
+
break;
|
|
570
|
+
case retailens::StitchErrorCode::InvalidArgument:
|
|
571
|
+
nsCode = 9000;
|
|
572
|
+
break;
|
|
573
|
+
default:
|
|
574
|
+
nsCode = 9999;
|
|
575
|
+
break;
|
|
576
|
+
}
|
|
577
|
+
NSString *msg =
|
|
578
|
+
[NSString stringWithUTF8String:r.errorMessage.c_str()] ?: @"Stitch failed";
|
|
579
|
+
capturedError = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
580
|
+
code:nsCode
|
|
581
|
+
userInfo:@{NSLocalizedDescriptionKey: msg}];
|
|
582
|
+
}
|
|
583
|
+
} // end @autoreleasepool — drains shared-C++ temporary NSStrings
|
|
584
|
+
// that we built from r.errorMessage / tag strings. result and
|
|
585
|
+
// capturedError survive the drain because they were assigned
|
|
586
|
+
// to strong locals declared ABOVE the pool.
|
|
587
|
+
|
|
588
|
+
// Failure path: outparameter assignment happens AFTER the pool
|
|
589
|
+
// drains so the NSError lives in the OUTER pool (drained by the
|
|
590
|
+
// GCD work item / Swift autoreleasing boundary).
|
|
591
|
+
if (capturedError != nil) {
|
|
592
|
+
if (error) {
|
|
593
|
+
*error = capturedError;
|
|
594
|
+
}
|
|
595
|
+
return nil;
|
|
596
|
+
}
|
|
597
|
+
return result;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
602
|
+
// Video → frames (AVFoundation, no OpenCV)
|
|
603
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
604
|
+
|
|
605
|
+
+ (nullable NSArray<NSString *> *)extractFramesFromVideoAtPath:(NSString *)videoPath
|
|
606
|
+
outputDir:(NSString *)outputDir
|
|
607
|
+
maxFrames:(NSInteger)maxFrames
|
|
608
|
+
jpegQuality:(NSInteger)quality
|
|
609
|
+
error:(NSError **)error {
|
|
610
|
+
if (maxFrames < 2) {
|
|
611
|
+
if (error) {
|
|
612
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
613
|
+
code:1010
|
|
614
|
+
userInfo:@{
|
|
615
|
+
NSLocalizedDescriptionKey:
|
|
616
|
+
@"maxFrames must be ≥ 2 for the stitcher to have something to align.",
|
|
617
|
+
}];
|
|
618
|
+
}
|
|
619
|
+
return nil;
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
NSString *cleanedVideoPath = normalizeImagePath(videoPath);
|
|
623
|
+
NSURL *videoURL = [NSURL fileURLWithPath:cleanedVideoPath];
|
|
624
|
+
if (![[NSFileManager defaultManager] fileExistsAtPath:cleanedVideoPath]) {
|
|
625
|
+
if (error) {
|
|
626
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
627
|
+
code:1011
|
|
628
|
+
userInfo:@{
|
|
629
|
+
NSLocalizedDescriptionKey:
|
|
630
|
+
[NSString stringWithFormat:@"Video file not found: %@", videoPath],
|
|
631
|
+
}];
|
|
632
|
+
}
|
|
633
|
+
return nil;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
// Make sure outputDir exists; the SDK call creates it but be
|
|
637
|
+
// defensive in case the host wrote a literal path that doesn't.
|
|
638
|
+
[[NSFileManager defaultManager] createDirectoryAtPath:normalizeImagePath(outputDir)
|
|
639
|
+
withIntermediateDirectories:YES
|
|
640
|
+
attributes:nil
|
|
641
|
+
error:nil];
|
|
642
|
+
|
|
643
|
+
AVURLAsset *asset = [AVURLAsset assetWithURL:videoURL];
|
|
644
|
+
CMTime duration = asset.duration;
|
|
645
|
+
Float64 totalSeconds = CMTimeGetSeconds(duration);
|
|
646
|
+
if (!isfinite(totalSeconds) || totalSeconds <= 0) {
|
|
647
|
+
if (error) {
|
|
648
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
649
|
+
code:1012
|
|
650
|
+
userInfo:@{
|
|
651
|
+
NSLocalizedDescriptionKey:
|
|
652
|
+
@"Could not read video duration — file may be corrupt or still being written.",
|
|
653
|
+
}];
|
|
654
|
+
}
|
|
655
|
+
return nil;
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
AVAssetImageGenerator *generator =
|
|
659
|
+
[AVAssetImageGenerator assetImageGeneratorWithAsset:asset];
|
|
660
|
+
// Honour the camera's recorded orientation — without this, all
|
|
661
|
+
// frames come out unrotated and stitch into a sideways panorama.
|
|
662
|
+
generator.appliesPreferredTrackTransform = YES;
|
|
663
|
+
// Tight tolerances → AVFoundation seeks to the requested timestamp
|
|
664
|
+
// exactly rather than the nearest keyframe. Cost: slower extract.
|
|
665
|
+
// Worth it; nearest-keyframe sampling can give near-duplicate frames
|
|
666
|
+
// when the keyframe interval lines up with our sample rate.
|
|
667
|
+
generator.requestedTimeToleranceBefore = kCMTimeZero;
|
|
668
|
+
generator.requestedTimeToleranceAfter = kCMTimeZero;
|
|
669
|
+
|
|
670
|
+
NSInteger clampedQuality = MAX(0, MIN(100, quality));
|
|
671
|
+
CGFloat compressionQuality = clampedQuality / 100.0;
|
|
672
|
+
|
|
673
|
+
NSMutableArray<NSString *> *paths =
|
|
674
|
+
[NSMutableArray arrayWithCapacity:(NSUInteger)maxFrames];
|
|
675
|
+
NSString *cleanedOutputDir = normalizeImagePath(outputDir);
|
|
676
|
+
|
|
677
|
+
// V12.13 — diagnostic for the landscape-only `EXC_BAD_ACCESS` crash
|
|
678
|
+
// Ram caught. Track per-frame extract progress + dimensions so the
|
|
679
|
+
// log breadcrumb pinpoints which stage and frame triggers the
|
|
680
|
+
// memory error if it recurs. Also log the asset's video track size
|
|
681
|
+
// + preferred transform up front so we know what AVFoundation is
|
|
682
|
+
// about to hand us before the loop runs.
|
|
683
|
+
AVAssetTrack *videoTrack = nil;
|
|
684
|
+
NSArray *videoTracks = [asset tracksWithMediaType:AVMediaTypeVideo];
|
|
685
|
+
if (videoTracks.count > 0) {
|
|
686
|
+
videoTrack = videoTracks.firstObject;
|
|
687
|
+
}
|
|
688
|
+
CGSize naturalSize = videoTrack ? videoTrack.naturalSize : CGSizeZero;
|
|
689
|
+
CGAffineTransform xform = videoTrack ? videoTrack.preferredTransform
|
|
690
|
+
: CGAffineTransformIdentity;
|
|
691
|
+
NSLog(@"[stitch-bc] extractFrames start: maxFrames=%ld duration=%.2fs "
|
|
692
|
+
@"track.naturalSize=%.0fx%.0f preferredTransform=[a=%.2f b=%.2f c=%.2f d=%.2f tx=%.2f ty=%.2f]",
|
|
693
|
+
(long)maxFrames, totalSeconds,
|
|
694
|
+
naturalSize.width, naturalSize.height,
|
|
695
|
+
xform.a, xform.b, xform.c, xform.d, xform.tx, xform.ty);
|
|
696
|
+
|
|
697
|
+
for (NSInteger i = 0; i < maxFrames; i++) {
|
|
698
|
+
// V12.13 — wrap each iteration in its own @autoreleasepool so
|
|
699
|
+
// UIImage / NSData / NSString temporaries get drained per-frame
|
|
700
|
+
// instead of accumulating to function exit. Without this, a
|
|
701
|
+
// 30-frame extract from a landscape video can hold ~100+ MB of
|
|
702
|
+
// autoreleased temporaries — combined with the video extractor's
|
|
703
|
+
// own caches this has historically triggered jetsam +
|
|
704
|
+
// EXC_BAD_ACCESS-during-tear-down (see the COMPOSE_MP comment
|
|
705
|
+
// around line 313 of stitchFramePaths for the same pattern).
|
|
706
|
+
@autoreleasepool {
|
|
707
|
+
// Even time spacing across [0, duration]. Dividing by
|
|
708
|
+
// (maxFrames - 1) gives endpoints at exactly 0 and `duration`,
|
|
709
|
+
// capturing the first and last useful moments.
|
|
710
|
+
Float64 fraction = (Float64)i / (Float64)(maxFrames - 1);
|
|
711
|
+
Float64 timeSeconds = fraction * totalSeconds;
|
|
712
|
+
CMTime cmTime = CMTimeMakeWithSeconds(timeSeconds, 600);
|
|
713
|
+
|
|
714
|
+
NSError *frameErr = nil;
|
|
715
|
+
CGImageRef cgImage =
|
|
716
|
+
[generator copyCGImageAtTime:cmTime actualTime:NULL error:&frameErr];
|
|
717
|
+
if (cgImage == NULL) {
|
|
718
|
+
NSLog(@"[stitch-bc] frame %ld/%ld: copyCGImageAtTime returned NULL "
|
|
719
|
+
@"(t=%.2fs, err=%@)",
|
|
720
|
+
(long)i, (long)maxFrames, timeSeconds,
|
|
721
|
+
frameErr.localizedDescription ?: @"nil");
|
|
722
|
+
// Skip an unreadable frame rather than aborting — sometimes
|
|
723
|
+
// the very-last-millisecond seek fails on short videos. The
|
|
724
|
+
// stitcher just gets one fewer frame.
|
|
725
|
+
continue;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
size_t cgW = CGImageGetWidth(cgImage);
|
|
729
|
+
size_t cgH = CGImageGetHeight(cgImage);
|
|
730
|
+
|
|
731
|
+
UIImage *uiImage = [UIImage imageWithCGImage:cgImage];
|
|
732
|
+
CGImageRelease(cgImage);
|
|
733
|
+
|
|
734
|
+
NSData *jpegData = UIImageJPEGRepresentation(uiImage, compressionQuality);
|
|
735
|
+
if (jpegData == nil) {
|
|
736
|
+
NSLog(@"[stitch-bc] frame %ld/%ld: UIImageJPEGRepresentation returned nil",
|
|
737
|
+
(long)i, (long)maxFrames);
|
|
738
|
+
continue;
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
NSString *framePath =
|
|
742
|
+
[cleanedOutputDir stringByAppendingPathComponent:
|
|
743
|
+
[NSString stringWithFormat:@"frame_%03ld.jpg", (long)i]];
|
|
744
|
+
BOOL wrote = [jpegData writeToFile:framePath atomically:YES];
|
|
745
|
+
NSLog(@"[stitch-bc] frame %ld/%ld: cgImage=%zux%zu jpeg=%lu bytes wrote=%d",
|
|
746
|
+
(long)i, (long)maxFrames, cgW, cgH,
|
|
747
|
+
(unsigned long)jpegData.length, (int)wrote);
|
|
748
|
+
if (wrote) {
|
|
749
|
+
[paths addObject:framePath];
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
NSLog(@"[stitch-bc] extractFrames done: produced %lu frames",
|
|
754
|
+
(unsigned long)paths.count);
|
|
755
|
+
|
|
756
|
+
if (paths.count < 2) {
|
|
757
|
+
if (error) {
|
|
758
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
759
|
+
code:1013
|
|
760
|
+
userInfo:@{
|
|
761
|
+
NSLocalizedDescriptionKey:
|
|
762
|
+
[NSString stringWithFormat:
|
|
763
|
+
@"Extracted only %lu frames from video — need ≥ 2. "
|
|
764
|
+
"The video may be too short or the file unreadable.",
|
|
765
|
+
(unsigned long)paths.count],
|
|
766
|
+
}];
|
|
767
|
+
}
|
|
768
|
+
return nil;
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
return paths;
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
776
|
+
// Combined pipeline: video → stitched panorama
|
|
777
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
778
|
+
|
|
779
|
+
+ (nullable RNStitchResult *)stitchVideoAtPath:(NSString *)videoPath
|
|
780
|
+
outputPath:(NSString *)outputPath
|
|
781
|
+
maxFrames:(NSInteger)maxFrames
|
|
782
|
+
jpegQuality:(NSInteger)quality
|
|
783
|
+
warperType:(NSString *)warperType
|
|
784
|
+
blenderType:(NSString *)blenderType
|
|
785
|
+
seamFinderType:(NSString *)seamFinderType
|
|
786
|
+
error:(NSError **)error {
|
|
787
|
+
// Tmp dir for extracted frames — UUID'd so concurrent stitches
|
|
788
|
+
// can't clobber each other's working state.
|
|
789
|
+
NSString *tmpDir =
|
|
790
|
+
[NSTemporaryDirectory() stringByAppendingPathComponent:
|
|
791
|
+
[NSString stringWithFormat:@"RNImageStitcherStitch-%@",
|
|
792
|
+
[[NSUUID UUID] UUIDString]]];
|
|
793
|
+
|
|
794
|
+
NSError *extractErr = nil;
|
|
795
|
+
NSArray<NSString *> *frames =
|
|
796
|
+
[self extractFramesFromVideoAtPath:videoPath
|
|
797
|
+
outputDir:tmpDir
|
|
798
|
+
maxFrames:maxFrames
|
|
799
|
+
jpegQuality:quality
|
|
800
|
+
error:&extractErr];
|
|
801
|
+
if (!frames) {
|
|
802
|
+
// Best-effort cleanup — the dir may not exist if extract bailed
|
|
803
|
+
// before creating it. Ignore the error.
|
|
804
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
805
|
+
if (error) *error = extractErr;
|
|
806
|
+
return nil;
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
NSError *stitchErr = nil;
|
|
810
|
+
// AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
|
|
811
|
+
// Legacy video-driven path: no AR-frame orientation context, so
|
|
812
|
+
// we pass nil for captureOrientation → the .mm side treats nil as
|
|
813
|
+
// "portrait" → no bake-rotation. Callers wanting rotation should
|
|
814
|
+
// use the keyframe-driven Swift path which carries the orientation
|
|
815
|
+
// from the JS accelerometer hook through IncrementalStitcher.
|
|
816
|
+
RNStitchResult *result =
|
|
817
|
+
[self stitchFramePaths:frames
|
|
818
|
+
outputPath:outputPath
|
|
819
|
+
jpegQuality:quality
|
|
820
|
+
warperType:warperType
|
|
821
|
+
blenderType:blenderType
|
|
822
|
+
seamFinderType:seamFinderType
|
|
823
|
+
captureOrientation:nil
|
|
824
|
+
useInscribedRectCrop:NO
|
|
825
|
+
error:&stitchErr];
|
|
826
|
+
|
|
827
|
+
// Always tear down the tmp dir, success or fail — leaving
|
|
828
|
+
// hundreds of MB of frame JPEGs in /tmp would balloon the app's
|
|
829
|
+
// working set across panoramas.
|
|
830
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
831
|
+
|
|
832
|
+
if (!result && error) *error = stitchErr;
|
|
833
|
+
return result;
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
838
|
+
// Phase 5: pose-driven video → panorama (ARKit/ARCore)
|
|
839
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
840
|
+
//
|
|
841
|
+
// Same end-to-end shape as `stitchVideoAtPath` but consumes
|
|
842
|
+
// pre-computed camera poses (from ARKit/ARCore via the host's
|
|
843
|
+
// RNSARSession) and skips the brittle features → matching
|
|
844
|
+
// → BundleAdjuster steps that the feature-matched path runs.
|
|
845
|
+
// The compose stage (warp + seam + blend + crop) is duplicated
|
|
846
|
+
// from `stitchFramePaths` rather than refactored — keeps the
|
|
847
|
+
// hard-won existing pipeline untouched while we field-test the
|
|
848
|
+
// pose path; both paths can be DRY'd into a shared helper once
|
|
849
|
+
// the new code is proven on real shelf captures.
|
|
850
|
+
|
|
851
|
+
+ (nullable RNStitchResult *)stitchVideoAtPath:(NSString *)videoPath
|
|
852
|
+
outputPath:(NSString *)outputPath
|
|
853
|
+
maxFrames:(NSInteger)maxFrames
|
|
854
|
+
jpegQuality:(NSInteger)quality
|
|
855
|
+
warperType:(NSString *)warperType
|
|
856
|
+
blenderType:(NSString *)blenderType
|
|
857
|
+
seamFinderType:(NSString *)seamFinderType
|
|
858
|
+
poses:(NSArray<NSDictionary *> *)poses
|
|
859
|
+
error:(NSError **)error {
|
|
860
|
+
if (warperType == nil || warperType.length == 0) warperType = @"plane";
|
|
861
|
+
if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
|
|
862
|
+
if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
|
|
863
|
+
if (poses.count < 2) {
|
|
864
|
+
if (error) {
|
|
865
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
866
|
+
code:1030
|
|
867
|
+
userInfo:@{
|
|
868
|
+
NSLocalizedDescriptionKey:
|
|
869
|
+
@"Pose-driven stitch needs at least 2 poses; got fewer.",
|
|
870
|
+
}];
|
|
871
|
+
}
|
|
872
|
+
return nil;
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
NSString *tmpDir =
|
|
876
|
+
[NSTemporaryDirectory() stringByAppendingPathComponent:
|
|
877
|
+
[NSString stringWithFormat:@"RNImageStitcherStitchAR-%@",
|
|
878
|
+
[[NSUUID UUID] UUIDString]]];
|
|
879
|
+
|
|
880
|
+
// Extract evenly-spaced frames from the video (same helper the
|
|
881
|
+
// feature-matched path uses). Returns paths only; we'll compute
|
|
882
|
+
// each frame's timestamp ourselves to match against `poses`.
|
|
883
|
+
NSError *extractErr = nil;
|
|
884
|
+
NSArray<NSString *> *framePaths =
|
|
885
|
+
[self extractFramesFromVideoAtPath:videoPath
|
|
886
|
+
outputDir:tmpDir
|
|
887
|
+
maxFrames:maxFrames
|
|
888
|
+
jpegQuality:quality
|
|
889
|
+
error:&extractErr];
|
|
890
|
+
if (!framePaths) {
|
|
891
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
892
|
+
if (error) *error = extractErr;
|
|
893
|
+
return nil;
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
// Compute total video duration so frame timestamps match what
|
|
897
|
+
// the AR session captured. Pose timestamps are in absolute ms;
|
|
898
|
+
// we normalise against poses[0] so they align with the mp4
|
|
899
|
+
// timeline (which AVAssetWriter wrote starting at 0).
|
|
900
|
+
NSURL *videoURL = [NSURL fileURLWithPath:
|
|
901
|
+
([videoPath hasPrefix:@"file://"]
|
|
902
|
+
? [videoPath substringFromIndex:[@"file://" length]]
|
|
903
|
+
: videoPath)];
|
|
904
|
+
AVURLAsset *asset = [AVURLAsset assetWithURL:videoURL];
|
|
905
|
+
Float64 totalSeconds = CMTimeGetSeconds(asset.duration);
|
|
906
|
+
if (!isfinite(totalSeconds) || totalSeconds <= 0) {
|
|
907
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
908
|
+
if (error) {
|
|
909
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
910
|
+
code:1031
|
|
911
|
+
userInfo:@{
|
|
912
|
+
NSLocalizedDescriptionKey:
|
|
913
|
+
@"Could not read video duration for pose-time alignment.",
|
|
914
|
+
}];
|
|
915
|
+
}
|
|
916
|
+
return nil;
|
|
917
|
+
}
|
|
918
|
+
double baseMs = [poses[0][@"timestampMs"] doubleValue];
|
|
919
|
+
|
|
920
|
+
// Match each extracted frame to its closest pose by timestamp.
|
|
921
|
+
// Tolerance is 100 ms — at 60 Hz pose log + 30 fps frame extract,
|
|
922
|
+
// worst case is ~17 ms drift, plenty of headroom.
|
|
923
|
+
NSInteger N = (NSInteger)framePaths.count;
|
|
924
|
+
std::vector<cv::Mat> frames;
|
|
925
|
+
std::vector<cv::detail::CameraParams> cameras;
|
|
926
|
+
frames.reserve(N);
|
|
927
|
+
cameras.reserve(N);
|
|
928
|
+
int matched = 0, dropped = 0;
|
|
929
|
+
for (NSInteger i = 0; i < N; i++) {
|
|
930
|
+
Float64 fraction = (N == 1) ? 0.0 : ((Float64)i / (Float64)(N - 1));
|
|
931
|
+
Float64 frameTimeMs = fraction * totalSeconds * 1000.0;
|
|
932
|
+
|
|
933
|
+
NSDictionary *bestPose = nil;
|
|
934
|
+
double bestDelta = INFINITY;
|
|
935
|
+
for (NSDictionary *pose in poses) {
|
|
936
|
+
double poseMs = [pose[@"timestampMs"] doubleValue] - baseMs;
|
|
937
|
+
double delta = fabs(poseMs - frameTimeMs);
|
|
938
|
+
if (delta < bestDelta) {
|
|
939
|
+
bestDelta = delta;
|
|
940
|
+
bestPose = pose;
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
if (!bestPose || bestDelta > 100.0) {
|
|
944
|
+
dropped++;
|
|
945
|
+
continue;
|
|
946
|
+
}
|
|
947
|
+
// V16 Phase 1.fix3 — IMREAD_IGNORE_ORIENTATION parity with the
|
|
948
|
+
// batch-keyframe path. AVAssetImageGenerator writes JPEGs with
|
|
949
|
+
// EXIF Orientation tags; cv::imread defaults (OpenCV 4.5+) apply
|
|
950
|
+
// them, returning rotated pixels that don't match the pose's
|
|
951
|
+
// intrinsics (which describe the unrotated landscape sensor).
|
|
952
|
+
// Force raw landscape pixels for the stitcher.
|
|
953
|
+
cv::Mat img = cv::imread([framePaths[i] UTF8String],
|
|
954
|
+
cv::IMREAD_COLOR | cv::IMREAD_IGNORE_ORIENTATION);
|
|
955
|
+
if (img.empty()) {
|
|
956
|
+
dropped++;
|
|
957
|
+
continue;
|
|
958
|
+
}
|
|
959
|
+
frames.push_back(img);
|
|
960
|
+
cameras.push_back(cameraParamsFromPose(bestPose));
|
|
961
|
+
matched++;
|
|
962
|
+
}
|
|
963
|
+
NSLog(@"[BatchStitcher] pose-driven: matched=%d dropped=%d",
|
|
964
|
+
matched, dropped);
|
|
965
|
+
|
|
966
|
+
if (frames.size() < 2) {
|
|
967
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
968
|
+
if (error) {
|
|
969
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
970
|
+
code:1032
|
|
971
|
+
userInfo:@{
|
|
972
|
+
NSLocalizedDescriptionKey:
|
|
973
|
+
@"Fewer than 2 frames matched a pose within tolerance — "
|
|
974
|
+
"AR tracking may have been lost during the pan.",
|
|
975
|
+
}];
|
|
976
|
+
}
|
|
977
|
+
return nil;
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
auto t0 = std::chrono::steady_clock::now();
|
|
981
|
+
cv::Mat panorama;
|
|
982
|
+
|
|
983
|
+
@autoreleasepool {
|
|
984
|
+
try {
|
|
985
|
+
// Pose-driven path: cameras already populated. intrinsics are
|
|
986
|
+
// at the source frame's native resolution, so work_scale = 1.0.
|
|
987
|
+
int origCols = frames[0].cols;
|
|
988
|
+
int origRows = frames[0].rows;
|
|
989
|
+
double origMp = (double)origCols * origRows / 1e6;
|
|
990
|
+
constexpr double COMPOSE_MP = 1.0;
|
|
991
|
+
double compose_scale = (origMp > COMPOSE_MP)
|
|
992
|
+
? std::sqrt(COMPOSE_MP / origMp)
|
|
993
|
+
: 1.0;
|
|
994
|
+
double compose_work_aspect = compose_scale; // work_scale == 1
|
|
995
|
+
|
|
996
|
+
// No camera-0 normalisation in the pose-driven path.
|
|
997
|
+
//
|
|
998
|
+
// I added one previously thinking it matched cv::Stitcher's BA
|
|
999
|
+
// convention. In fact it BROKE the natural orientation: BA
|
|
1000
|
+
// normalises into a frame where camera 0's "up" is the panorama
|
|
1001
|
+
// up; for pose-driven, the cameras already live in ARKit's
|
|
1002
|
+
// gravity-aligned world (Y-up = scene up regardless of phone
|
|
1003
|
+
// orientation), so passing R values in ARKit's world frame is
|
|
1004
|
+
// exactly what cv::detail::SphericalWarper wants — it unwraps
|
|
1005
|
+
// the sphere with world's +Y as up, giving correct orientation
|
|
1006
|
+
// for any phone pose + any pan direction. Normalising rotated
|
|
1007
|
+
// the panorama 90° (the user's left-to-right pan in portrait
|
|
1008
|
+
// came out with natural-up on the side).
|
|
1009
|
+
//
|
|
1010
|
+
// waveCorrect below provides the per-camera fine alignment that
|
|
1011
|
+
// BA would have done in the feature-matched path.
|
|
1012
|
+
|
|
1013
|
+
// Optional waveCorrect — uses HORIZ to match the feature-
|
|
1014
|
+
// matched path. Operators may pan in any direction; HORIZ
|
|
1015
|
+
// aligns each camera's "up" to the world Y axis (gravity),
|
|
1016
|
+
// which is what we want for both portrait+horizontal and
|
|
1017
|
+
// landscape+vertical pans (assuming the user keeps the phone
|
|
1018
|
+
// oriented to gravity, which is the typical handheld case).
|
|
1019
|
+
std::vector<cv::Mat> rmats;
|
|
1020
|
+
rmats.reserve(cameras.size());
|
|
1021
|
+
for (const auto &cam : cameras) rmats.push_back(cam.R.clone());
|
|
1022
|
+
try {
|
|
1023
|
+
cv::detail::waveCorrect(rmats, cv::detail::WAVE_CORRECT_HORIZ);
|
|
1024
|
+
for (size_t i = 0; i < cameras.size(); i++) {
|
|
1025
|
+
cameras[i].R = rmats[i];
|
|
1026
|
+
}
|
|
1027
|
+
} catch (const cv::Exception &e) {
|
|
1028
|
+
NSLog(@"[BatchStitcher] pose: wave correction skipped: %s", e.what());
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
// Rescale intrinsics for compose-scale warping.
|
|
1032
|
+
for (auto &cam : cameras) {
|
|
1033
|
+
cam.focal *= compose_work_aspect;
|
|
1034
|
+
cam.ppx *= compose_work_aspect;
|
|
1035
|
+
cam.ppy *= compose_work_aspect;
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
std::vector<double> focals;
|
|
1039
|
+
for (const auto &cam : cameras) focals.push_back(cam.focal);
|
|
1040
|
+
std::sort(focals.begin(), focals.end());
|
|
1041
|
+
float warpedScale = focals.empty() ? 1.0f
|
|
1042
|
+
: (float)focals[focals.size() / 2];
|
|
1043
|
+
|
|
1044
|
+
cv::Ptr<cv::WarperCreator> warperCreator;
|
|
1045
|
+
if ([warperType isEqualToString:@"cylindrical"]) {
|
|
1046
|
+
warperCreator = cv::makePtr<cv::CylindricalWarper>();
|
|
1047
|
+
} else if ([warperType isEqualToString:@"spherical"]) {
|
|
1048
|
+
warperCreator = cv::makePtr<cv::SphericalWarper>();
|
|
1049
|
+
} else {
|
|
1050
|
+
warperCreator = cv::makePtr<cv::PlaneWarper>();
|
|
1051
|
+
}
|
|
1052
|
+
cv::Ptr<cv::detail::RotationWarper> warper =
|
|
1053
|
+
warperCreator->create(warpedScale);
|
|
1054
|
+
|
|
1055
|
+
// Build composeFrames at COMPOSE_MP from full-res input.
|
|
1056
|
+
std::vector<cv::Mat> composeFrames;
|
|
1057
|
+
composeFrames.reserve(frames.size());
|
|
1058
|
+
for (const auto &f : frames) {
|
|
1059
|
+
cv::Mat scaled;
|
|
1060
|
+
if (std::abs(compose_scale - 1.0) > 1e-3) {
|
|
1061
|
+
cv::resize(f, scaled, cv::Size(), compose_scale, compose_scale,
|
|
1062
|
+
cv::INTER_AREA);
|
|
1063
|
+
} else {
|
|
1064
|
+
scaled = f.clone();
|
|
1065
|
+
}
|
|
1066
|
+
composeFrames.push_back(scaled);
|
|
1067
|
+
}
|
|
1068
|
+
for (auto &f : frames) f.release();
|
|
1069
|
+
frames.clear();
|
|
1070
|
+
|
|
1071
|
+
// Build the blender (same selection logic as the feature-matched
|
|
1072
|
+
// path). The "u != 0" UMat assertion the original feature-matched
|
|
1073
|
+
// builds hit was OOM-induced; with the per-frame Mat releases
|
|
1074
|
+
// and @autoreleasepool from that path's stabilisation, MultiBand
|
|
1075
|
+
// + GraphCut are safe here too.
|
|
1076
|
+
BOOL useSeam = [seamFinderType isEqualToString:@"graphcut"];
|
|
1077
|
+
cv::Ptr<cv::detail::Blender> blender;
|
|
1078
|
+
if ([blenderType isEqualToString:@"feather"]) {
|
|
1079
|
+
blender = cv::detail::Blender::createDefault(
|
|
1080
|
+
cv::detail::Blender::FEATHER, false);
|
|
1081
|
+
auto fb = blender.dynamicCast<cv::detail::FeatherBlender>();
|
|
1082
|
+
if (fb) fb->setSharpness(0.02f);
|
|
1083
|
+
} else {
|
|
1084
|
+
blender = cv::detail::Blender::createDefault(
|
|
1085
|
+
cv::detail::Blender::MULTI_BAND, false);
|
|
1086
|
+
auto mbb = blender.dynamicCast<cv::detail::MultiBandBlender>();
|
|
1087
|
+
if (mbb) mbb->setNumBands(5);
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
if (useSeam) {
|
|
1091
|
+
const size_t M = composeFrames.size();
|
|
1092
|
+
std::vector<cv::Point> corners(M);
|
|
1093
|
+
std::vector<cv::Mat> imagesWarped(M);
|
|
1094
|
+
std::vector<cv::Mat> masksWarped(M);
|
|
1095
|
+
std::vector<cv::Size> sizes(M);
|
|
1096
|
+
for (size_t i = 0; i < M; i++) {
|
|
1097
|
+
cv::Mat K;
|
|
1098
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1099
|
+
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1100
|
+
corners[i] = warper->warp(
|
|
1101
|
+
composeFrames[i], K, cameras[i].R, cv::INTER_LINEAR,
|
|
1102
|
+
cv::BORDER_CONSTANT, imagesWarped[i]);
|
|
1103
|
+
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1104
|
+
cv::BORDER_CONSTANT, masksWarped[i]);
|
|
1105
|
+
sizes[i] = imagesWarped[i].size();
|
|
1106
|
+
}
|
|
1107
|
+
for (auto &cf : composeFrames) cf.release();
|
|
1108
|
+
composeFrames.clear();
|
|
1109
|
+
|
|
1110
|
+
// Seam finder at SEAM_MP scale (same downscale-find-upscale
|
|
1111
|
+
// pattern as the feature-matched path).
|
|
1112
|
+
const double SEAM_MP = 0.1;
|
|
1113
|
+
double seam_scale = std::min(1.0, std::sqrt(SEAM_MP / origMp));
|
|
1114
|
+
double seam_compose_aspect = seam_scale / compose_scale;
|
|
1115
|
+
std::vector<cv::UMat> imagesWarpedF_seam(M);
|
|
1116
|
+
std::vector<cv::UMat> masksWarpedU_seam(M);
|
|
1117
|
+
std::vector<cv::Point> corners_seam(M);
|
|
1118
|
+
for (size_t i = 0; i < M; i++) {
|
|
1119
|
+
cv::Mat seamImage, seamMask;
|
|
1120
|
+
cv::resize(imagesWarped[i], seamImage, cv::Size(),
|
|
1121
|
+
seam_compose_aspect, seam_compose_aspect,
|
|
1122
|
+
cv::INTER_LINEAR);
|
|
1123
|
+
cv::resize(masksWarped[i], seamMask, cv::Size(),
|
|
1124
|
+
seam_compose_aspect, seam_compose_aspect,
|
|
1125
|
+
cv::INTER_NEAREST);
|
|
1126
|
+
seamImage.convertTo(imagesWarpedF_seam[i], CV_32F);
|
|
1127
|
+
seamMask.copyTo(masksWarpedU_seam[i]);
|
|
1128
|
+
corners_seam[i] = cv::Point(
|
|
1129
|
+
cvRound(corners[i].x * seam_compose_aspect),
|
|
1130
|
+
cvRound(corners[i].y * seam_compose_aspect));
|
|
1131
|
+
}
|
|
1132
|
+
cv::Ptr<cv::detail::SeamFinder> seamFinder =
|
|
1133
|
+
cv::makePtr<cv::detail::GraphCutSeamFinder>(
|
|
1134
|
+
cv::detail::GraphCutSeamFinder::COST_COLOR);
|
|
1135
|
+
seamFinder->find(imagesWarpedF_seam, corners_seam, masksWarpedU_seam);
|
|
1136
|
+
imagesWarpedF_seam.clear();
|
|
1137
|
+
for (size_t i = 0; i < M; i++) {
|
|
1138
|
+
cv::Mat seamMaskCpu, seamMaskDilated, seamMaskFull;
|
|
1139
|
+
masksWarpedU_seam[i].copyTo(seamMaskCpu);
|
|
1140
|
+
cv::dilate(seamMaskCpu, seamMaskDilated, cv::Mat());
|
|
1141
|
+
cv::resize(seamMaskDilated, seamMaskFull,
|
|
1142
|
+
masksWarped[i].size(), 0, 0, cv::INTER_LINEAR);
|
|
1143
|
+
cv::bitwise_and(seamMaskFull, masksWarped[i], masksWarped[i]);
|
|
1144
|
+
}
|
|
1145
|
+
masksWarpedU_seam.clear();
|
|
1146
|
+
|
|
1147
|
+
blender->prepare(corners, sizes);
|
|
1148
|
+
for (size_t i = 0; i < M; i++) {
|
|
1149
|
+
cv::Mat imgS;
|
|
1150
|
+
imagesWarped[i].convertTo(imgS, CV_16S);
|
|
1151
|
+
blender->feed(imgS, masksWarped[i], corners[i]);
|
|
1152
|
+
imagesWarped[i].release();
|
|
1153
|
+
masksWarped[i].release();
|
|
1154
|
+
imgS.release();
|
|
1155
|
+
}
|
|
1156
|
+
imagesWarped.clear();
|
|
1157
|
+
masksWarped.clear();
|
|
1158
|
+
} else {
|
|
1159
|
+
// STREAM path
|
|
1160
|
+
const size_t M = composeFrames.size();
|
|
1161
|
+
std::vector<cv::Point> corners(M);
|
|
1162
|
+
std::vector<cv::Size> sizes(M);
|
|
1163
|
+
for (size_t i = 0; i < M; i++) {
|
|
1164
|
+
cv::Mat K;
|
|
1165
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1166
|
+
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1167
|
+
cv::Mat tmpMaskWarped;
|
|
1168
|
+
corners[i] = warper->warp(
|
|
1169
|
+
mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1170
|
+
cv::BORDER_CONSTANT, tmpMaskWarped);
|
|
1171
|
+
sizes[i] = tmpMaskWarped.size();
|
|
1172
|
+
}
|
|
1173
|
+
blender->prepare(corners, sizes);
|
|
1174
|
+
for (size_t i = 0; i < M; i++) {
|
|
1175
|
+
cv::Mat K;
|
|
1176
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1177
|
+
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1178
|
+
cv::Mat imgWarped, maskWarped;
|
|
1179
|
+
warper->warp(composeFrames[i], K, cameras[i].R,
|
|
1180
|
+
cv::INTER_LINEAR, cv::BORDER_CONSTANT, imgWarped);
|
|
1181
|
+
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1182
|
+
cv::BORDER_CONSTANT, maskWarped);
|
|
1183
|
+
cv::Mat imgS;
|
|
1184
|
+
imgWarped.convertTo(imgS, CV_16S);
|
|
1185
|
+
blender->feed(imgS, maskWarped, corners[i]);
|
|
1186
|
+
composeFrames[i].release();
|
|
1187
|
+
}
|
|
1188
|
+
composeFrames.clear();
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
cv::Mat panoramaS, panoramaMask;
|
|
1192
|
+
blender->blend(panoramaS, panoramaMask);
|
|
1193
|
+
panoramaS.convertTo(panorama, CV_8U);
|
|
1194
|
+
} catch (const cv::Exception &e) {
|
|
1195
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
1196
|
+
if (error) {
|
|
1197
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1198
|
+
code:1100
|
|
1199
|
+
userInfo:@{
|
|
1200
|
+
NSLocalizedDescriptionKey:
|
|
1201
|
+
[NSString stringWithFormat:
|
|
1202
|
+
@"OpenCV exception during pose-driven stitch: %s", e.what()],
|
|
1203
|
+
}];
|
|
1204
|
+
}
|
|
1205
|
+
return nil;
|
|
1206
|
+
} catch (...) {
|
|
1207
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
1208
|
+
if (error) {
|
|
1209
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1210
|
+
code:1102
|
|
1211
|
+
userInfo:@{
|
|
1212
|
+
NSLocalizedDescriptionKey:
|
|
1213
|
+
@"Unknown exception during pose-driven stitch.",
|
|
1214
|
+
}];
|
|
1215
|
+
}
|
|
1216
|
+
return nil;
|
|
1217
|
+
}
|
|
1218
|
+
} // end @autoreleasepool
|
|
1219
|
+
|
|
1220
|
+
if (panorama.empty()) {
|
|
1221
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
1222
|
+
if (error) {
|
|
1223
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1224
|
+
code:1003
|
|
1225
|
+
userInfo:@{
|
|
1226
|
+
NSLocalizedDescriptionKey:
|
|
1227
|
+
@"Pose-driven stitch produced an empty panorama.",
|
|
1228
|
+
}];
|
|
1229
|
+
}
|
|
1230
|
+
return nil;
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
// Crop to bounding box (skip the column-projection rect crop —
|
|
1234
|
+
// pose-driven stitches don't have the hourglass shape that
|
|
1235
|
+
// plane-warper feature-matched panoramas produce).
|
|
1236
|
+
cv::Mat finalImage = panorama;
|
|
1237
|
+
try {
|
|
1238
|
+
cv::Mat gray;
|
|
1239
|
+
cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
|
|
1240
|
+
cv::Mat mask;
|
|
1241
|
+
cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
|
|
1242
|
+
cv::Rect bbox = cv::boundingRect(mask);
|
|
1243
|
+
if (bbox.width > 0 && bbox.height > 0
|
|
1244
|
+
&& bbox.width <= panorama.cols && bbox.height <= panorama.rows) {
|
|
1245
|
+
finalImage = panorama(bbox).clone();
|
|
1246
|
+
}
|
|
1247
|
+
} catch (...) {
|
|
1248
|
+
finalImage = panorama;
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
auto t1 = std::chrono::steady_clock::now();
|
|
1252
|
+
double durationMs =
|
|
1253
|
+
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
|
|
1254
|
+
|
|
1255
|
+
NSInteger clampedQuality = MAX(0, MIN(100, quality));
|
|
1256
|
+
std::vector<int> params = {
|
|
1257
|
+
cv::IMWRITE_JPEG_QUALITY, static_cast<int>(clampedQuality),
|
|
1258
|
+
};
|
|
1259
|
+
NSString *cleanedOutPath = ([outputPath hasPrefix:@"file://"]
|
|
1260
|
+
? [outputPath substringFromIndex:[@"file://" length]]
|
|
1261
|
+
: outputPath);
|
|
1262
|
+
bool wrote = cv::imwrite([cleanedOutPath UTF8String], finalImage, params);
|
|
1263
|
+
|
|
1264
|
+
// Cleanup the tmp dir always.
|
|
1265
|
+
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
1266
|
+
|
|
1267
|
+
if (!wrote) {
|
|
1268
|
+
if (error) {
|
|
1269
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1270
|
+
code:1002
|
|
1271
|
+
userInfo:@{
|
|
1272
|
+
NSLocalizedDescriptionKey:
|
|
1273
|
+
[NSString stringWithFormat:
|
|
1274
|
+
@"Pose-driven stitch succeeded but could not write JPEG to %@",
|
|
1275
|
+
outputPath],
|
|
1276
|
+
}];
|
|
1277
|
+
}
|
|
1278
|
+
return nil;
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
return [[RNStitchResult alloc]
|
|
1282
|
+
initWithOutputPath:outputPath
|
|
1283
|
+
width:(NSInteger)finalImage.cols
|
|
1284
|
+
height:(NSInteger)finalImage.rows
|
|
1285
|
+
durationMs:durationMs];
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
1290
|
+
// V16 Phase 1: pose-driven stitch over explicit keyframe paths
|
|
1291
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
1292
|
+
//
|
|
1293
|
+
// Same compose stage as the video-driven pose path above, minus the
|
|
1294
|
+
// AVAssetImageGenerator extract + timestamp-matching step. Frames
|
|
1295
|
+
// arrive as already-on-disk JPEGs from the AR-keyframe capture flow;
|
|
1296
|
+
// poses are 1:1 with frames (KeyframeGate saved both as the user
|
|
1297
|
+
// panned). Compose code is duplicated per the convention noted
|
|
1298
|
+
// above ("DRY when the new path is proven on real shelf captures").
|
|
1299
|
+
//
|
|
1300
|
+
// AUDIT NOTE (2026-05-15, sibling @autoreleasepool-return audit)
|
|
1301
|
+
// ──────────────────────────────────────────────────────────────
|
|
1302
|
+
//
|
|
1303
|
+
// This method (and the pose-driven `stitchVideoAtPath:withPoses:`
|
|
1304
|
+
// variant earlier in this file at ~line 2162) BOTH have the same
|
|
1305
|
+
// @autoreleasepool-return-UAF pattern that V16 fix-10 closed in
|
|
1306
|
+
// `stitchFramePaths:` at line 597 — autoreleased NSError* assigned
|
|
1307
|
+
// to the `error` outparameter from inside an @autoreleasepool, then
|
|
1308
|
+
// the function returns, the pool drains, the NSError dangles, the
|
|
1309
|
+
// caller crashes dereferencing. See:
|
|
1310
|
+
// docs/site-content/design/2026-05-12-finalize-crash-investigation.md
|
|
1311
|
+
//
|
|
1312
|
+
// CURRENT REACHABILITY: BOTH methods are dead code as of 2026-05-15.
|
|
1313
|
+
// Confirmed by grep — only referenced in dSYM debug symbols + comments,
|
|
1314
|
+
// never actually called from Swift/Obj-C/Kotlin source paths. V16
|
|
1315
|
+
// batch-keyframe uses `stitchFramePaths:` exclusively; this method
|
|
1316
|
+
// was the earlier per-keyframe-with-pose design that was superseded.
|
|
1317
|
+
//
|
|
1318
|
+
// IF/WHEN RE-ENABLED, apply fix-10's pattern (also in this file
|
|
1319
|
+
// around `stitchFramePaths:` lines 562-571 + 1519-1527):
|
|
1320
|
+
//
|
|
1321
|
+
// NSError *capturedError = nil;
|
|
1322
|
+
// RNStitchResult *result = nil;
|
|
1323
|
+
// @autoreleasepool {
|
|
1324
|
+
// do {
|
|
1325
|
+
// try { ... ; result = [[RNStitchResult alloc] init...]; break; }
|
|
1326
|
+
// catch (cv::Exception &e) { capturedError = [NSError ...]; break; }
|
|
1327
|
+
// catch (...) { capturedError = [NSError ...]; break; }
|
|
1328
|
+
// } while (0);
|
|
1329
|
+
// }
|
|
1330
|
+
// if (capturedError) { if (error) *error = capturedError; return nil; }
|
|
1331
|
+
// return result;
|
|
1332
|
+
//
|
|
1333
|
+
// Strong locals (`capturedError`, `result`) are declared OUTSIDE the
|
|
1334
|
+
// @autoreleasepool so their refcount survives the pool drain. Both
|
|
1335
|
+
// success + failure paths exit the pool via `break` rather than
|
|
1336
|
+
// `return nil;` so the pool drains cleanly before the function
|
|
1337
|
+
// returns.
|
|
1338
|
+
//
|
|
1339
|
+
// Not applied now because the methods aren't called; risk is latent
|
|
1340
|
+
// not active. Refactoring dead code carries its own risk (subtle
|
|
1341
|
+
// behaviour changes) without active testing.
|
|
1342
|
+
|
|
1343
|
+
+ (nullable RNStitchResult *)stitchKeyframePaths:(NSArray<NSString *> *)framePaths
|
|
1344
|
+
outputPath:(NSString *)outputPath
|
|
1345
|
+
jpegQuality:(NSInteger)quality
|
|
1346
|
+
warperType:(NSString *)warperType
|
|
1347
|
+
blenderType:(NSString *)blenderType
|
|
1348
|
+
seamFinderType:(NSString *)seamFinderType
|
|
1349
|
+
poses:(NSArray<NSDictionary *> *)poses
|
|
1350
|
+
error:(NSError **)error {
|
|
1351
|
+
if (warperType == nil || warperType.length == 0) warperType = @"plane";
|
|
1352
|
+
if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
|
|
1353
|
+
if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
|
|
1354
|
+
if (framePaths.count < 2) {
|
|
1355
|
+
if (error) {
|
|
1356
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1357
|
+
code:1030
|
|
1358
|
+
userInfo:@{
|
|
1359
|
+
NSLocalizedDescriptionKey:
|
|
1360
|
+
@"Keyframe stitch needs at least 2 frames; got fewer.",
|
|
1361
|
+
}];
|
|
1362
|
+
}
|
|
1363
|
+
return nil;
|
|
1364
|
+
}
|
|
1365
|
+
if (framePaths.count != poses.count) {
|
|
1366
|
+
if (error) {
|
|
1367
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1368
|
+
code:1033
|
|
1369
|
+
userInfo:@{
|
|
1370
|
+
NSLocalizedDescriptionKey:
|
|
1371
|
+
[NSString stringWithFormat:
|
|
1372
|
+
@"Keyframe stitch requires 1:1 paths/poses; "
|
|
1373
|
+
"got %lu paths, %lu poses.",
|
|
1374
|
+
(unsigned long)framePaths.count,
|
|
1375
|
+
(unsigned long)poses.count],
|
|
1376
|
+
}];
|
|
1377
|
+
}
|
|
1378
|
+
return nil;
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
// V16 Phase 1 — memory diagnostic instrumentation. Each stage
|
|
1382
|
+
// logs phys_footprint (the metric jetsam evaluates) so we can
|
|
1383
|
+
// bisect the stage that pushed us into OS-watchdog termination.
|
|
1384
|
+
// FAULT level so iOS doesn't drop logs under burst.
|
|
1385
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1386
|
+
"[V16-stitch-mem] ENTER framePaths=%d posesCount=%d phys=%.1fMB",
|
|
1387
|
+
(int)framePaths.count, (int)poses.count, StitcherResidentMB());
|
|
1388
|
+
|
|
1389
|
+
// Load each path → cv::Mat + cameraParams. Drop any that fail
|
|
1390
|
+
// to load (corrupt JPEG, missing file) — but require ≥2 to
|
|
1391
|
+
// succeed for a panorama to be possible.
|
|
1392
|
+
//
|
|
1393
|
+
// V16 Phase 1.fix2 — IMREAD_IGNORE_ORIENTATION: collector saves
|
|
1394
|
+
// JPEGs with an EXIF Orientation tag so iOS Image renderers (e.g.
|
|
1395
|
+
// LiveFrameStrip) display correctly. cv::imread defaults (since
|
|
1396
|
+
// OpenCV 4.5+) APPLY the EXIF rotation; that would re-introduce
|
|
1397
|
+
// the image-vs-intrinsics mismatch fix1 was meant to remove. Pass
|
|
1398
|
+
// IMREAD_IGNORE_ORIENTATION explicitly to get raw landscape sensor
|
|
1399
|
+
// pixels for the stitcher.
|
|
1400
|
+
std::vector<cv::Mat> frames;
|
|
1401
|
+
std::vector<cv::detail::CameraParams> cameras;
|
|
1402
|
+
frames.reserve(framePaths.count);
|
|
1403
|
+
cameras.reserve(framePaths.count);
|
|
1404
|
+
int loaded = 0, dropped = 0;
|
|
1405
|
+
for (NSInteger i = 0; i < (NSInteger)framePaths.count; i++) {
|
|
1406
|
+
NSString *path = framePaths[i];
|
|
1407
|
+
NSString *cleaned = ([path hasPrefix:@"file://"]
|
|
1408
|
+
? [path substringFromIndex:[@"file://" length]]
|
|
1409
|
+
: path);
|
|
1410
|
+
cv::Mat img = cv::imread([cleaned UTF8String],
|
|
1411
|
+
cv::IMREAD_COLOR | cv::IMREAD_IGNORE_ORIENTATION);
|
|
1412
|
+
if (img.empty()) {
|
|
1413
|
+
dropped++;
|
|
1414
|
+
continue;
|
|
1415
|
+
}
|
|
1416
|
+
frames.push_back(img);
|
|
1417
|
+
cameras.push_back(cameraParamsFromPose(poses[i]));
|
|
1418
|
+
loaded++;
|
|
1419
|
+
}
|
|
1420
|
+
NSLog(@"[BatchStitcher] keyframe-stitch: loaded=%d dropped=%d",
|
|
1421
|
+
loaded, dropped);
|
|
1422
|
+
if (!frames.empty()) {
|
|
1423
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1424
|
+
"[V16-stitch-mem] AFTER imread N=%d size=%dx%d totalMB=%.1f phys=%.1fMB",
|
|
1425
|
+
(int)frames.size(),
|
|
1426
|
+
frames[0].cols, frames[0].rows,
|
|
1427
|
+
(double)frames.size() * frames[0].cols * frames[0].rows * 3
|
|
1428
|
+
/ (1024.0 * 1024.0),
|
|
1429
|
+
StitcherResidentMB());
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
if (frames.size() < 2) {
|
|
1433
|
+
if (error) {
|
|
1434
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1435
|
+
code:1032
|
|
1436
|
+
userInfo:@{
|
|
1437
|
+
NSLocalizedDescriptionKey:
|
|
1438
|
+
@"Fewer than 2 keyframes loaded successfully — JPEGs may "
|
|
1439
|
+
"have been corrupted or removed before stitch ran.",
|
|
1440
|
+
}];
|
|
1441
|
+
}
|
|
1442
|
+
return nil;
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
auto t0 = std::chrono::steady_clock::now();
|
|
1446
|
+
cv::Mat panorama;
|
|
1447
|
+
|
|
1448
|
+
@autoreleasepool {
|
|
1449
|
+
try {
|
|
1450
|
+
int origCols = frames[0].cols;
|
|
1451
|
+
int origRows = frames[0].rows;
|
|
1452
|
+
double origMp = (double)origCols * origRows / 1e6;
|
|
1453
|
+
constexpr double COMPOSE_MP = 1.0;
|
|
1454
|
+
double compose_scale = (origMp > COMPOSE_MP)
|
|
1455
|
+
? std::sqrt(COMPOSE_MP / origMp)
|
|
1456
|
+
: 1.0;
|
|
1457
|
+
double compose_work_aspect = compose_scale; // work_scale == 1
|
|
1458
|
+
|
|
1459
|
+
// V16 Phase 1.fix2 — auto-detect pan axis from camera rotation
|
|
1460
|
+
// spread. Compute the std-dev of camera "forward" vectors
|
|
1461
|
+
// projected onto each world axis; the axis with the smallest
|
|
1462
|
+
// spread is the pan-rotation axis (i.e. rotation about that
|
|
1463
|
+
// axis is what differs across frames most). HORIZ_PAN means
|
|
1464
|
+
// rotation about world Y (yaw): use WAVE_CORRECT_HORIZ.
|
|
1465
|
+
// VERT_PAN means rotation about world X (pitch): use WAVE_CORRECT_VERT.
|
|
1466
|
+
//
|
|
1467
|
+
// Earlier hardcoded HORIZ produced misaligned panoramas for
|
|
1468
|
+
// Ram's top-to-bottom landscape pan (no yaw spread; pitch
|
|
1469
|
+
// spread). Picking the right axis lets waveCorrect actually
|
|
1470
|
+
// help instead of being a no-op (or flipping the panorama).
|
|
1471
|
+
cv::detail::WaveCorrectKind waveKind = cv::detail::WAVE_CORRECT_HORIZ;
|
|
1472
|
+
if (cameras.size() >= 2) {
|
|
1473
|
+
// forward[i] = -3rd-column of R (camera looks along -Z in cv)
|
|
1474
|
+
double minF[3] = { 1e9, 1e9, 1e9};
|
|
1475
|
+
double maxF[3] = {-1e9,-1e9,-1e9};
|
|
1476
|
+
for (const auto &cam : cameras) {
|
|
1477
|
+
for (int axis = 0; axis < 3; axis++) {
|
|
1478
|
+
double v = -cam.R.at<float>(2, axis);
|
|
1479
|
+
if (v < minF[axis]) minF[axis] = v;
|
|
1480
|
+
if (v > maxF[axis]) maxF[axis] = v;
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
double rangeX = maxF[0] - minF[0];
|
|
1484
|
+
double rangeY = maxF[1] - minF[1];
|
|
1485
|
+
// Larger Y-range of forward => more vertical (pitch) variation
|
|
1486
|
+
// => vertical pan => WAVE_CORRECT_VERT.
|
|
1487
|
+
if (rangeY > rangeX) {
|
|
1488
|
+
waveKind = cv::detail::WAVE_CORRECT_VERT;
|
|
1489
|
+
}
|
|
1490
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1491
|
+
"[V16-stitch-mem] waveKind=%{public}s "
|
|
1492
|
+
"rangeForwardX=%.3f rangeForwardY=%.3f",
|
|
1493
|
+
(waveKind == cv::detail::WAVE_CORRECT_VERT)
|
|
1494
|
+
? "VERT (vertical pan)"
|
|
1495
|
+
: "HORIZ (horizontal pan)",
|
|
1496
|
+
rangeX, rangeY);
|
|
1497
|
+
}
|
|
1498
|
+
std::vector<cv::Mat> rmats;
|
|
1499
|
+
rmats.reserve(cameras.size());
|
|
1500
|
+
for (const auto &cam : cameras) rmats.push_back(cam.R.clone());
|
|
1501
|
+
try {
|
|
1502
|
+
cv::detail::waveCorrect(rmats, waveKind);
|
|
1503
|
+
for (size_t i = 0; i < cameras.size(); i++) {
|
|
1504
|
+
cameras[i].R = rmats[i];
|
|
1505
|
+
}
|
|
1506
|
+
} catch (const cv::Exception &e) {
|
|
1507
|
+
NSLog(@"[BatchStitcher] keyframe: wave correction skipped: %s",
|
|
1508
|
+
e.what());
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
// Rescale intrinsics for compose-scale warping.
|
|
1512
|
+
for (auto &cam : cameras) {
|
|
1513
|
+
cam.focal *= compose_work_aspect;
|
|
1514
|
+
cam.ppx *= compose_work_aspect;
|
|
1515
|
+
cam.ppy *= compose_work_aspect;
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
std::vector<double> focals;
|
|
1519
|
+
for (const auto &cam : cameras) focals.push_back(cam.focal);
|
|
1520
|
+
std::sort(focals.begin(), focals.end());
|
|
1521
|
+
float warpedScale = focals.empty() ? 1.0f
|
|
1522
|
+
: (float)focals[focals.size() / 2];
|
|
1523
|
+
|
|
1524
|
+
cv::Ptr<cv::WarperCreator> warperCreator;
|
|
1525
|
+
if ([warperType isEqualToString:@"cylindrical"]) {
|
|
1526
|
+
warperCreator = cv::makePtr<cv::CylindricalWarper>();
|
|
1527
|
+
} else if ([warperType isEqualToString:@"spherical"]) {
|
|
1528
|
+
warperCreator = cv::makePtr<cv::SphericalWarper>();
|
|
1529
|
+
} else {
|
|
1530
|
+
warperCreator = cv::makePtr<cv::PlaneWarper>();
|
|
1531
|
+
}
|
|
1532
|
+
cv::Ptr<cv::detail::RotationWarper> warper =
|
|
1533
|
+
warperCreator->create(warpedScale);
|
|
1534
|
+
|
|
1535
|
+
// Build composeFrames at COMPOSE_MP from full-res input.
|
|
1536
|
+
std::vector<cv::Mat> composeFrames;
|
|
1537
|
+
composeFrames.reserve(frames.size());
|
|
1538
|
+
for (const auto &f : frames) {
|
|
1539
|
+
cv::Mat scaled;
|
|
1540
|
+
if (std::abs(compose_scale - 1.0) > 1e-3) {
|
|
1541
|
+
cv::resize(f, scaled, cv::Size(), compose_scale, compose_scale,
|
|
1542
|
+
cv::INTER_AREA);
|
|
1543
|
+
} else {
|
|
1544
|
+
scaled = f.clone();
|
|
1545
|
+
}
|
|
1546
|
+
composeFrames.push_back(scaled);
|
|
1547
|
+
}
|
|
1548
|
+
for (auto &f : frames) f.release();
|
|
1549
|
+
frames.clear();
|
|
1550
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1551
|
+
"[V16-stitch-mem] AFTER composeFrames built+frames cleared "
|
|
1552
|
+
"compose_scale=%.3f compose_size=%dx%d phys=%.1fMB",
|
|
1553
|
+
compose_scale,
|
|
1554
|
+
composeFrames.empty() ? 0 : composeFrames[0].cols,
|
|
1555
|
+
composeFrames.empty() ? 0 : composeFrames[0].rows,
|
|
1556
|
+
StitcherResidentMB());
|
|
1557
|
+
|
|
1558
|
+
BOOL useSeam = [seamFinderType isEqualToString:@"graphcut"];
|
|
1559
|
+
cv::Ptr<cv::detail::Blender> blender;
|
|
1560
|
+
if ([blenderType isEqualToString:@"feather"]) {
|
|
1561
|
+
blender = cv::detail::Blender::createDefault(
|
|
1562
|
+
cv::detail::Blender::FEATHER, false);
|
|
1563
|
+
auto fb = blender.dynamicCast<cv::detail::FeatherBlender>();
|
|
1564
|
+
if (fb) fb->setSharpness(0.02f);
|
|
1565
|
+
} else {
|
|
1566
|
+
blender = cv::detail::Blender::createDefault(
|
|
1567
|
+
cv::detail::Blender::MULTI_BAND, false);
|
|
1568
|
+
auto mbb = blender.dynamicCast<cv::detail::MultiBandBlender>();
|
|
1569
|
+
if (mbb) mbb->setNumBands(5);
|
|
1570
|
+
}
|
|
1571
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1572
|
+
"[V16-stitch-mem] config blender=%{public}@ seam=%{public}@ warper=%{public}@ phys=%.1fMB",
|
|
1573
|
+
blenderType, seamFinderType, warperType, StitcherResidentMB());
|
|
1574
|
+
|
|
1575
|
+
if (useSeam) {
|
|
1576
|
+
const size_t M = composeFrames.size();
|
|
1577
|
+
std::vector<cv::Point> corners(M);
|
|
1578
|
+
std::vector<cv::Mat> imagesWarped(M);
|
|
1579
|
+
std::vector<cv::Mat> masksWarped(M);
|
|
1580
|
+
std::vector<cv::Size> sizes(M);
|
|
1581
|
+
for (size_t i = 0; i < M; i++) {
|
|
1582
|
+
cv::Mat K;
|
|
1583
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1584
|
+
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1585
|
+
corners[i] = warper->warp(
|
|
1586
|
+
composeFrames[i], K, cameras[i].R, cv::INTER_LINEAR,
|
|
1587
|
+
cv::BORDER_CONSTANT, imagesWarped[i]);
|
|
1588
|
+
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1589
|
+
cv::BORDER_CONSTANT, masksWarped[i]);
|
|
1590
|
+
sizes[i] = imagesWarped[i].size();
|
|
1591
|
+
}
|
|
1592
|
+
// Compute panorama bbox so we can see if the warped span is
|
|
1593
|
+
// unexpectedly large (drives MultiBand pyramid memory).
|
|
1594
|
+
int minX = INT_MAX, minY = INT_MAX, maxX = INT_MIN, maxY = INT_MIN;
|
|
1595
|
+
for (size_t i = 0; i < M; i++) {
|
|
1596
|
+
minX = std::min(minX, corners[i].x);
|
|
1597
|
+
minY = std::min(minY, corners[i].y);
|
|
1598
|
+
maxX = std::max(maxX, corners[i].x + (int)sizes[i].width);
|
|
1599
|
+
maxY = std::max(maxY, corners[i].y + (int)sizes[i].height);
|
|
1600
|
+
}
|
|
1601
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1602
|
+
"[V16-stitch-mem] AFTER warps M=%d bbox=%dx%d "
|
|
1603
|
+
"warpedTotalMB=%.1f phys=%.1fMB",
|
|
1604
|
+
(int)M,
|
|
1605
|
+
(maxX > minX ? maxX - minX : 0),
|
|
1606
|
+
(maxY > minY ? maxY - minY : 0),
|
|
1607
|
+
(double)M * (M ? sizes[0].width : 0)
|
|
1608
|
+
* (M ? sizes[0].height : 0) * 3 / (1024.0 * 1024.0),
|
|
1609
|
+
StitcherResidentMB());
|
|
1610
|
+
const int panBboxW = (maxX > minX ? maxX - minX : 0);
|
|
1611
|
+
const int panBboxH = (maxY > minY ? maxY - minY : 0);
|
|
1612
|
+
// Quiet `unused variable` warnings if the inner os_log calls
|
|
1613
|
+
// are stripped by the compiler in release builds.
|
|
1614
|
+
(void)panBboxW; (void)panBboxH;
|
|
1615
|
+
for (auto &cf : composeFrames) cf.release();
|
|
1616
|
+
composeFrames.clear();
|
|
1617
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1618
|
+
"[V16-stitch-mem] AFTER composeFrames cleared (warps held) phys=%.1fMB",
|
|
1619
|
+
StitcherResidentMB());
|
|
1620
|
+
|
|
1621
|
+
const double SEAM_MP = 0.1;
|
|
1622
|
+
double seam_scale = std::min(1.0, std::sqrt(SEAM_MP / origMp));
|
|
1623
|
+
double seam_compose_aspect = seam_scale / compose_scale;
|
|
1624
|
+
std::vector<cv::UMat> imagesWarpedF_seam(M);
|
|
1625
|
+
std::vector<cv::UMat> masksWarpedU_seam(M);
|
|
1626
|
+
std::vector<cv::Point> corners_seam(M);
|
|
1627
|
+
for (size_t i = 0; i < M; i++) {
|
|
1628
|
+
cv::Mat seamImage, seamMask;
|
|
1629
|
+
cv::resize(imagesWarped[i], seamImage, cv::Size(),
|
|
1630
|
+
seam_compose_aspect, seam_compose_aspect,
|
|
1631
|
+
cv::INTER_LINEAR);
|
|
1632
|
+
cv::resize(masksWarped[i], seamMask, cv::Size(),
|
|
1633
|
+
seam_compose_aspect, seam_compose_aspect,
|
|
1634
|
+
cv::INTER_NEAREST);
|
|
1635
|
+
seamImage.convertTo(imagesWarpedF_seam[i], CV_32F);
|
|
1636
|
+
seamMask.copyTo(masksWarpedU_seam[i]);
|
|
1637
|
+
corners_seam[i] = cv::Point(
|
|
1638
|
+
cvRound(corners[i].x * seam_compose_aspect),
|
|
1639
|
+
cvRound(corners[i].y * seam_compose_aspect));
|
|
1640
|
+
}
|
|
1641
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1642
|
+
"[V16-stitch-mem] BEFORE GraphCutSeamFinder seam_scale=%.3f phys=%.1fMB",
|
|
1643
|
+
seam_scale, StitcherResidentMB());
|
|
1644
|
+
cv::Ptr<cv::detail::SeamFinder> seamFinder =
|
|
1645
|
+
cv::makePtr<cv::detail::GraphCutSeamFinder>(
|
|
1646
|
+
cv::detail::GraphCutSeamFinder::COST_COLOR);
|
|
1647
|
+
seamFinder->find(imagesWarpedF_seam, corners_seam, masksWarpedU_seam);
|
|
1648
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1649
|
+
"[V16-stitch-mem] AFTER GraphCutSeamFinder phys=%.1fMB",
|
|
1650
|
+
StitcherResidentMB());
|
|
1651
|
+
imagesWarpedF_seam.clear();
|
|
1652
|
+
for (size_t i = 0; i < M; i++) {
|
|
1653
|
+
cv::Mat seamMaskCpu, seamMaskDilated, seamMaskFull;
|
|
1654
|
+
masksWarpedU_seam[i].copyTo(seamMaskCpu);
|
|
1655
|
+
cv::dilate(seamMaskCpu, seamMaskDilated, cv::Mat());
|
|
1656
|
+
cv::resize(seamMaskDilated, seamMaskFull,
|
|
1657
|
+
masksWarped[i].size(), 0, 0, cv::INTER_LINEAR);
|
|
1658
|
+
cv::bitwise_and(seamMaskFull, masksWarped[i], masksWarped[i]);
|
|
1659
|
+
}
|
|
1660
|
+
masksWarpedU_seam.clear();
|
|
1661
|
+
|
|
1662
|
+
blender->prepare(corners, sizes);
|
|
1663
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1664
|
+
"[V16-stitch-mem] AFTER blender->prepare() phys=%.1fMB",
|
|
1665
|
+
StitcherResidentMB());
|
|
1666
|
+
for (size_t i = 0; i < M; i++) {
|
|
1667
|
+
cv::Mat imgS;
|
|
1668
|
+
imagesWarped[i].convertTo(imgS, CV_16S);
|
|
1669
|
+
blender->feed(imgS, masksWarped[i], corners[i]);
|
|
1670
|
+
imagesWarped[i].release();
|
|
1671
|
+
masksWarped[i].release();
|
|
1672
|
+
imgS.release();
|
|
1673
|
+
}
|
|
1674
|
+
imagesWarped.clear();
|
|
1675
|
+
masksWarped.clear();
|
|
1676
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1677
|
+
"[V16-stitch-mem] AFTER blender->feed() loop (graphcut) phys=%.1fMB",
|
|
1678
|
+
StitcherResidentMB());
|
|
1679
|
+
} else {
|
|
1680
|
+
// STREAM path
|
|
1681
|
+
const size_t M = composeFrames.size();
|
|
1682
|
+
std::vector<cv::Point> corners(M);
|
|
1683
|
+
std::vector<cv::Size> sizes(M);
|
|
1684
|
+
for (size_t i = 0; i < M; i++) {
|
|
1685
|
+
cv::Mat K;
|
|
1686
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1687
|
+
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1688
|
+
cv::Mat tmpMaskWarped;
|
|
1689
|
+
corners[i] = warper->warp(
|
|
1690
|
+
mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1691
|
+
cv::BORDER_CONSTANT, tmpMaskWarped);
|
|
1692
|
+
sizes[i] = tmpMaskWarped.size();
|
|
1693
|
+
}
|
|
1694
|
+
blender->prepare(corners, sizes);
|
|
1695
|
+
for (size_t i = 0; i < M; i++) {
|
|
1696
|
+
cv::Mat K;
|
|
1697
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1698
|
+
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1699
|
+
cv::Mat imgWarped, maskWarped;
|
|
1700
|
+
warper->warp(composeFrames[i], K, cameras[i].R,
|
|
1701
|
+
cv::INTER_LINEAR, cv::BORDER_CONSTANT, imgWarped);
|
|
1702
|
+
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1703
|
+
cv::BORDER_CONSTANT, maskWarped);
|
|
1704
|
+
cv::Mat imgS;
|
|
1705
|
+
imgWarped.convertTo(imgS, CV_16S);
|
|
1706
|
+
blender->feed(imgS, maskWarped, corners[i]);
|
|
1707
|
+
composeFrames[i].release();
|
|
1708
|
+
}
|
|
1709
|
+
composeFrames.clear();
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1713
|
+
"[V16-stitch-mem] BEFORE blender->blend() phys=%.1fMB",
|
|
1714
|
+
StitcherResidentMB());
|
|
1715
|
+
cv::Mat panoramaS, panoramaMask;
|
|
1716
|
+
blender->blend(panoramaS, panoramaMask);
|
|
1717
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1718
|
+
"[V16-stitch-mem] AFTER blender->blend() panorama=%dx%d phys=%.1fMB",
|
|
1719
|
+
panoramaS.cols, panoramaS.rows, StitcherResidentMB());
|
|
1720
|
+
panoramaS.convertTo(panorama, CV_8U);
|
|
1721
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1722
|
+
"[V16-stitch-mem] AFTER 16S->8U convert phys=%.1fMB",
|
|
1723
|
+
StitcherResidentMB());
|
|
1724
|
+
} catch (const cv::Exception &e) {
|
|
1725
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1726
|
+
"[V16-stitch-mem] cv::Exception: %{public}s phys=%.1fMB",
|
|
1727
|
+
e.what(), StitcherResidentMB());
|
|
1728
|
+
if (error) {
|
|
1729
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1730
|
+
code:1100
|
|
1731
|
+
userInfo:@{
|
|
1732
|
+
NSLocalizedDescriptionKey:
|
|
1733
|
+
[NSString stringWithFormat:
|
|
1734
|
+
@"OpenCV exception during keyframe stitch: %s", e.what()],
|
|
1735
|
+
}];
|
|
1736
|
+
}
|
|
1737
|
+
return nil;
|
|
1738
|
+
} catch (...) {
|
|
1739
|
+
if (error) {
|
|
1740
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1741
|
+
code:1102
|
|
1742
|
+
userInfo:@{
|
|
1743
|
+
NSLocalizedDescriptionKey:
|
|
1744
|
+
@"Unknown exception during keyframe stitch.",
|
|
1745
|
+
}];
|
|
1746
|
+
}
|
|
1747
|
+
return nil;
|
|
1748
|
+
}
|
|
1749
|
+
} // end @autoreleasepool
|
|
1750
|
+
|
|
1751
|
+
if (panorama.empty()) {
|
|
1752
|
+
if (error) {
|
|
1753
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1754
|
+
code:1003
|
|
1755
|
+
userInfo:@{
|
|
1756
|
+
NSLocalizedDescriptionKey:
|
|
1757
|
+
@"Keyframe stitch produced an empty panorama.",
|
|
1758
|
+
}];
|
|
1759
|
+
}
|
|
1760
|
+
return nil;
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
// Crop to bounding box.
|
|
1764
|
+
cv::Mat finalImage = panorama;
|
|
1765
|
+
try {
|
|
1766
|
+
cv::Mat gray;
|
|
1767
|
+
cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
|
|
1768
|
+
cv::Mat mask;
|
|
1769
|
+
cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
|
|
1770
|
+
cv::Rect bbox = cv::boundingRect(mask);
|
|
1771
|
+
if (bbox.width > 0 && bbox.height > 0
|
|
1772
|
+
&& bbox.width <= panorama.cols && bbox.height <= panorama.rows) {
|
|
1773
|
+
finalImage = panorama(bbox).clone();
|
|
1774
|
+
}
|
|
1775
|
+
} catch (...) {
|
|
1776
|
+
finalImage = panorama;
|
|
1777
|
+
}
|
|
1778
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1779
|
+
"[V16-stitch-mem] AFTER crop final=%dx%d phys=%.1fMB",
|
|
1780
|
+
finalImage.cols, finalImage.rows, StitcherResidentMB());
|
|
1781
|
+
|
|
1782
|
+
auto t1 = std::chrono::steady_clock::now();
|
|
1783
|
+
double durationMs =
|
|
1784
|
+
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
|
|
1785
|
+
|
|
1786
|
+
NSInteger clampedQuality = MAX(0, MIN(100, quality));
|
|
1787
|
+
std::vector<int> params = {
|
|
1788
|
+
cv::IMWRITE_JPEG_QUALITY, static_cast<int>(clampedQuality),
|
|
1789
|
+
};
|
|
1790
|
+
NSString *cleanedOutPath = ([outputPath hasPrefix:@"file://"]
|
|
1791
|
+
? [outputPath substringFromIndex:[@"file://" length]]
|
|
1792
|
+
: outputPath);
|
|
1793
|
+
bool wrote = cv::imwrite([cleanedOutPath UTF8String], finalImage, params);
|
|
1794
|
+
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1795
|
+
"[V16-stitch-mem] AFTER cv::imwrite ok=%d total=%.0fms phys=%.1fMB",
|
|
1796
|
+
wrote ? 1 : 0, durationMs, StitcherResidentMB());
|
|
1797
|
+
|
|
1798
|
+
if (!wrote) {
|
|
1799
|
+
if (error) {
|
|
1800
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1801
|
+
code:1002
|
|
1802
|
+
userInfo:@{
|
|
1803
|
+
NSLocalizedDescriptionKey:
|
|
1804
|
+
[NSString stringWithFormat:
|
|
1805
|
+
@"Keyframe stitch succeeded but could not write JPEG to %@",
|
|
1806
|
+
outputPath],
|
|
1807
|
+
}];
|
|
1808
|
+
}
|
|
1809
|
+
return nil;
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
return [[RNStitchResult alloc]
|
|
1813
|
+
initWithOutputPath:outputPath
|
|
1814
|
+
width:(NSInteger)finalImage.cols
|
|
1815
|
+
height:(NSInteger)finalImage.rows
|
|
1816
|
+
durationMs:durationMs];
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
|
|
1820
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
1821
|
+
// Photo orientation normalisation
|
|
1822
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
1823
|
+
// Round-trip through cv::imread / cv::imwrite to bake the EXIF
|
|
1824
|
+
// rotation into the pixel buffer, then write a plain JPEG with no
|
|
1825
|
+
// orientation metadata. Cheap (~ms for a typical iPhone JPEG) and
|
|
1826
|
+
// idempotent on already-normalised files.
|
|
1827
|
+
|
|
1828
|
+
+ (NSDictionary<NSString *, NSNumber *> *)normaliseImageAtPath:(NSString *)imagePath
|
|
1829
|
+
error:(NSError **)error {
|
|
1830
|
+
NSString *cleaned = normalizeImagePath(imagePath);
|
|
1831
|
+
if (![[NSFileManager defaultManager] fileExistsAtPath:cleaned]) {
|
|
1832
|
+
if (error) {
|
|
1833
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1834
|
+
code:1020
|
|
1835
|
+
userInfo:@{
|
|
1836
|
+
NSLocalizedDescriptionKey:
|
|
1837
|
+
[NSString stringWithFormat:@"Image not found: %@", imagePath],
|
|
1838
|
+
}];
|
|
1839
|
+
}
|
|
1840
|
+
return nil;
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
std::string nativePath(cleaned.UTF8String);
|
|
1844
|
+
cv::Mat img = cv::imread(nativePath, cv::IMREAD_COLOR);
|
|
1845
|
+
if (img.empty()) {
|
|
1846
|
+
if (error) {
|
|
1847
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1848
|
+
code:1021
|
|
1849
|
+
userInfo:@{
|
|
1850
|
+
NSLocalizedDescriptionKey:
|
|
1851
|
+
[NSString stringWithFormat:@"Could not decode image at %@", imagePath],
|
|
1852
|
+
}];
|
|
1853
|
+
}
|
|
1854
|
+
return nil;
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
std::vector<int> writeParams = {
|
|
1858
|
+
cv::IMWRITE_JPEG_QUALITY, 92,
|
|
1859
|
+
};
|
|
1860
|
+
bool ok = cv::imwrite(nativePath, img, writeParams);
|
|
1861
|
+
if (!ok) {
|
|
1862
|
+
if (error) {
|
|
1863
|
+
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1864
|
+
code:1022
|
|
1865
|
+
userInfo:@{
|
|
1866
|
+
NSLocalizedDescriptionKey:
|
|
1867
|
+
[NSString stringWithFormat:
|
|
1868
|
+
@"Could not rewrite image at %@", imagePath],
|
|
1869
|
+
}];
|
|
1870
|
+
}
|
|
1871
|
+
return nil;
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
return @{
|
|
1875
|
+
@"width": @((NSInteger)img.cols),
|
|
1876
|
+
@"height": @((NSInteger)img.rows),
|
|
1877
|
+
};
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
@end
|