react-native-image-stitcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/LICENSE +201 -0
- package/NOTICE +21 -0
- package/README.md +189 -0
- package/RNImageStitcher.podspec +76 -0
- package/android/build.gradle +224 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/cpp/CMakeLists.txt +124 -0
- package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
- package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
- package/cpp/ar_frame_pose.h +63 -0
- package/cpp/keyframe_gate.cpp +927 -0
- package/cpp/keyframe_gate.hpp +240 -0
- package/cpp/stitcher.cpp +2207 -0
- package/cpp/stitcher.hpp +275 -0
- package/dist/ar/useARSession.d.ts +102 -0
- package/dist/ar/useARSession.js +133 -0
- package/dist/camera/ARCameraView.d.ts +93 -0
- package/dist/camera/ARCameraView.js +170 -0
- package/dist/camera/Camera.d.ts +134 -0
- package/dist/camera/Camera.js +688 -0
- package/dist/camera/CameraShutter.d.ts +80 -0
- package/dist/camera/CameraShutter.js +237 -0
- package/dist/camera/CameraView.d.ts +65 -0
- package/dist/camera/CameraView.js +117 -0
- package/dist/camera/CaptureControlsBar.d.ts +87 -0
- package/dist/camera/CaptureControlsBar.js +82 -0
- package/dist/camera/CaptureHeader.d.ts +62 -0
- package/dist/camera/CaptureHeader.js +81 -0
- package/dist/camera/CapturePreview.d.ts +70 -0
- package/dist/camera/CapturePreview.js +188 -0
- package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
- package/dist/camera/CaptureStatusOverlay.js +326 -0
- package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
- package/dist/camera/CaptureThumbnailStrip.js +177 -0
- package/dist/camera/IncrementalPanGuide.d.ts +83 -0
- package/dist/camera/IncrementalPanGuide.js +267 -0
- package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
- package/dist/camera/PanoramaBandOverlay.js +399 -0
- package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
- package/dist/camera/PanoramaConfirmModal.js +128 -0
- package/dist/camera/PanoramaGuidance.d.ts +79 -0
- package/dist/camera/PanoramaGuidance.js +246 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
- package/dist/camera/PanoramaSettingsModal.js +611 -0
- package/dist/camera/ViewportCropOverlay.d.ts +46 -0
- package/dist/camera/ViewportCropOverlay.js +67 -0
- package/dist/camera/useCapture.d.ts +111 -0
- package/dist/camera/useCapture.js +160 -0
- package/dist/camera/useDeviceOrientation.d.ts +48 -0
- package/dist/camera/useDeviceOrientation.js +131 -0
- package/dist/camera/useVideoCapture.d.ts +79 -0
- package/dist/camera/useVideoCapture.js +151 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.js +39 -0
- package/dist/quality/normaliseOrientation.d.ts +36 -0
- package/dist/quality/normaliseOrientation.js +62 -0
- package/dist/quality/runQualityCheck.d.ts +41 -0
- package/dist/quality/runQualityCheck.js +98 -0
- package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
- package/dist/sensors/useIMUTranslationGate.js +235 -0
- package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
- package/dist/stitching/IncrementalStitcherView.js +157 -0
- package/dist/stitching/incremental.d.ts +930 -0
- package/dist/stitching/incremental.js +133 -0
- package/dist/stitching/stitchFrames.d.ts +55 -0
- package/dist/stitching/stitchFrames.js +56 -0
- package/dist/stitching/stitchVideo.d.ts +119 -0
- package/dist/stitching/stitchVideo.js +57 -0
- package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
- package/dist/stitching/useIncrementalJSDriver.js +199 -0
- package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
- package/dist/stitching/useIncrementalStitcher.js +172 -0
- package/dist/types.d.ts +58 -0
- package/dist/types.js +15 -0
- package/ios/Package.swift +72 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
- package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
- package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
- package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
- package/package.json +73 -0
- package/react-native.config.js +34 -0
- package/scripts/opencv-version.txt +1 -0
- package/scripts/postinstall-fetch-binaries.js +286 -0
- package/src/ar/useARSession.ts +210 -0
- package/src/camera/.gitkeep +0 -0
- package/src/camera/ARCameraView.tsx +256 -0
- package/src/camera/Camera.tsx +1053 -0
- package/src/camera/CameraShutter.tsx +292 -0
- package/src/camera/CameraView.tsx +157 -0
- package/src/camera/CaptureControlsBar.tsx +204 -0
- package/src/camera/CaptureHeader.tsx +184 -0
- package/src/camera/CapturePreview.tsx +318 -0
- package/src/camera/CaptureStatusOverlay.tsx +391 -0
- package/src/camera/CaptureThumbnailStrip.tsx +277 -0
- package/src/camera/IncrementalPanGuide.tsx +328 -0
- package/src/camera/PanoramaBandOverlay.tsx +498 -0
- package/src/camera/PanoramaConfirmModal.tsx +206 -0
- package/src/camera/PanoramaGuidance.tsx +327 -0
- package/src/camera/PanoramaSettingsModal.tsx +1357 -0
- package/src/camera/ViewportCropOverlay.tsx +81 -0
- package/src/camera/useCapture.ts +279 -0
- package/src/camera/useDeviceOrientation.ts +140 -0
- package/src/camera/useVideoCapture.ts +236 -0
- package/src/index.ts +53 -0
- package/src/quality/.gitkeep +0 -0
- package/src/quality/normaliseOrientation.ts +79 -0
- package/src/quality/runQualityCheck.ts +131 -0
- package/src/sensors/useIMUTranslationGate.ts +347 -0
- package/src/stitching/.gitkeep +0 -0
- package/src/stitching/IncrementalStitcherView.tsx +198 -0
- package/src/stitching/incremental.ts +1021 -0
- package/src/stitching/stitchFrames.ts +88 -0
- package/src/stitching/stitchVideo.ts +153 -0
- package/src/stitching/useIncrementalJSDriver.ts +273 -0
- package/src/stitching/useIncrementalStitcher.ts +252 -0
- package/src/types.ts +78 -0
package/cpp/stitcher.cpp
ADDED
|
@@ -0,0 +1,2207 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// stitcher.cpp — shared cv::Stitcher orchestration. See stitcher.hpp
|
|
4
|
+
// for design rationale.
|
|
5
|
+
//
|
|
6
|
+
// V1 (2026-05-15): ported from image_stitcher_jni.cpp (Android JNI
|
|
7
|
+
// shim) verbatim with the platform-specific Obj-C / JNI marshalling
|
|
8
|
+
// stripped. Both platforms now call this through thin bridges.
|
|
9
|
+
//
|
|
10
|
+
// V2 (planned): port iOS's manual cv::detail::* pipeline features
|
|
11
|
+
// (explicit leaveBiggestComponent at a SEPARATE retry granularity,
|
|
12
|
+
// wave correction, exposure compensator) so iOS doesn't regress
|
|
13
|
+
// from where OpenCVStitcher.mm had it. Selectable via a future
|
|
14
|
+
// StitchConfig::useManualPipeline flag.
|
|
15
|
+
|
|
16
|
+
#include "stitcher.hpp"
|
|
17
|
+
|
|
18
|
+
#include <opencv2/core.hpp>
|
|
19
|
+
#include <opencv2/features2d.hpp>
|
|
20
|
+
#include <opencv2/imgcodecs.hpp>
|
|
21
|
+
#include <opencv2/imgproc.hpp>
|
|
22
|
+
#include <opencv2/stitching.hpp>
|
|
23
|
+
#include <opencv2/stitching/detail/blenders.hpp>
|
|
24
|
+
#include <opencv2/stitching/detail/camera.hpp>
|
|
25
|
+
#include <opencv2/stitching/detail/matchers.hpp>
|
|
26
|
+
#include <opencv2/stitching/detail/motion_estimators.hpp>
|
|
27
|
+
#include <opencv2/stitching/detail/seam_finders.hpp>
|
|
28
|
+
#include <opencv2/stitching/detail/warpers.hpp>
|
|
29
|
+
#include <opencv2/stitching/warpers.hpp>
|
|
30
|
+
|
|
31
|
+
#include <algorithm>
|
|
32
|
+
#include <chrono>
|
|
33
|
+
#include <cfloat>
|
|
34
|
+
#include <cmath>
|
|
35
|
+
#include <cstdarg>
|
|
36
|
+
#include <cstdio>
|
|
37
|
+
#include <cstring>
|
|
38
|
+
#include <string>
|
|
39
|
+
#include <unistd.h>
|
|
40
|
+
#include <vector>
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
namespace retailens {
|
|
44
|
+
|
|
45
|
+
namespace {
|
|
46
|
+
|
|
47
|
+
// Lightweight logging helper. When logFn is null, the message is
|
|
48
|
+
// dropped (no allocation past the snprintf temp buffer).
|
|
49
|
+
void log_info(const LogFn& logFn, const char* tag, const char* fmt, ...) {
|
|
50
|
+
if (!logFn) return;
|
|
51
|
+
char buf[1024];
|
|
52
|
+
va_list ap;
|
|
53
|
+
va_start(ap, fmt);
|
|
54
|
+
std::vsnprintf(buf, sizeof(buf), fmt, ap);
|
|
55
|
+
va_end(ap);
|
|
56
|
+
logFn(0, tag, buf);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
void log_error(const LogFn& logFn, const char* tag, const char* fmt, ...) {
|
|
60
|
+
if (!logFn) return;
|
|
61
|
+
char buf[1024];
|
|
62
|
+
va_list ap;
|
|
63
|
+
va_start(ap, fmt);
|
|
64
|
+
std::vsnprintf(buf, sizeof(buf), fmt, ap);
|
|
65
|
+
va_end(ap);
|
|
66
|
+
logFn(2, tag, buf);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Read /proc/self/statm to get RSS in MB. Cheap (~20 µs). Used at
|
|
70
|
+
// pipeline phase boundaries to correlate logged peak memory with the
|
|
71
|
+
// staging-resolution + retry decisions. Returns -1 on read failure
|
|
72
|
+
// (e.g., procfs not mounted — never happens on iOS/Android but we
|
|
73
|
+
// guard for portability).
|
|
74
|
+
double rss_mb() {
|
|
75
|
+
FILE* f = std::fopen("/proc/self/statm", "r");
|
|
76
|
+
if (f == nullptr) return -1.0;
|
|
77
|
+
long size_pages = 0, resident_pages = 0;
|
|
78
|
+
int n = std::fscanf(f, "%ld %ld", &size_pages, &resident_pages);
|
|
79
|
+
std::fclose(f);
|
|
80
|
+
if (n != 2) return -1.0;
|
|
81
|
+
long page_bytes = sysconf(_SC_PAGESIZE);
|
|
82
|
+
return (double) resident_pages * (double) page_bytes / (1024.0 * 1024.0);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
double mat_mb(const cv::Mat& m) {
|
|
86
|
+
if (m.empty()) return 0.0;
|
|
87
|
+
return (double)(m.total() * m.elemSize()) / (1024.0 * 1024.0);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Map string → cv::WarperCreator. Returns nullptr for unknown names
|
|
91
|
+
// (caller falls back to PlaneWarper for SCANS mode anyway).
|
|
92
|
+
cv::Ptr<cv::WarperCreator> make_warper(const std::string& name) {
|
|
93
|
+
if (name == "plane") return cv::makePtr<cv::PlaneWarper>();
|
|
94
|
+
if (name == "cylindrical") return cv::makePtr<cv::CylindricalWarper>();
|
|
95
|
+
if (name == "spherical") return cv::makePtr<cv::SphericalWarper>();
|
|
96
|
+
return nullptr;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
cv::Ptr<cv::detail::Blender> make_blender(const std::string& name) {
|
|
100
|
+
if (name == "feather") {
|
|
101
|
+
return cv::detail::Blender::createDefault(cv::detail::Blender::FEATHER, false);
|
|
102
|
+
}
|
|
103
|
+
return cv::detail::Blender::createDefault(cv::detail::Blender::MULTI_BAND, false);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
cv::Ptr<cv::detail::SeamFinder> make_seam_finder(const std::string& name) {
|
|
107
|
+
if (name == "skip" || name == "no") {
|
|
108
|
+
return cv::makePtr<cv::detail::NoSeamFinder>();
|
|
109
|
+
}
|
|
110
|
+
if (name == "voronoi") {
|
|
111
|
+
return cv::makePtr<cv::detail::VoronoiSeamFinder>();
|
|
112
|
+
}
|
|
113
|
+
return cv::makePtr<cv::detail::GraphCutSeamFinder>(
|
|
114
|
+
cv::detail::GraphCutSeamFinder::COST_COLOR_GRAD);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Bake an output rotation per the capture orientation. Rotation
|
|
118
|
+
// table mirrors OpenCVStitcher.mm and the previous
|
|
119
|
+
// image_stitcher_jni.cpp — kept verbatim so behaviour is unchanged.
|
|
120
|
+
cv::Mat bake_rotation(const cv::Mat& src, const std::string& orientation,
|
|
121
|
+
const LogFn& logFn) {
|
|
122
|
+
cv::Mat rotated;
|
|
123
|
+
if (orientation == "landscape-left") {
|
|
124
|
+
cv::rotate(src, rotated, cv::ROTATE_90_COUNTERCLOCKWISE);
|
|
125
|
+
log_info(logFn, "[stitch]",
|
|
126
|
+
"bake-rotated 90° CCW for landscape-left (%dx%d → %dx%d)",
|
|
127
|
+
src.cols, src.rows, rotated.cols, rotated.rows);
|
|
128
|
+
return rotated;
|
|
129
|
+
}
|
|
130
|
+
if (orientation == "landscape-right") {
|
|
131
|
+
cv::rotate(src, rotated, cv::ROTATE_90_CLOCKWISE);
|
|
132
|
+
log_info(logFn, "[stitch]",
|
|
133
|
+
"bake-rotated 90° CW for landscape-right (%dx%d → %dx%d)",
|
|
134
|
+
src.cols, src.rows, rotated.cols, rotated.rows);
|
|
135
|
+
return rotated;
|
|
136
|
+
}
|
|
137
|
+
if (orientation == "portrait-upside-down") {
|
|
138
|
+
cv::rotate(src, rotated, cv::ROTATE_180);
|
|
139
|
+
log_info(logFn, "[stitch]",
|
|
140
|
+
"bake-rotated 180° for portrait-upside-down (%dx%d)",
|
|
141
|
+
src.cols, src.rows);
|
|
142
|
+
return rotated;
|
|
143
|
+
}
|
|
144
|
+
log_info(logFn, "[stitch]", "no bake-rotation (orientation=%s, %dx%d)",
|
|
145
|
+
orientation.c_str(), src.cols, src.rows);
|
|
146
|
+
return src;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Crop the non-zero bounding rect from the stitched panorama. cv::
|
|
150
|
+
// Stitcher's compose stage leaves a black border around the warped
|
|
151
|
+
// region; we trim that here.
|
|
152
|
+
cv::Mat crop_bbox(const cv::Mat& panorama) {
|
|
153
|
+
cv::Mat gray;
|
|
154
|
+
cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
|
|
155
|
+
cv::Mat mask;
|
|
156
|
+
cv::threshold(gray, mask, 0, 255, cv::THRESH_BINARY);
|
|
157
|
+
cv::Rect bbox = cv::boundingRect(mask);
|
|
158
|
+
if (bbox.width <= 0 || bbox.height <= 0) {
|
|
159
|
+
return panorama;
|
|
160
|
+
}
|
|
161
|
+
return panorama(bbox).clone();
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Map cv::Stitcher::Status → StitchErrorCode. cv::Stitcher's enum
|
|
165
|
+
// values aren't documented as ABI-stable so we don't rely on
|
|
166
|
+
// numeric equality; switch through the named constants.
|
|
167
|
+
StitchErrorCode statusToErrorCode(cv::Stitcher::Status status) {
|
|
168
|
+
switch (status) {
|
|
169
|
+
case cv::Stitcher::OK: return StitchErrorCode::Ok;
|
|
170
|
+
case cv::Stitcher::ERR_NEED_MORE_IMGS: return StitchErrorCode::NeedMoreImages;
|
|
171
|
+
case cv::Stitcher::ERR_HOMOGRAPHY_EST_FAIL: return StitchErrorCode::HomographyEstimationFailed;
|
|
172
|
+
case cv::Stitcher::ERR_CAMERA_PARAMS_ADJUST_FAIL: return StitchErrorCode::CameraParamsAdjustFailed;
|
|
173
|
+
default: return StitchErrorCode::UnknownCvException;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// V16 Phase 1b.fix3 — find the largest axis-aligned rectangle that
|
|
178
|
+
// fits ENTIRELY inside the non-zero region of `mask` (CV_8UC1).
|
|
179
|
+
// Used to crop the post-stitch panorama tightly: the regular
|
|
180
|
+
// boundingRect of non-zero pixels still includes the black corners
|
|
181
|
+
// where the projection didn't fill; the max-inscribed rectangle
|
|
182
|
+
// excludes those entirely.
|
|
183
|
+
//
|
|
184
|
+
// Algorithm: maximum-rectangle-in-histogram swept row by row.
|
|
185
|
+
// O(W * H). For a 4-6 MP panorama on iPhone 16 Pro, completes in
|
|
186
|
+
// 30-60 ms.
|
|
187
|
+
//
|
|
188
|
+
// Returns cv::Rect(0,0,0,0) if `mask` is empty or fully zero.
|
|
189
|
+
cv::Rect maxInscribedRectFromMask(const cv::Mat& mask) {
|
|
190
|
+
if (mask.empty() || mask.type() != CV_8UC1) {
|
|
191
|
+
return cv::Rect();
|
|
192
|
+
}
|
|
193
|
+
const int H = mask.rows;
|
|
194
|
+
const int W = mask.cols;
|
|
195
|
+
|
|
196
|
+
// Per-column running heights of consecutive non-zero pixels
|
|
197
|
+
// ending at the current row.
|
|
198
|
+
std::vector<int> heights((size_t)W, 0);
|
|
199
|
+
cv::Rect bestRect(0, 0, 0, 0);
|
|
200
|
+
long long bestArea = 0;
|
|
201
|
+
|
|
202
|
+
// Reusable monotonic stack for the row's largest-rectangle-in-
|
|
203
|
+
// histogram subroutine.
|
|
204
|
+
std::vector<int> stack;
|
|
205
|
+
stack.reserve((size_t)W + 1);
|
|
206
|
+
|
|
207
|
+
for (int row = 0; row < H; ++row) {
|
|
208
|
+
const uchar* m = mask.ptr<uchar>(row);
|
|
209
|
+
for (int col = 0; col < W; ++col) {
|
|
210
|
+
heights[(size_t)col] =
|
|
211
|
+
(m[col] != 0) ? heights[(size_t)col] + 1 : 0;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// Largest rectangle in the histogram for this row.
|
|
215
|
+
stack.clear();
|
|
216
|
+
for (int col = 0; col <= W; ++col) {
|
|
217
|
+
const int h = (col == W) ? 0 : heights[(size_t)col];
|
|
218
|
+
while (!stack.empty()
|
|
219
|
+
&& heights[(size_t)stack.back()] > h) {
|
|
220
|
+
const int topIdx = stack.back();
|
|
221
|
+
stack.pop_back();
|
|
222
|
+
const int leftIdx =
|
|
223
|
+
stack.empty() ? -1 : stack.back();
|
|
224
|
+
const int width = col - leftIdx - 1;
|
|
225
|
+
const long long area =
|
|
226
|
+
(long long)heights[(size_t)topIdx]
|
|
227
|
+
* (long long)width;
|
|
228
|
+
if (area > bestArea) {
|
|
229
|
+
bestArea = area;
|
|
230
|
+
bestRect = cv::Rect(
|
|
231
|
+
leftIdx + 1,
|
|
232
|
+
row - heights[(size_t)topIdx] + 1,
|
|
233
|
+
width,
|
|
234
|
+
heights[(size_t)topIdx]
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
stack.push_back(col);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return bestRect;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
} // namespace
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
StitchResult stitchFramePaths(
|
|
248
|
+
const std::vector<std::string>& framePaths,
|
|
249
|
+
const std::string& outputPath,
|
|
250
|
+
const StitchConfig& config,
|
|
251
|
+
LogFn logFn)
|
|
252
|
+
{
|
|
253
|
+
// V2 routing — when caller opts in, hand off to the manual
|
|
254
|
+
// cv::detail::* pipeline. See stitcher.hpp::StitchConfig::
|
|
255
|
+
// useManualPipeline for the tradeoffs. Routing here keeps the
|
|
256
|
+
// call-site signature identical so existing bridges (iOS Obj-C++,
|
|
257
|
+
// Android JNI) don't need to know which path runs internally.
|
|
258
|
+
if (config.useManualPipeline) {
|
|
259
|
+
return stitchFramePathsManual(framePaths, outputPath, config, logFn);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const auto t0 = std::chrono::steady_clock::now();
|
|
263
|
+
StitchResult result;
|
|
264
|
+
result.framesRequested = static_cast<int32_t>(framePaths.size());
|
|
265
|
+
|
|
266
|
+
if (framePaths.size() < 2) {
|
|
267
|
+
result.errorCode = StitchErrorCode::InvalidArgument;
|
|
268
|
+
result.errorMessage = "Need at least 2 frames to stitch (got " +
|
|
269
|
+
std::to_string(framePaths.size()) + ")";
|
|
270
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
271
|
+
return result;
|
|
272
|
+
}
|
|
273
|
+
if (outputPath.empty()) {
|
|
274
|
+
result.errorCode = StitchErrorCode::InvalidArgument;
|
|
275
|
+
result.errorMessage = "outputPath must not be empty";
|
|
276
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
277
|
+
return result;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
log_info(logFn, "[stitch]",
|
|
281
|
+
"stitchFramePaths: frames=%zu warper=%s blender=%s seam=%s "
|
|
282
|
+
"mode=%s orientation=%s quality=%d inscribedRect=%d",
|
|
283
|
+
framePaths.size(),
|
|
284
|
+
config.warperType.c_str(),
|
|
285
|
+
config.blenderType.c_str(),
|
|
286
|
+
config.seamFinderType.c_str(),
|
|
287
|
+
config.stitchMode == StitchMode::Scans ? "scans" : "panorama",
|
|
288
|
+
config.captureOrientation.c_str(),
|
|
289
|
+
config.jpegQuality,
|
|
290
|
+
config.useInscribedRectCrop ? 1 : 0);
|
|
291
|
+
log_info(logFn, "[memstat]", "phase=entry rss=%.1f MB", rss_mb());
|
|
292
|
+
|
|
293
|
+
// ── 1. Load input frames ───────────────────────────────────────
|
|
294
|
+
std::vector<cv::Mat> images;
|
|
295
|
+
images.reserve(framePaths.size());
|
|
296
|
+
double totalInputMB = 0.0;
|
|
297
|
+
for (size_t i = 0; i < framePaths.size(); ++i) {
|
|
298
|
+
cv::Mat img = cv::imread(framePaths[i], cv::IMREAD_COLOR);
|
|
299
|
+
if (img.empty()) {
|
|
300
|
+
result.errorCode = StitchErrorCode::ImageReadFailed;
|
|
301
|
+
result.errorMessage = "Failed to load frame: " + framePaths[i];
|
|
302
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
303
|
+
return result;
|
|
304
|
+
}
|
|
305
|
+
const double mb = mat_mb(img);
|
|
306
|
+
totalInputMB += mb;
|
|
307
|
+
log_info(logFn, "[dimstat]",
|
|
308
|
+
"input[%zu] %dx%d %dch elemSize=%zu data=%.2f MB",
|
|
309
|
+
i, img.cols, img.rows, img.channels(), img.elemSize(), mb);
|
|
310
|
+
images.push_back(std::move(img));
|
|
311
|
+
}
|
|
312
|
+
log_info(logFn, "[dimstat]", "loaded %zu frames total_input_data=%.2f MB",
|
|
313
|
+
images.size(), totalInputMB);
|
|
314
|
+
log_info(logFn, "[memstat]", "phase=after_imread rss=%.1f MB", rss_mb());
|
|
315
|
+
|
|
316
|
+
// ── 2. Configure cv::Stitcher ──────────────────────────────────
|
|
317
|
+
const cv::Stitcher::Mode cvMode = (config.stitchMode == StitchMode::Scans)
|
|
318
|
+
? cv::Stitcher::SCANS : cv::Stitcher::PANORAMA;
|
|
319
|
+
cv::Ptr<cv::Stitcher> stitcher;
|
|
320
|
+
try {
|
|
321
|
+
stitcher = cv::Stitcher::create(cvMode);
|
|
322
|
+
} catch (const cv::Exception& e) {
|
|
323
|
+
result.errorCode = StitchErrorCode::UnknownCvException;
|
|
324
|
+
result.errorMessage = std::string("Stitcher::create threw: ") + e.what();
|
|
325
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
326
|
+
return result;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// Warper only applies in PANORAMA mode (SCANS hard-wires PlaneWarper
|
|
330
|
+
// internally; setting a different warper there silently breaks the
|
|
331
|
+
// affine BA's assumptions — see learning doc on pipeline coherence).
|
|
332
|
+
if (cvMode == cv::Stitcher::PANORAMA) {
|
|
333
|
+
if (auto warper = make_warper(config.warperType)) {
|
|
334
|
+
stitcher->setWarper(warper);
|
|
335
|
+
}
|
|
336
|
+
} else {
|
|
337
|
+
log_info(logFn, "[stitch]",
|
|
338
|
+
"SCANS mode: skipping setWarper (PlaneWarper hard-wired)");
|
|
339
|
+
}
|
|
340
|
+
stitcher->setBlender(make_blender(config.blenderType));
|
|
341
|
+
stitcher->setSeamFinder(make_seam_finder(config.seamFinderType));
|
|
342
|
+
|
|
343
|
+
// Resolution budgets. Negative => keep cv::Stitcher library default
|
|
344
|
+
// for registration / seam. compositingResolMP is the exception:
|
|
345
|
+
// cv::Stitcher's library default is ORIG_RESOL (-1.0 = full sensor
|
|
346
|
+
// resolution), which trivially OOMs on Android — so for the high-
|
|
347
|
+
// level entry we substitute 1.0 MP when the caller leaves the
|
|
348
|
+
// sentinel. (Manual entry uses a different fallback; see
|
|
349
|
+
// stitchFramePathsManual().)
|
|
350
|
+
if (config.registrationResolMP > 0.0) {
|
|
351
|
+
stitcher->setRegistrationResol(config.registrationResolMP);
|
|
352
|
+
}
|
|
353
|
+
if (config.seamEstimationResolMP > 0.0) {
|
|
354
|
+
stitcher->setSeamEstimationResol(config.seamEstimationResolMP);
|
|
355
|
+
}
|
|
356
|
+
const double kHighLevelComposeFallbackMP = 1.0;
|
|
357
|
+
const double composeMP = (config.compositingResolMP > 0.0)
|
|
358
|
+
? config.compositingResolMP : kHighLevelComposeFallbackMP;
|
|
359
|
+
stitcher->setCompositingResol(composeMP);
|
|
360
|
+
log_info(logFn, "[dimstat]",
|
|
361
|
+
"cv::Stitcher resol budgets (per frame, MP):"
|
|
362
|
+
" registration=%.3f seam=%.3f compositing=%.3f%s",
|
|
363
|
+
stitcher->registrationResol(),
|
|
364
|
+
stitcher->seamEstimationResol(),
|
|
365
|
+
stitcher->compositingResol(),
|
|
366
|
+
stitcher->compositingResol() < 0
|
|
367
|
+
? " (ORIG_RESOL = no downscale!)" : "");
|
|
368
|
+
|
|
369
|
+
// ── 3. Stitch with progressive-confidence retry (C+D) ──────────
|
|
370
|
+
//
|
|
371
|
+
// cv::Stitcher::leaveBiggestComponent drops frames whose pairwise
|
|
372
|
+
// confidence is below `panoConfidenceThresh`. Boundary frames
|
|
373
|
+
// (first/last 1-2) statistically fall below first. We retry
|
|
374
|
+
// with progressively lower thresholds [1.0 → 0.5 → 0.3] until
|
|
375
|
+
// every frame is retained or we hit the floor. SCANS skips the
|
|
376
|
+
// higher thresholds (its default is already 0.3).
|
|
377
|
+
log_info(logFn, "[memstat]", "phase=before_stitch rss=%.1f MB", rss_mb());
|
|
378
|
+
const double kRetryThresholds[] = {1.0, 0.5, 0.3};
|
|
379
|
+
const int kNumAttempts = sizeof(kRetryThresholds) / sizeof(double);
|
|
380
|
+
cv::Mat panorama;
|
|
381
|
+
cv::Stitcher::Status status = cv::Stitcher::ERR_NEED_MORE_IMGS;
|
|
382
|
+
int framesIncluded = 0;
|
|
383
|
+
double finalThreshold = -1.0;
|
|
384
|
+
int finalAttempt = 0;
|
|
385
|
+
for (int attempt = 0; attempt < kNumAttempts; ++attempt) {
|
|
386
|
+
const double thresh = kRetryThresholds[attempt];
|
|
387
|
+
if (cvMode == cv::Stitcher::SCANS && thresh > 0.31) continue;
|
|
388
|
+
stitcher->setPanoConfidenceThresh(thresh);
|
|
389
|
+
finalAttempt = attempt + 1;
|
|
390
|
+
finalThreshold = thresh;
|
|
391
|
+
log_info(logFn, "[stitch-retry]",
|
|
392
|
+
"attempt %d/%d panoConfidenceThresh=%.2f",
|
|
393
|
+
finalAttempt, kNumAttempts, thresh);
|
|
394
|
+
try {
|
|
395
|
+
status = stitcher->stitch(images, panorama);
|
|
396
|
+
} catch (const cv::Exception& e) {
|
|
397
|
+
result.errorCode = StitchErrorCode::UnknownCvException;
|
|
398
|
+
result.errorMessage = std::string("Stitcher::stitch threw on attempt ") +
|
|
399
|
+
std::to_string(finalAttempt) + ": " + e.what();
|
|
400
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
401
|
+
return result;
|
|
402
|
+
}
|
|
403
|
+
if (status != cv::Stitcher::OK) {
|
|
404
|
+
log_info(logFn, "[stitch-retry]",
|
|
405
|
+
"attempt %d FAILED with status=%d, trying next threshold",
|
|
406
|
+
finalAttempt, static_cast<int>(status));
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
const std::vector<int>& component = stitcher->component();
|
|
410
|
+
framesIncluded = static_cast<int>(component.size());
|
|
411
|
+
log_info(logFn, "[stitch-retry]",
|
|
412
|
+
"attempt %d OK: framesIncluded=%d of %zu (thresh=%.2f)",
|
|
413
|
+
finalAttempt, framesIncluded, framePaths.size(), thresh);
|
|
414
|
+
if (framesIncluded >= static_cast<int>(framePaths.size())) {
|
|
415
|
+
break; // all retained — done
|
|
416
|
+
}
|
|
417
|
+
if (attempt + 1 < kNumAttempts) {
|
|
418
|
+
log_info(logFn, "[stitch-retry]",
|
|
419
|
+
"%d frames dropped — retrying with lower threshold",
|
|
420
|
+
(int)framePaths.size() - framesIncluded);
|
|
421
|
+
} else {
|
|
422
|
+
log_info(logFn, "[stitch-retry]",
|
|
423
|
+
"%d frames dropped at lowest threshold %.2f — accepting result",
|
|
424
|
+
(int)framePaths.size() - framesIncluded, thresh);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
if (status != cv::Stitcher::OK) {
|
|
428
|
+
result.errorCode = statusToErrorCode(status);
|
|
429
|
+
result.errorMessage = "Stitcher::stitch failed at all " +
|
|
430
|
+
std::to_string(finalAttempt) + " thresholds, last status code " +
|
|
431
|
+
std::to_string(static_cast<int>(status));
|
|
432
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
433
|
+
return result;
|
|
434
|
+
}
|
|
435
|
+
log_info(logFn, "[dimstat]",
|
|
436
|
+
"post-stitch panorama %dx%d %dch data=%.2f MB"
|
|
437
|
+
" (framesIncluded=%d/%zu, finalThresh=%.2f, attempts=%d)",
|
|
438
|
+
panorama.cols, panorama.rows, panorama.channels(),
|
|
439
|
+
mat_mb(panorama),
|
|
440
|
+
framesIncluded, framePaths.size(), finalThreshold, finalAttempt);
|
|
441
|
+
log_info(logFn, "[memstat]", "phase=after_stitch rss=%.1f MB", rss_mb());
|
|
442
|
+
|
|
443
|
+
// ── 4. Crop to non-zero bounding rect ──────────────────────────
|
|
444
|
+
cv::Mat cropped = crop_bbox(panorama);
|
|
445
|
+
log_info(logFn, "[dimstat]",
|
|
446
|
+
"post-crop_bbox %dx%d → %dx%d data=%.2f MB (inscribedRect=%d, currently ignored)",
|
|
447
|
+
panorama.cols, panorama.rows, cropped.cols, cropped.rows,
|
|
448
|
+
mat_mb(cropped),
|
|
449
|
+
config.useInscribedRectCrop ? 1 : 0);
|
|
450
|
+
log_info(logFn, "[memstat]", "phase=after_crop rss=%.1f MB", rss_mb());
|
|
451
|
+
|
|
452
|
+
// ── 5. Bake rotation per capture orientation ───────────────────
|
|
453
|
+
cv::Mat final_image = bake_rotation(cropped, config.captureOrientation, logFn);
|
|
454
|
+
log_info(logFn, "[dimstat]",
|
|
455
|
+
"post-bake_rotation %dx%d data=%.2f MB",
|
|
456
|
+
final_image.cols, final_image.rows, mat_mb(final_image));
|
|
457
|
+
log_info(logFn, "[memstat]", "phase=after_bake_rotation rss=%.1f MB", rss_mb());
|
|
458
|
+
|
|
459
|
+
// ── 6. Write JPEG ──────────────────────────────────────────────
|
|
460
|
+
const int q = std::max(0, std::min(100, config.jpegQuality));
|
|
461
|
+
std::vector<int> params{cv::IMWRITE_JPEG_QUALITY, q};
|
|
462
|
+
bool wrote = false;
|
|
463
|
+
try {
|
|
464
|
+
wrote = cv::imwrite(outputPath, final_image, params);
|
|
465
|
+
} catch (const cv::Exception& e) {
|
|
466
|
+
result.errorCode = StitchErrorCode::ImageWriteFailed;
|
|
467
|
+
result.errorMessage = std::string("cv::imwrite threw: ") + e.what();
|
|
468
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
469
|
+
return result;
|
|
470
|
+
}
|
|
471
|
+
if (!wrote) {
|
|
472
|
+
result.errorCode = StitchErrorCode::ImageWriteFailed;
|
|
473
|
+
result.errorMessage = "cv::imwrite returned false (path=" + outputPath + ")";
|
|
474
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
475
|
+
return result;
|
|
476
|
+
}
|
|
477
|
+
log_info(logFn, "[stitch]",
|
|
478
|
+
"output written: %s (%dx%d)",
|
|
479
|
+
outputPath.c_str(), final_image.cols, final_image.rows);
|
|
480
|
+
log_info(logFn, "[memstat]", "phase=after_imwrite rss=%.1f MB", rss_mb());
|
|
481
|
+
|
|
482
|
+
// ── 7. Fill the result ─────────────────────────────────────────
|
|
483
|
+
const auto t1 = std::chrono::steady_clock::now();
|
|
484
|
+
result.success = true;
|
|
485
|
+
result.errorCode = StitchErrorCode::Ok;
|
|
486
|
+
result.width = final_image.cols;
|
|
487
|
+
result.height = final_image.rows;
|
|
488
|
+
result.framesIncluded = framesIncluded;
|
|
489
|
+
result.finalConfidenceThresh = finalThreshold;
|
|
490
|
+
result.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
491
|
+
t1 - t0).count();
|
|
492
|
+
return result;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
// ════════════════════════════════════════════════════════════════════
|
|
497
|
+
// stitchFramePathsManual — manual cv::detail::* pipeline
|
|
498
|
+
// ════════════════════════════════════════════════════════════════════
|
|
499
|
+
//
|
|
500
|
+
// Ported from OpenCVStitcher.mm:stitchFramePaths: (the iOS-only
|
|
501
|
+
// ~1500-line method, lines 401-1911 of that file). Pipeline matches
|
|
502
|
+
// cv::Stitcher::PANORAMA's internal algorithm EXCEPT we drive the
|
|
503
|
+
// stages ourselves so we can:
|
|
504
|
+
//
|
|
505
|
+
// * Run GraphCutSeamFinder at SEAM_MP (0.1) instead of compose_MP —
|
|
506
|
+
// ~100× faster.
|
|
507
|
+
// * Re-warp at COMPOSE_MP (0.6-1.0) after BA runs at REGISTRATION_MP
|
|
508
|
+
// (0.3) — gives us back the cylindrical-era sharpness without the
|
|
509
|
+
// OOM that comes from composing at ORIG_RESOL.
|
|
510
|
+
// * Retry leaveBiggestComponent at PRUNE granularity (cheap) rather
|
|
511
|
+
// than around the whole stitch (cv::Stitcher's C+D approach).
|
|
512
|
+
// * Catch BA exceptions and fall back to estimator cameras instead
|
|
513
|
+
// of aborting.
|
|
514
|
+
//
|
|
515
|
+
// The 9-step structure is preserved with the original Step N: comments
|
|
516
|
+
// so iOS↔shared traceability stays intact. Many of the comments
|
|
517
|
+
// reference iOS-specific incidents (V12.x / V16 phases, Ram's traces,
|
|
518
|
+
// Console.app rate-limit behaviour) — those references are KEPT so
|
|
519
|
+
// the institutional memory survives the port. The behaviours they
|
|
520
|
+
// describe still apply on Android too: cv::resize allocator state,
|
|
521
|
+
// jetsam/lmkd-equivalent OOM-kill behaviour, BA convergence failures
|
|
522
|
+
// on landscape inputs.
|
|
523
|
+
//
|
|
524
|
+
// Notable differences from the iOS original:
|
|
525
|
+
// * No @autoreleasepool / ARC machinery — pure C++ stack semantics
|
|
526
|
+
// handle scope-exit cleanup. The fix-10 "capture failure into
|
|
527
|
+
// strong local + break out of pool" pattern collapses to ordinary
|
|
528
|
+
// C++ early-returns / goto-style break-out — we use a do/while(0)
|
|
529
|
+
// wrapper so all failure paths set the result and `break` once.
|
|
530
|
+
// * No os_log — replaced with the shared log_info/log_error
|
|
531
|
+
// callbacks. Originals' OS_LOG_TYPE_FAULT importance is encoded
|
|
532
|
+
// by routing to log_error.
|
|
533
|
+
// * No `loadFramesOrFail` helper — we inline the cv::imread loop
|
|
534
|
+
// because the shared cpp/stitcher.cpp already has its own loader
|
|
535
|
+
// pattern for the high-level path; reusing the same approach
|
|
536
|
+
// keeps the file's load-error handling consistent.
|
|
537
|
+
// * No EXIF-tag JPEG writer (WriteJPEGWithEXIFTag). iOS's
|
|
538
|
+
// ImageIO-backed writer baked an EXIF Orientation=1 tag into the
|
|
539
|
+
// output. We rely on `bake_rotation` already rotating the pixels
|
|
540
|
+
// in-place, so a tag-less cv::imwrite gives the same visual
|
|
541
|
+
// result on iOS image renderers. When this function is wired to
|
|
542
|
+
// iOS, if EXIF=1 is required, the iOS bridge can re-encode after
|
|
543
|
+
// the fact OR we add an EXIF-aware writer to the shared layer
|
|
544
|
+
// (see TODO[shared-stitcher-port-part-2] below).
|
|
545
|
+
// * Pre-stitch memory abort + kMaxFramesForStitch=8 frame cap are
|
|
546
|
+
// KEPT — they exist for the same reason on both platforms.
|
|
547
|
+
// * StitcherResidentMB / phys_footprint reporting collapses to the
|
|
548
|
+
// existing rss_mb() helper. rss_mb reads /proc/self/statm which
|
|
549
|
+
// works on both Android (Linux procfs) and iOS (procfs is mounted
|
|
550
|
+
// in the simulator at least; on real device this falls back to
|
|
551
|
+
// -1.0 which is harmless). TODO below covers re-introducing the
|
|
552
|
+
// mach task_info path if iOS reports -1 in production traces.
|
|
553
|
+
StitchResult stitchFramePathsManual(
|
|
554
|
+
const std::vector<std::string>& framePaths,
|
|
555
|
+
const std::string& outputPath,
|
|
556
|
+
const StitchConfig& config,
|
|
557
|
+
LogFn logFn)
|
|
558
|
+
{
|
|
559
|
+
const auto t0 = std::chrono::steady_clock::now();
|
|
560
|
+
StitchResult result;
|
|
561
|
+
result.framesRequested = static_cast<int32_t>(framePaths.size());
|
|
562
|
+
|
|
563
|
+
// V12.14.2 — FAULT-level sentinel. Survives Console.app rate-limit;
|
|
564
|
+
// proves the function entered. If a future trace doesn't show this
|
|
565
|
+
// line for a crashed run, the crash is BEFORE stitchFramePaths
|
|
566
|
+
// (e.g., in extractFramesFromVideoAtPath or in the dispatch_async
|
|
567
|
+
// block in StitcherBridge).
|
|
568
|
+
const double kStartResidentMB = rss_mb();
|
|
569
|
+
log_info(logFn, "[stitch-bc]",
|
|
570
|
+
"STITCH START: %zu frames mem=%.1fMB",
|
|
571
|
+
framePaths.size(), kStartResidentMB);
|
|
572
|
+
// 2026-05-18 (Iss #1 diag): mirror the high-level path's entry log so we
|
|
573
|
+
// can verify captureOrientation propagation through the manual pipeline.
|
|
574
|
+
// The high-level entry logs "orientation=" at line 280-290; the manual
|
|
575
|
+
// path was silent on this field, leaving us unable to tell, from a
|
|
576
|
+
// device-log dump alone, whether bake_rotation got the right input.
|
|
577
|
+
log_info(logFn, "[stitch]",
|
|
578
|
+
"stitchFramePathsManual: frames=%zu warper=%s blender=%s seam=%s "
|
|
579
|
+
"orientation=%s quality=%d inscribedRect=%d",
|
|
580
|
+
framePaths.size(),
|
|
581
|
+
config.warperType.c_str(),
|
|
582
|
+
config.blenderType.c_str(),
|
|
583
|
+
config.seamFinderType.c_str(),
|
|
584
|
+
config.captureOrientation.c_str(),
|
|
585
|
+
config.jpegQuality,
|
|
586
|
+
config.useInscribedRectCrop ? 1 : 0);
|
|
587
|
+
|
|
588
|
+
// V16 Phase 1b.fix1 — device-aware pre-stitch memory abort.
|
|
589
|
+
//
|
|
590
|
+
// Original V12.14.8 fixed the threshold at 700 MB, sized for legacy
|
|
591
|
+
// iPhones (~2 GB total RAM, ~720 MB jetsam kill point on camera-
|
|
592
|
+
// active foreground apps). That ceiling is irrelevant on modern
|
|
593
|
+
// hardware: iPhone 16 Pro has 8 GB RAM and a per-process limit of
|
|
594
|
+
// ~3 GB on iOS 26 (confirmed by JetsamEvent at 3.38 GB).
|
|
595
|
+
//
|
|
596
|
+
// Also, the V12.14.8 assumption — "vision-camera CameraView is
|
|
597
|
+
// unmounted before stitch, so baseline drops to ~350-450 MB" —
|
|
598
|
+
// doesn't hold for the V16 batch-keyframe flow, where the AR
|
|
599
|
+
// session keeps running during stitch (baseline naturally 600-800
|
|
600
|
+
// MB). AR pause is now done at the bridge level (Phase 1b.fix1
|
|
601
|
+
// in IncrementalStitcher.swift), but even with that, the
|
|
602
|
+
// 700 MB threshold throttles modern devices for no reason.
|
|
603
|
+
//
|
|
604
|
+
// New formula: max(700, totalRAMGB × 300). Leaves ~30% headroom
|
|
605
|
+
// below the per-process limit for the stitch peak.
|
|
606
|
+
// 2 GB device → 700 MB threshold (clamped, legacy protection)
|
|
607
|
+
// 4 GB device → 1200 MB
|
|
608
|
+
// 6 GB device → 1800 MB
|
|
609
|
+
// 8 GB device → 2400 MB
|
|
610
|
+
//
|
|
611
|
+
// Total-RAM source: prefer the caller-provided StitchConfig::
|
|
612
|
+
// availableRamMB (plumbed via NSProcessInfo.processInfo.physicalMemory
|
|
613
|
+
// on iOS, ActivityManager.getMemoryInfo().totalMem or sysconf on
|
|
614
|
+
// Android — see the StitchConfig field doc in stitcher.hpp). When
|
|
615
|
+
// the caller leaves the sentinel, fall back to the conservative
|
|
616
|
+
// 4 GB assumption so the threshold lands at 1200 MB — high enough
|
|
617
|
+
// not to throttle real devices, low enough to still abort on
|
|
618
|
+
// degenerate baselines. Plumbing the actual physicalMemory is
|
|
619
|
+
// important on modern iOS hardware: iPhone 16 Pro has 8 GB → 2400
|
|
620
|
+
// MB threshold (real-device headroom) rather than 1200 MB (legacy
|
|
621
|
+
// protection that caps a high-RAM device at low-RAM headroom).
|
|
622
|
+
const double kAssumedTotalRAMGB = 4.0;
|
|
623
|
+
const double availableRamMB = (config.availableRamMB > 0.0)
|
|
624
|
+
? config.availableRamMB
|
|
625
|
+
: (kAssumedTotalRAMGB * 1024.0);
|
|
626
|
+
const double availableRamGB = availableRamMB / 1024.0;
|
|
627
|
+
const double kPreStitchAbortMB = std::max(700.0, availableRamGB * 300.0);
|
|
628
|
+
if (kStartResidentMB > kPreStitchAbortMB) {
|
|
629
|
+
log_error(logFn, "[stitch-bc]",
|
|
630
|
+
"PRE-STITCH ABORT: mem=%.1fMB > %.1fMB threshold (totalRamMB=%.0f)",
|
|
631
|
+
kStartResidentMB, kPreStitchAbortMB, availableRamMB);
|
|
632
|
+
// V16 fix-attempt 9 — sentinel return. See validPairs<1 site
|
|
633
|
+
// below for the full root-cause analysis. In the iOS original
|
|
634
|
+
// this returned an empty RNStitchResult; here we return
|
|
635
|
+
// a StitchResult with success=false + a stable error code so
|
|
636
|
+
// both bridges see a clean failure rather than an
|
|
637
|
+
// ambiguous "output written but zero pixels" surface.
|
|
638
|
+
result.errorCode = StitchErrorCode::PreStitchMemoryAbort;
|
|
639
|
+
result.errorMessage = "Pre-stitch memory abort";
|
|
640
|
+
// framesIncluded reflects best-known retained count at the
|
|
641
|
+
// abort site — nothing has been loaded or matched yet.
|
|
642
|
+
result.framesIncluded = 0;
|
|
643
|
+
return result;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
if (framePaths.size() < 2) {
|
|
647
|
+
// V16 fix-attempt 9 — sentinel return.
|
|
648
|
+
result.errorCode = StitchErrorCode::InvalidArgument;
|
|
649
|
+
result.errorMessage = "Need at least 2 frames to stitch (got " +
|
|
650
|
+
std::to_string(framePaths.size()) + ")";
|
|
651
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
652
|
+
return result;
|
|
653
|
+
}
|
|
654
|
+
if (outputPath.empty()) {
|
|
655
|
+
result.errorCode = StitchErrorCode::InvalidArgument;
|
|
656
|
+
result.errorMessage = "outputPath must not be empty";
|
|
657
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
658
|
+
return result;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
// V12.14.2 — defensive frame cap. Ram's V12.14 traces showed a
|
|
662
|
+
// landscape capture with 12 frames (144 pairwise) crash inside
|
|
663
|
+
// BundleAdjusterRay. A 7-frame capture (49 pairwise) succeeded.
|
|
664
|
+
// Above ~10 frames the BA solver becomes unstable on landscape
|
|
665
|
+
// inputs — most likely the Levenberg-Marquardt Jacobian conditions
|
|
666
|
+
// get bad with the wider aspect ratio + more pairwise constraints.
|
|
667
|
+
// Cap framePaths to kMaxFramesForStitch evenly-spaced indices
|
|
668
|
+
// BEFORE the imread loop so we don't even pay the imread cost
|
|
669
|
+
// for the discarded frames. Trade-off: long pans get slightly
|
|
670
|
+
// less overlap (a 5-second pan at 3 fps = 15 frames is downsampled
|
|
671
|
+
// to 8 evenly-spaced). Quality regression is minor; stability is
|
|
672
|
+
// huge — this kills the EXC_BAD_ACCESS deterministically.
|
|
673
|
+
static const size_t kMaxFramesForStitch = 8;
|
|
674
|
+
std::vector<std::string> workFramePaths = framePaths;
|
|
675
|
+
if (workFramePaths.size() > kMaxFramesForStitch) {
|
|
676
|
+
std::vector<std::string> downsampled;
|
|
677
|
+
downsampled.reserve(kMaxFramesForStitch);
|
|
678
|
+
const size_t origCount = workFramePaths.size();
|
|
679
|
+
for (size_t i = 0; i < kMaxFramesForStitch; i++) {
|
|
680
|
+
size_t idx = (i * (origCount - 1)) / (kMaxFramesForStitch - 1);
|
|
681
|
+
downsampled.push_back(workFramePaths[idx]);
|
|
682
|
+
}
|
|
683
|
+
workFramePaths = std::move(downsampled);
|
|
684
|
+
log_info(logFn, "[stitch-bc]",
|
|
685
|
+
"downsampled %zu -> %zu frames (BA stability cap)",
|
|
686
|
+
origCount, kMaxFramesForStitch);
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
// Load all input frames before invoking the stitcher. Memory cost
|
|
690
|
+
// is N × frame size — for typical shelf captures (~2048×1536 RGB,
|
|
691
|
+
// ~9 MB / frame raw, but cv::imread decodes JPEG so resident
|
|
692
|
+
// footprint is bounded by the original sensor resolution).
|
|
693
|
+
//
|
|
694
|
+
// V12.13 — breadcrumb each load. If the landscape-only crash is
|
|
695
|
+
// in cv::imread (e.g., decoding a JPEG produced by the new
|
|
696
|
+
// per-frame autoreleasepool extract) the LAST log line tells us
|
|
697
|
+
// which frame index + path triggered it.
|
|
698
|
+
std::vector<cv::Mat> frames;
|
|
699
|
+
frames.reserve(workFramePaths.size());
|
|
700
|
+
for (size_t idx = 0; idx < workFramePaths.size(); ++idx) {
|
|
701
|
+
const std::string& path = workFramePaths[idx];
|
|
702
|
+
cv::Mat img = cv::imread(path);
|
|
703
|
+
log_info(logFn, "[stitch-bc]",
|
|
704
|
+
"loadFrames %zu/%zu: %s -> %dx%d (channels=%d, empty=%d)",
|
|
705
|
+
idx, workFramePaths.size(),
|
|
706
|
+
path.c_str(), img.cols, img.rows, img.channels(),
|
|
707
|
+
(int)img.empty());
|
|
708
|
+
if (img.empty()) {
|
|
709
|
+
result.errorCode = StitchErrorCode::ImageReadFailed;
|
|
710
|
+
result.errorMessage = "Could not read image at path: " + path;
|
|
711
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
712
|
+
// framesIncluded reflects best-known retained count at the
|
|
713
|
+
// abort site — number of frames successfully loaded so far.
|
|
714
|
+
result.framesIncluded = static_cast<int32_t>(frames.size());
|
|
715
|
+
return result;
|
|
716
|
+
}
|
|
717
|
+
frames.push_back(img);
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
// ── Hand-rolled stitch via cv::detail::* with CylindricalWarper ────
|
|
721
|
+
//
|
|
722
|
+
// The high-level cv::Stitcher::PANORAMA uses SphericalWarper, which
|
|
723
|
+
// produces the "panorama bowl" shape on short shelf-scan arcs.
|
|
724
|
+
// Calling setWarper(CylindricalWarper) on the high-level stitcher
|
|
725
|
+
// crashes (PANORAMA's BundleAdjusterRay's R-matrix outputs are
|
|
726
|
+
// structured for spherical warp). So we drive the pipeline
|
|
727
|
+
// ourselves, replicating PANORAMA's algorithm exactly EXCEPT we
|
|
728
|
+
// swap the warper at the end. This is also the same path
|
|
729
|
+
// Phase 5 will populate with AR-derived poses (skipping
|
|
730
|
+
// features→matching→BA when poses are known).
|
|
731
|
+
//
|
|
732
|
+
// Pipeline:
|
|
733
|
+
// 1. ORB features per frame
|
|
734
|
+
// 2. BestOf2NearestMatcher (PANORAMA's default)
|
|
735
|
+
// 3. HomographyBasedEstimator → camera initial guesses
|
|
736
|
+
// 4. BundleAdjusterRay (PANORAMA's default) refines cameras
|
|
737
|
+
// 5. CylindricalWarper warps each frame using cameras
|
|
738
|
+
// 6. GraphCutSeamFinder + MultiBandBlender produce final panorama
|
|
739
|
+
cv::Mat panorama;
|
|
740
|
+
// Breadcrumbs in the device console. If the next stitch
|
|
741
|
+
// crashes, the last logged step pinpoints the failure point —
|
|
742
|
+
// makes debugging without Xcode much faster. Prefix is
|
|
743
|
+
// grep-able in Console.app / logcat.
|
|
744
|
+
log_info(logFn, "[BatchStitcher]", "start: %zu frames", frames.size());
|
|
745
|
+
|
|
746
|
+
// V16 fix-10 (2026-05-13) — STRUCTURAL: NO return statement
|
|
747
|
+
// executes inside the @autoreleasepool block. Failure paths
|
|
748
|
+
// capture the result into a strong local declared above the
|
|
749
|
+
// pool, then `break` out of the do/while(0) wrapper. In the
|
|
750
|
+
// pure-C++ port the @autoreleasepool is gone (no ObjC autorelease
|
|
751
|
+
// semantics), but we keep the do/while(0) wrapper so all failure
|
|
752
|
+
// paths converge on a single result-construction site below.
|
|
753
|
+
// Helps reading + matches the iOS original's control flow line
|
|
754
|
+
// for line.
|
|
755
|
+
bool failedInsidePool = false;
|
|
756
|
+
bool sentinelInsidePool = false;
|
|
757
|
+
StitchErrorCode capturedErrorCode = StitchErrorCode::UnknownCvException;
|
|
758
|
+
std::string capturedErrorMessage;
|
|
759
|
+
|
|
760
|
+
do {
|
|
761
|
+
try {
|
|
762
|
+
// Two-stage resolution pipeline (matches cv::Stitcher::PANORAMA):
|
|
763
|
+
//
|
|
764
|
+
// REGISTRATION_MP (0.3): downscale used for features, matching,
|
|
765
|
+
// BA, wave-correct. The expensive optimisation stages run
|
|
766
|
+
// here. cv::Stitcher uses 0.6; we use 0.3 because BA still
|
|
767
|
+
// converges reliably on shelf-scan inputs and the smaller
|
|
768
|
+
// matrices make BA noticeably faster on iPhone.
|
|
769
|
+
//
|
|
770
|
+
// COMPOSE_MP (0.6-1.0): RE-WARP + blend at this larger resolution
|
|
771
|
+
// to produce the FINAL panorama. cv::Stitcher uses ORIG_RESOL
|
|
772
|
+
// (full input size) — gorgeous output but iPhones at 12 MP × N
|
|
773
|
+
// frames blow past the jetsam threshold. 1.0 MP is the
|
|
774
|
+
// sweet spot: ~2× linear sharpness over single-stage 0.3 MP,
|
|
775
|
+
// with peak compose memory still under ~120 MB thanks to the
|
|
776
|
+
// per-frame release pattern in the blender feed loop below.
|
|
777
|
+
//
|
|
778
|
+
// The cylindrical-era sharpness came from cv::Stitcher's
|
|
779
|
+
// automatic two-stage flow. When we hand-rolled the pipeline
|
|
780
|
+
// to use PlaneWarper safely, we collapsed it to a single 0.3 MP
|
|
781
|
+
// stage — output went from ~1500×800 (cylindrical) to ~700×400
|
|
782
|
+
// (plane). This restores the multi-stage structure while
|
|
783
|
+
// keeping the PlaneWarper that the host app actually wants.
|
|
784
|
+
//
|
|
785
|
+
// V2 port note: registrationResolMP / compositingResolMP from
|
|
786
|
+
// StitchConfig let the CALLER override the defaults if needed
|
|
787
|
+
// (e.g., a low-RAM device build can drop COMPOSE_MP to 0.4).
|
|
788
|
+
// When config values are <0, we use the hard-coded defaults
|
|
789
|
+
// ported from the iOS original (REGISTRATION=0.3, COMPOSE=0.6).
|
|
790
|
+
const double REGISTRATION_MP = (config.registrationResolMP > 0.0)
|
|
791
|
+
? config.registrationResolMP : 0.3;
|
|
792
|
+
// 0.6 MP matches cv::Stitcher::PANORAMA's registration_resol
|
|
793
|
+
// default and is the "safe sharp" setting on Debug builds —
|
|
794
|
+
// 1.0 MP was visibly sharper but pushed memory peak into iOS
|
|
795
|
+
// jetsam territory (Sentry caught WatchdogTermination + the
|
|
796
|
+
// EXC_BAD_ACCESS-during-tear-down variant under the same root
|
|
797
|
+
// cause). Release builds free ~200-300 MB of RN baseline
|
|
798
|
+
// overhead and would tolerate 1.0 MP fine; if/when a Release
|
|
799
|
+
// build is the test target, bump this back up.
|
|
800
|
+
const double COMPOSE_MP = (config.compositingResolMP > 0.0)
|
|
801
|
+
? config.compositingResolMP : 0.6;
|
|
802
|
+
|
|
803
|
+
// Capture original size BEFORE downscaling — we need it later
|
|
804
|
+
// to compute the compose scale relative to full-res input.
|
|
805
|
+
int origCols = frames[0].cols;
|
|
806
|
+
int origRows = frames[0].rows;
|
|
807
|
+
double origMp = (double)origCols * origRows / 1e6;
|
|
808
|
+
|
|
809
|
+
// Stage 1: downscale to REGISTRATION_MP for features+matching+BA.
|
|
810
|
+
std::vector<cv::Mat> workFrames;
|
|
811
|
+
workFrames.reserve(frames.size());
|
|
812
|
+
double work_scale = (origMp > REGISTRATION_MP)
|
|
813
|
+
? std::sqrt(REGISTRATION_MP / origMp)
|
|
814
|
+
: 1.0;
|
|
815
|
+
if (work_scale < 1.0) {
|
|
816
|
+
for (const auto& f : frames) {
|
|
817
|
+
cv::Mat scaled;
|
|
818
|
+
cv::resize(f, scaled, cv::Size(), work_scale, work_scale,
|
|
819
|
+
cv::INTER_AREA);
|
|
820
|
+
workFrames.push_back(scaled);
|
|
821
|
+
}
|
|
822
|
+
} else {
|
|
823
|
+
for (const auto& f : frames) workFrames.push_back(f);
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
log_info(logFn, "[BatchStitcher]",
|
|
827
|
+
"step1: features (work scale %d×%d)",
|
|
828
|
+
workFrames.empty() ? 0 : workFrames[0].cols,
|
|
829
|
+
workFrames.empty() ? 0 : workFrames[0].rows);
|
|
830
|
+
// V12.14 Commit B — paired fprintf(stderr) breadcrumb. iOS'
|
|
831
|
+
// Console.app rate-limits NSLog under high-frequency emission
|
|
832
|
+
// (Ram's V12.13 trace had loadFrames 4-7 + step1 missing while
|
|
833
|
+
// loadFrames 0-3 + step2 made it through). Stderr is not rate-
|
|
834
|
+
// limited and flushes promptly, so the LAST stderr line before
|
|
835
|
+
// the crash reliably pinpoints the failing stage.
|
|
836
|
+
//
|
|
837
|
+
// Shared-port note: this breadcrumb collapses to log_info under
|
|
838
|
+
// the shared LogFn callback. On Android the bridge sinks to
|
|
839
|
+
// logcat (not rate-limited). On iOS, when wired up, the bridge
|
|
840
|
+
// can still emit to os_log if rate-limit pressure returns.
|
|
841
|
+
log_info(logFn, "[stitch-bc]",
|
|
842
|
+
"step1 enter (work %d×%d, %zu frames)",
|
|
843
|
+
workFrames.empty() ? 0 : workFrames[0].cols,
|
|
844
|
+
workFrames.empty() ? 0 : workFrames[0].rows,
|
|
845
|
+
workFrames.size());
|
|
846
|
+
|
|
847
|
+
// Step 1: features. 800 ORB features is enough for matching
|
|
848
|
+
// ~50% overlap between adjacent frames; 1500 was overkill and
|
|
849
|
+
// doubled the matching work for marginal quality gain.
|
|
850
|
+
auto featuresFinder = cv::ORB::create(800);
|
|
851
|
+
std::vector<cv::detail::ImageFeatures> imgFeatures(workFrames.size());
|
|
852
|
+
for (size_t i = 0; i < workFrames.size(); i++) {
|
|
853
|
+
cv::detail::computeImageFeatures(featuresFinder, workFrames[i],
|
|
854
|
+
imgFeatures[i]);
|
|
855
|
+
imgFeatures[i].img_idx = (int)i;
|
|
856
|
+
log_info(logFn, "[stitch-bc]",
|
|
857
|
+
"step1 frame %zu: %zu features",
|
|
858
|
+
i, imgFeatures[i].keypoints.size());
|
|
859
|
+
}
|
|
860
|
+
log_info(logFn, "[stitch-bc]", "step1 done");
|
|
861
|
+
|
|
862
|
+
// Step 2: pairwise matching. match_conf=0.65 matches what
|
|
863
|
+
// cv::Stitcher::PANORAMA uses internally — looser values
|
|
864
|
+
// (counter-intuitively) hurt BA convergence by letting through
|
|
865
|
+
// contradictory low-confidence matches that don't fit a
|
|
866
|
+
// consistent rotation model. Stick with the proven default.
|
|
867
|
+
// V16 fix-11 (2026-05-13) — REVERTED the AffineBestOf2NearestMatcher
|
|
868
|
+
// swap. The swap (commit 505c6f1) targeted the validPairs=0
|
|
869
|
+
// symptom on translation-heavy captures, but produced a downstream
|
|
870
|
+
// regression: "Warp stage failed: matrix.cpp:246 setSize s >= 0".
|
|
871
|
+
//
|
|
872
|
+
// Root cause: cv::Stitcher's pipeline has TWO coherent end-to-end
|
|
873
|
+
// modes documented in OpenCV:
|
|
874
|
+
//
|
|
875
|
+
// PANORAMA: BestOf2NearestMatcher → HomographyBasedEstimator →
|
|
876
|
+
// BundleAdjusterRay → SphericalWarper/etc.
|
|
877
|
+
// All stages assume rotation-only camera motion.
|
|
878
|
+
//
|
|
879
|
+
// SCANS: AffineBestOf2NearestMatcher → AffineBasedEstimator →
|
|
880
|
+
// BundleAdjusterAffinePartial → AffineWarper.
|
|
881
|
+
// All stages assume affine (rotation+translation+
|
|
882
|
+
// scale+shear) camera motion.
|
|
883
|
+
//
|
|
884
|
+
// Swapping ONLY step 2 to affine while keeping the rotation-only
|
|
885
|
+
// estimator/BA/warper downstream produced incoherent camera
|
|
886
|
+
// parameters: the affine matcher passed inliers with parallax-
|
|
887
|
+
// induced inconsistencies that the rotation estimator turned into
|
|
888
|
+
// non-orthonormal "rotation" matrices. The warper then computed
|
|
889
|
+
// negative destination canvas sizes and the cv::Mat::setSize
|
|
890
|
+
// assertion fired at step 8b.
|
|
891
|
+
//
|
|
892
|
+
// Fix: revert to BestOf2NearestMatcher so the WHOLE pipeline is
|
|
893
|
+
// coherent in PANORAMA mode. Translation-heavy captures fall
|
|
894
|
+
// back to validPairs=0 → sentinel result → clean toast (no
|
|
895
|
+
// crash, thanks to fix-10's @autoreleasepool restructure). The
|
|
896
|
+
// gate's translation-budget force-accept (`flowMaxTranslationCm`
|
|
897
|
+
// in Settings) is the operator's lever to keep per-pair
|
|
898
|
+
// translation small enough that BestOf2NearestMatcher's
|
|
899
|
+
// rotation-homography RANSAC produces useful inliers.
|
|
900
|
+
//
|
|
901
|
+
// Longer-term: see docs/site-content/design/2026-05-13-stitch-
|
|
902
|
+
// pipeline-mode-selection.md for the architectural answer —
|
|
903
|
+
// motion-classified per-capture routing between PANORAMA and
|
|
904
|
+
// SCANS modes at finalize() time.
|
|
905
|
+
log_info(logFn, "[BatchStitcher]", "step2: matching");
|
|
906
|
+
log_info(logFn, "[stitch-bc]",
|
|
907
|
+
"step2 enter: BestOf2Nearest matching (PANORAMA mode — coherent end-to-end)");
|
|
908
|
+
cv::detail::BestOf2NearestMatcher matcher(false, 0.65f);
|
|
909
|
+
std::vector<cv::detail::MatchesInfo> pairwise;
|
|
910
|
+
matcher(imgFeatures, pairwise);
|
|
911
|
+
matcher.collectGarbage();
|
|
912
|
+
log_info(logFn, "[stitch-bc]",
|
|
913
|
+
"step2 done: %zu pairwise entries", pairwise.size());
|
|
914
|
+
|
|
915
|
+
// Step 3: leave-best-of-2 keeps only well-connected images at
|
|
916
|
+
// confThresh=1.0 — also matches cv::Stitcher::PANORAMA's
|
|
917
|
+
// default. Pairs with weaker overlap get dropped before BA.
|
|
918
|
+
// Pre-check: count how many pairwise matches actually have
|
|
919
|
+
// non-trivial features matched. cv::Stitcher's
|
|
920
|
+
// leaveBiggestComponent / HomographyBasedEstimator fire
|
|
921
|
+
// CV_Assert internally if no useful pairwise data exists —
|
|
922
|
+
// and CV_Assert can SIGABRT in our build (signal not caught
|
|
923
|
+
// by C++ try/catch). Throwing our own structured error here
|
|
924
|
+
// is the only way to fail-fast before that abort.
|
|
925
|
+
int validPairs = 0;
|
|
926
|
+
for (const auto& m : pairwise) {
|
|
927
|
+
if (m.confidence > 0.0 && m.matches.size() >= 6) {
|
|
928
|
+
validPairs++;
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
log_info(logFn, "[BatchStitcher]",
|
|
932
|
+
"step2.5: %d valid pairwise matches", validPairs);
|
|
933
|
+
if (validPairs < 1) {
|
|
934
|
+
// V16 fix-attempt 9 (NULL TEST, 2026-05-13). Eight prior
|
|
935
|
+
// attempts chased a deterministic SEGV inside Swift's try-bridge
|
|
936
|
+
// on this *error→throw path. ASan-on-device with Sentry
|
|
937
|
+
// disabled (incident-2026-05-13-172125.ips) showed
|
|
938
|
+
// EXC_BAD_ACCESS at 0x60007a530 (UNMAPPED VM, ASan
|
|
939
|
+
// ReportDeadlySignal — no shadow-memory match) firing inside
|
|
940
|
+
// objc_retain immediately after this return. By returning a
|
|
941
|
+
// non-nil SENTINEL result (width=0, height=0) instead of
|
|
942
|
+
// populating *error and returning nil, we bypass Swift's
|
|
943
|
+
// autoreleasing NSError out-parameter retain entirely. The
|
|
944
|
+
// Swift caller in IncrementalStitcher.finalize checks
|
|
945
|
+
// `r.width == 0` and constructs a Swift-native NSError to pass
|
|
946
|
+
// to its completion block.
|
|
947
|
+
//
|
|
948
|
+
// Hypothesis under test:
|
|
949
|
+
// (A) If this path no longer crashes → the throw bridge IS
|
|
950
|
+
// the proximate trigger. Permanent: keep sentinel,
|
|
951
|
+
// document why.
|
|
952
|
+
// (B) If it still crashes the same way → corruption is
|
|
953
|
+
// upstream of our return (likely inside opencv2.framework
|
|
954
|
+
// stitcher allocator pool). Revert and escalate to C3
|
|
955
|
+
// (stitch on isolated DispatchQueue).
|
|
956
|
+
//
|
|
957
|
+
// See: docs/site-content/design/2026-05-12-finalize-crash-investigation.md
|
|
958
|
+
//
|
|
959
|
+
// Shared-port note: in the pure-C++ port there is no ObjC
|
|
960
|
+
// autoreleasing-NSError pad, so the historical reason for
|
|
961
|
+
// the sentinel is gone. We still mark success=false +
|
|
962
|
+
// emit a structured StitchErrorCode so the JS layer sees
|
|
963
|
+
// a clean failure (it's the JS surface that surfaces the
|
|
964
|
+
// "all frames dropped" toast). Kept the long comment
|
|
965
|
+
// because it explains WHY the iOS bridge added a
|
|
966
|
+
// sentinel-path check — historically valuable.
|
|
967
|
+
log_error(logFn, "[BatchStitcher]",
|
|
968
|
+
"step2.5: 0 valid pairs — sentinel result (port: signalling AllFramesDroppedByConfidence)");
|
|
969
|
+
capturedErrorCode = StitchErrorCode::AllFramesDroppedByConfidence;
|
|
970
|
+
capturedErrorMessage = "Stitcher found 0 valid pairwise matches — frames may not overlap enough.";
|
|
971
|
+
// framesIncluded reflects best-known retained count at the
|
|
972
|
+
// abort site — pre-prune so all loaded frames are still in
|
|
973
|
+
// play even though none have valid pairwise overlap.
|
|
974
|
+
result.framesIncluded = static_cast<int32_t>(imgFeatures.size());
|
|
975
|
+
sentinelInsidePool = true;
|
|
976
|
+
break;
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
log_info(logFn, "[BatchStitcher]", "step3: leave-biggest");
|
|
980
|
+
log_info(logFn, "[stitch-bc]", "step3 enter: leave-biggest");
|
|
981
|
+
// leaveBiggestComponent mutates imgFeatures and pairwise IN
|
|
982
|
+
// PLACE to drop frames that aren't part of the biggest
|
|
983
|
+
// connected component. We MUST also subset workFrames to
|
|
984
|
+
// match — otherwise cameras.size() (built from the trimmed
|
|
985
|
+
// imgFeatures) will be smaller than workFrames.size() and the
|
|
986
|
+
// warp loop reads cameras[i] out of bounds. That's a likely
|
|
987
|
+
// root cause of the SIGABRT seen on second-stitch attempts.
|
|
988
|
+
//
|
|
989
|
+
// C+D progressive-confidence retry at PRUNE granularity.
|
|
990
|
+
// Mirrors the high-level entry's [1.0, 0.5, 0.3] threshold
|
|
991
|
+
// sweep, but the retry only re-runs leaveBiggestComponent
|
|
992
|
+
// (cheap) rather than every stage of cv::Stitcher::stitch
|
|
993
|
+
// (5-10× more expensive). cv::detail::leaveBiggestComponent
|
|
994
|
+
// MUTATES imgFeatures + pairwise in place, so we keep
|
|
995
|
+
// defensive backup copies and restore them before each retry
|
|
996
|
+
// (approach (a) — copy beats rematching, since
|
|
997
|
+
// BestOf2NearestMatcher is the dominant cost).
|
|
998
|
+
//
|
|
999
|
+
// SCANS mode skips thresholds > 0.31 — its default is already
|
|
1000
|
+
// 0.3 and dropping pairs at 1.0 / 0.5 produces vacuous results.
|
|
1001
|
+
// 2026-05-18 (Issue #2 RCA): the previous break condition was
|
|
1002
|
+
// `workFrames.size() >= 2` which exited on the FIRST attempt
|
|
1003
|
+
// that retained the minimum-stitchable count. But
|
|
1004
|
+
// leaveBiggestComponent is monotonic in inclusion: lower
|
|
1005
|
+
// threshold = MORE frames retained. So if attempt 1
|
|
1006
|
+
// (thresh=1.0) retains 2/4, attempts 2/3 (thresh=0.5/0.3)
|
|
1007
|
+
// might retain 3/4 or 4/4. The early break threw away that
|
|
1008
|
+
// signal, so user-visible captures of 4 keyframes
|
|
1009
|
+
// consistently shipped with only 2 in the panorama at
|
|
1010
|
+
// thresh=1.0, never benefiting from the retry sweep.
|
|
1011
|
+
//
|
|
1012
|
+
// New behaviour: ONLY break early when all input frames are
|
|
1013
|
+
// retained (no point trying lower thresholds — they can't do
|
|
1014
|
+
// better). Otherwise let the loop run to its lowest
|
|
1015
|
+
// threshold; the resulting workFrames carries the most
|
|
1016
|
+
// inclusive prune at the end. pruneSucceeded flips true on
|
|
1017
|
+
// any attempt that yields >=2 frames; pruneThresholdUsed
|
|
1018
|
+
// tracks the threshold of the latest successful attempt.
|
|
1019
|
+
const float kPruneThresholds[] = {1.0f, 0.5f, 0.3f};
|
|
1020
|
+
const int kNumPruneAttempts =
|
|
1021
|
+
sizeof(kPruneThresholds) / sizeof(kPruneThresholds[0]);
|
|
1022
|
+
const std::vector<cv::detail::ImageFeatures> imgFeaturesBackup =
|
|
1023
|
+
imgFeatures;
|
|
1024
|
+
const std::vector<cv::detail::MatchesInfo> pairwiseBackup = pairwise;
|
|
1025
|
+
const std::vector<cv::Mat> workFramesBackup = workFrames;
|
|
1026
|
+
const std::vector<cv::Mat> framesBackup = frames;
|
|
1027
|
+
const size_t initialFrameCount = imgFeatures.size();
|
|
1028
|
+
float pruneThresholdUsed = -1.0f;
|
|
1029
|
+
bool pruneSucceeded = false;
|
|
1030
|
+
for (int attempt = 0; attempt < kNumPruneAttempts; ++attempt) {
|
|
1031
|
+
const float thresh = kPruneThresholds[attempt];
|
|
1032
|
+
if (config.stitchMode == StitchMode::Scans && thresh > 0.31f) {
|
|
1033
|
+
continue;
|
|
1034
|
+
}
|
|
1035
|
+
// Restore from backups before each attempt — leaveBiggest-
|
|
1036
|
+
// Component mutated them last time. First attempt sees the
|
|
1037
|
+
// originals (backup == current), subsequent attempts get a
|
|
1038
|
+
// clean slate.
|
|
1039
|
+
if (attempt > 0) {
|
|
1040
|
+
imgFeatures = imgFeaturesBackup;
|
|
1041
|
+
pairwise = pairwiseBackup;
|
|
1042
|
+
workFrames = workFramesBackup;
|
|
1043
|
+
frames = framesBackup;
|
|
1044
|
+
}
|
|
1045
|
+
log_info(logFn, "[stitch-bc]",
|
|
1046
|
+
"step3 prune-retry attempt %d: thresh=%.2f",
|
|
1047
|
+
attempt + 1, thresh);
|
|
1048
|
+
std::vector<int> indices = cv::detail::leaveBiggestComponent(
|
|
1049
|
+
imgFeatures, pairwise, thresh);
|
|
1050
|
+
// Trim BOTH workFrames AND the full-res frames using the same
|
|
1051
|
+
// indices. workFrames feeds BA below; full-res frames feed the
|
|
1052
|
+
// compose stage further down (re-warped at COMPOSE_MP). Both
|
|
1053
|
+
// must stay aligned with cameras[i] / imgFeatures[i] post-trim.
|
|
1054
|
+
std::vector<cv::Mat> trimmedWorkFrames;
|
|
1055
|
+
std::vector<cv::Mat> trimmedFrames;
|
|
1056
|
+
trimmedWorkFrames.reserve(indices.size());
|
|
1057
|
+
trimmedFrames.reserve(indices.size());
|
|
1058
|
+
for (int idx : indices) {
|
|
1059
|
+
if (idx >= 0 && idx < (int)workFrames.size()) {
|
|
1060
|
+
trimmedWorkFrames.push_back(workFrames[idx]);
|
|
1061
|
+
trimmedFrames.push_back(frames[idx]);
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
workFrames = std::move(trimmedWorkFrames);
|
|
1065
|
+
frames = std::move(trimmedFrames);
|
|
1066
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1067
|
+
"step3.5: thresh=%.2f kept %zu of %zu frames in biggest component",
|
|
1068
|
+
thresh, workFrames.size(), initialFrameCount);
|
|
1069
|
+
if (workFrames.size() >= 2) {
|
|
1070
|
+
pruneThresholdUsed = thresh;
|
|
1071
|
+
pruneSucceeded = true;
|
|
1072
|
+
}
|
|
1073
|
+
if (workFrames.size() == initialFrameCount) {
|
|
1074
|
+
// All retained — no point trying lower thresholds.
|
|
1075
|
+
log_info(logFn, "[stitch-bc]",
|
|
1076
|
+
"step3 prune-retry attempt %d: all %zu frames "
|
|
1077
|
+
"retained — stopping retry sweep",
|
|
1078
|
+
attempt + 1, initialFrameCount);
|
|
1079
|
+
break;
|
|
1080
|
+
}
|
|
1081
|
+
// Partial retention. Either keep trying lower thresholds
|
|
1082
|
+
// (might retain more), or — if this is the last attempt
|
|
1083
|
+
// — accept the partial result that pruneSucceeded captured.
|
|
1084
|
+
if (attempt + 1 < kNumPruneAttempts) {
|
|
1085
|
+
log_info(logFn, "[stitch-bc]",
|
|
1086
|
+
"step3 prune-retry attempt %d kept only %zu/%zu "
|
|
1087
|
+
"frames — retrying with lower threshold",
|
|
1088
|
+
attempt + 1, workFrames.size(), initialFrameCount);
|
|
1089
|
+
} else {
|
|
1090
|
+
log_info(logFn, "[stitch-bc]",
|
|
1091
|
+
"step3 prune-retry attempt %d kept %zu/%zu "
|
|
1092
|
+
"frames at lowest threshold %.2f — accepting "
|
|
1093
|
+
"(success=%d)",
|
|
1094
|
+
attempt + 1, workFrames.size(), initialFrameCount,
|
|
1095
|
+
thresh, pruneSucceeded ? 1 : 0);
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
if (!pruneSucceeded) {
|
|
1099
|
+
// V16 fix-attempt 9 (NULL TEST) — same rationale as the
|
|
1100
|
+
// validPairs<1 sentinel above. Bypass the *error→throw bridge
|
|
1101
|
+
// by returning a width=0/height=0 sentinel result instead.
|
|
1102
|
+
log_error(logFn, "[BatchStitcher]",
|
|
1103
|
+
"step3.5: <2 frames after leaveBiggestComponent at all thresholds — sentinel result");
|
|
1104
|
+
capturedErrorCode = StitchErrorCode::AllFramesDroppedByConfidence;
|
|
1105
|
+
capturedErrorMessage = "Less than 2 frames remain after leaveBiggestComponent at all retry thresholds.";
|
|
1106
|
+
// framesIncluded reflects best-known retained count at the
|
|
1107
|
+
// abort site — the most recent attempt's trim outcome.
|
|
1108
|
+
result.framesIncluded = static_cast<int32_t>(workFrames.size());
|
|
1109
|
+
sentinelInsidePool = true;
|
|
1110
|
+
break;
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
// Step 4: estimator
|
|
1114
|
+
log_info(logFn, "[BatchStitcher]", "step4: estimator");
|
|
1115
|
+
log_info(logFn, "[stitch-bc]", "step4 enter: estimator");
|
|
1116
|
+
cv::detail::HomographyBasedEstimator estimator;
|
|
1117
|
+
std::vector<cv::detail::CameraParams> cameras;
|
|
1118
|
+
if (!estimator(imgFeatures, pairwise, cameras)) {
|
|
1119
|
+
// V16 fix-attempt 9 — sentinel return (see validPairs<1 site
|
|
1120
|
+
// above for full RCA). Estimator failures are a real production
|
|
1121
|
+
// hazard on borderline-dissimilar frame sequences (typical mode:
|
|
1122
|
+
// user pans through occluded regions or featureless walls
|
|
1123
|
+
// mid-arc). Returning sentinel keeps the failure surface clean
|
|
1124
|
+
// even though the immediate V16 batch-keyframe repro doesn't
|
|
1125
|
+
// typically reach this path.
|
|
1126
|
+
log_error(logFn, "[BatchStitcher]",
|
|
1127
|
+
"step4: HomographyBasedEstimator failed — sentinel result");
|
|
1128
|
+
capturedErrorCode = StitchErrorCode::HomographyEstimationFailed;
|
|
1129
|
+
capturedErrorMessage = "HomographyBasedEstimator failed.";
|
|
1130
|
+
// framesIncluded reflects best-known retained count at the
|
|
1131
|
+
// abort site — the post-prune workFrames count.
|
|
1132
|
+
result.framesIncluded = static_cast<int32_t>(workFrames.size());
|
|
1133
|
+
sentinelInsidePool = true;
|
|
1134
|
+
break;
|
|
1135
|
+
}
|
|
1136
|
+
for (auto& cam : cameras) {
|
|
1137
|
+
cv::Mat R32;
|
|
1138
|
+
cam.R.convertTo(R32, CV_32F);
|
|
1139
|
+
cam.R = R32;
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
// Step 5: bundle adjustment (the slow step). BundleAdjusterRay
|
|
1143
|
+
// is what cv::Stitcher::PANORAMA uses internally. confThresh=1.0
|
|
1144
|
+
// matches cv::Stitcher's default — drops weak match-pair
|
|
1145
|
+
// constraints from the optimisation so BA converges reliably.
|
|
1146
|
+
// Cap iterations at 100 (default 1000) so a poorly-conditioned
|
|
1147
|
+
// problem can't run away into a 60s timeout. BA typically
|
|
1148
|
+
// converges in 20-50 iters on good input; if 100 isn't enough,
|
|
1149
|
+
// the inputs themselves are unstitchable and we want to fail
|
|
1150
|
+
// fast rather than spin.
|
|
1151
|
+
{
|
|
1152
|
+
auto _t = std::chrono::steady_clock::now();
|
|
1153
|
+
double _ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
1154
|
+
_t - t0).count();
|
|
1155
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1156
|
+
"step5: bundle adjustment (t+%.0fms)", _ms);
|
|
1157
|
+
log_info(logFn, "[stitch-bc]", "step5 enter: bundle adjustment");
|
|
1158
|
+
}
|
|
1159
|
+
auto adjuster = cv::makePtr<cv::detail::BundleAdjusterRay>();
|
|
1160
|
+
adjuster->setConfThresh(1.0f);
|
|
1161
|
+
adjuster->setTermCriteria(cv::TermCriteria(
|
|
1162
|
+
cv::TermCriteria::EPS + cv::TermCriteria::COUNT,
|
|
1163
|
+
100,
|
|
1164
|
+
DBL_EPSILON));
|
|
1165
|
+
|
|
1166
|
+
// V12.14.2 — FAULT-level sentinel + camera sanity dump. These
|
|
1167
|
+
// bracket the BA call so a future trace can pinpoint whether
|
|
1168
|
+
// the crash is BEFORE BA invocation, INSIDE BA, or AFTER BA.
|
|
1169
|
+
// Also dump the first camera's R[0,0] + focal so we can see if
|
|
1170
|
+
// estimator produced NaN/Inf values that would crash BA's
|
|
1171
|
+
// Levenberg-Marquardt.
|
|
1172
|
+
{
|
|
1173
|
+
double r00 = cameras.empty() ? 0.0 :
|
|
1174
|
+
(cameras[0].R.empty() ? 0.0 : (double)cameras[0].R.at<float>(0, 0));
|
|
1175
|
+
double focal = cameras.empty() ? 0.0 : cameras[0].focal;
|
|
1176
|
+
log_info(logFn, "[stitch-bc]",
|
|
1177
|
+
"step5 BA INVOKE: cameras=%zu cam0.R[0,0]=%.4f cam0.focal=%.2f",
|
|
1178
|
+
cameras.size(), r00, focal);
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
// V12.14.2 — wrap BA in try/catch. Catches cv::Exception (most
|
|
1182
|
+
// likely if BA detects a bad input) and std::exception (defensive).
|
|
1183
|
+
// On exception, fall back to the estimator cameras (skipping the
|
|
1184
|
+
// BA refinement step). Pano quality is slightly lower without
|
|
1185
|
+
// BA but it WON'T CRASH. Note: this catches C++ exceptions;
|
|
1186
|
+
// raw SIGSEGV from BA's internal pointer deref would still
|
|
1187
|
+
// terminate the process — for that, the kMaxFramesForStitch=8
|
|
1188
|
+
// cap above is the primary defence.
|
|
1189
|
+
bool baSucceeded = false;
|
|
1190
|
+
try {
|
|
1191
|
+
baSucceeded = (*adjuster)(imgFeatures, pairwise, cameras);
|
|
1192
|
+
} catch (const cv::Exception& e) {
|
|
1193
|
+
log_error(logFn, "[stitch-bc]",
|
|
1194
|
+
"step5 BA threw cv::Exception: %s — fallback to estimator cameras",
|
|
1195
|
+
e.what());
|
|
1196
|
+
baSucceeded = false;
|
|
1197
|
+
} catch (const std::exception& e) {
|
|
1198
|
+
log_error(logFn, "[stitch-bc]",
|
|
1199
|
+
"step5 BA threw std::exception: %s — fallback to estimator cameras",
|
|
1200
|
+
e.what());
|
|
1201
|
+
baSucceeded = false;
|
|
1202
|
+
} catch (...) {
|
|
1203
|
+
log_error(logFn, "[stitch-bc]",
|
|
1204
|
+
"step5 BA threw unknown exception — fallback to estimator cameras");
|
|
1205
|
+
baSucceeded = false;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
if (!baSucceeded) {
|
|
1209
|
+
// Fall through with the cameras the estimator produced —
|
|
1210
|
+
// step5.5 wave correction + step6+ compose can still run on
|
|
1211
|
+
// unrefined cameras. Result quality will be lower (no global
|
|
1212
|
+
// optimisation) but the engine returns a panorama instead of
|
|
1213
|
+
// crashing.
|
|
1214
|
+
log_info(logFn, "[stitch-bc]",
|
|
1215
|
+
"step5 BA SKIPPED — proceeding with estimator cameras");
|
|
1216
|
+
} else {
|
|
1217
|
+
log_info(logFn, "[stitch-bc]", "step5 BA OK");
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
// Step 5.5: WAVE CORRECTION. cv::Stitcher::PANORAMA does
|
|
1221
|
+
// this automatically; my hand-rolled pipeline was missing it.
|
|
1222
|
+
// After BA produces camera rotation matrices, waveCorrect
|
|
1223
|
+
// globally rotates them so all cameras share a consistent
|
|
1224
|
+
// up-vector. Without this, the cylindrical (or spherical)
|
|
1225
|
+
// projection produces visible "wavy" top / bottom edges where
|
|
1226
|
+
// edge frames hit the projection surface at slightly
|
|
1227
|
+
// different vertical angles.
|
|
1228
|
+
//
|
|
1229
|
+
// WAVE_CORRECT_HORIZ — this is what was working yesterday for
|
|
1230
|
+
// BOTH portrait+horizontal-pan and landscape+vertical-pan.
|
|
1231
|
+
// Why it works for both: HORIZ aligns each camera's "up" vector
|
|
1232
|
+
// to the world Y axis (gravity). vision-camera writes mp4s
|
|
1233
|
+
// with `outputOrientation="device"` so the saved frames are
|
|
1234
|
+
// already in the user's view orientation; after BA + waveCorrect
|
|
1235
|
+
// HORIZ, the panorama's vertical axis matches world's vertical
|
|
1236
|
+
// axis regardless of pan direction.
|
|
1237
|
+
//
|
|
1238
|
+
// I briefly switched to autoDetectWaveCorrectKind thinking it'd
|
|
1239
|
+
// handle vertical pans better — it actually picked the wrong
|
|
1240
|
+
// kind for portrait+horizontal pans, breaking yesterday's
|
|
1241
|
+
// working normal-mode capture. Reverting.
|
|
1242
|
+
{
|
|
1243
|
+
auto _t = std::chrono::steady_clock::now();
|
|
1244
|
+
double _ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
1245
|
+
_t - t0).count();
|
|
1246
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1247
|
+
"step5.5: wave correction (BA done, t+%.0fms)", _ms);
|
|
1248
|
+
log_info(logFn, "[stitch-bc]", "step5.5 enter: wave correction");
|
|
1249
|
+
}
|
|
1250
|
+
std::vector<cv::Mat> rmats;
|
|
1251
|
+
rmats.reserve(cameras.size());
|
|
1252
|
+
for (const auto& cam : cameras) {
|
|
1253
|
+
rmats.push_back(cam.R.clone());
|
|
1254
|
+
}
|
|
1255
|
+
try {
|
|
1256
|
+
cv::detail::waveCorrect(rmats, cv::detail::WAVE_CORRECT_HORIZ);
|
|
1257
|
+
for (size_t i = 0; i < cameras.size(); i++) {
|
|
1258
|
+
cameras[i].R = rmats[i];
|
|
1259
|
+
}
|
|
1260
|
+
} catch (const cv::Exception& e) {
|
|
1261
|
+
// Wave correction can fail on degenerate input (only 1-2
|
|
1262
|
+
// cameras with collinear rotations). Swallow the failure
|
|
1263
|
+
// and continue without correction — the panorama will have
|
|
1264
|
+
// the wave artifact but is still better than aborting.
|
|
1265
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1266
|
+
"wave correction skipped: %s", e.what());
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
// Step 6: COMPOSE rescale. This is the key step that gives us
|
|
1270
|
+
// back the cylindrical-era sharpness. cv::Stitcher does this
|
|
1271
|
+
// internally as `composePanorama`: rescale camera intrinsics
|
|
1272
|
+
// by (compose_scale / work_scale), recreate the warper at
|
|
1273
|
+
// the new scale, then warp+blend on freshly-resized frames at
|
|
1274
|
+
// COMPOSE_MP. Without this step, output stays at REGISTRATION_MP
|
|
1275
|
+
// and is visibly blurry.
|
|
1276
|
+
double compose_scale = (origMp > COMPOSE_MP)
|
|
1277
|
+
? std::sqrt(COMPOSE_MP / origMp)
|
|
1278
|
+
: 1.0;
|
|
1279
|
+
double compose_work_aspect = compose_scale / work_scale;
|
|
1280
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1281
|
+
"step6: compose rescale "
|
|
1282
|
+
"(work_scale=%.3f → compose_scale=%.3f, aspect=%.3f)",
|
|
1283
|
+
work_scale, compose_scale, compose_work_aspect);
|
|
1284
|
+
for (auto& cam : cameras) {
|
|
1285
|
+
cam.focal *= compose_work_aspect;
|
|
1286
|
+
cam.ppx *= compose_work_aspect;
|
|
1287
|
+
cam.ppy *= compose_work_aspect;
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
// Step 6.5: median focal length determines the warper scale.
|
|
1291
|
+
// Computed AFTER compose rescale so warpedScale is already in
|
|
1292
|
+
// compose units — matches cv::Stitcher's flow.
|
|
1293
|
+
std::vector<double> focals;
|
|
1294
|
+
for (const auto& cam : cameras) focals.push_back(cam.focal);
|
|
1295
|
+
std::sort(focals.begin(), focals.end());
|
|
1296
|
+
float warpedScale =
|
|
1297
|
+
focals.empty() ? 1.0f
|
|
1298
|
+
: (float)focals[focals.size() / 2];
|
|
1299
|
+
|
|
1300
|
+
// Step 7: PLANE warper. The crucial swap.
|
|
1301
|
+
//
|
|
1302
|
+
// For close-up shelf scans (~30° pan, mostly translational
|
|
1303
|
+
// gesture across a planar product face), plane projection is
|
|
1304
|
+
// the right choice — it produces a flat output with no
|
|
1305
|
+
// cylindrical curve and no spherical bowl.
|
|
1306
|
+
//
|
|
1307
|
+
// Cylindrical/spherical only buy you something for wider arcs
|
|
1308
|
+
// where the per-frame perspective curves matter. Below ~45°
|
|
1309
|
+
// arc, plane is empirically the most natural-looking option
|
|
1310
|
+
// and exactly what SCANS mode used (just SCANS coupled it
|
|
1311
|
+
// with affine BA which we just established was the wrong
|
|
1312
|
+
// estimator for our motion).
|
|
1313
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1314
|
+
"step7: warper (%s)", config.warperType.c_str());
|
|
1315
|
+
log_info(logFn, "[stitch-bc]",
|
|
1316
|
+
"step7 enter: warper=%s", config.warperType.c_str());
|
|
1317
|
+
// Plane / Cylindrical / Spherical — runtime-selectable so
|
|
1318
|
+
// the host's settings UI can A/B test which projection looks
|
|
1319
|
+
// best for the operator's actual gesture (close-up planar
|
|
1320
|
+
// subject vs partial-arc rotation vs wide pan).
|
|
1321
|
+
cv::Ptr<cv::WarperCreator> warperCreator;
|
|
1322
|
+
if (config.warperType == "cylindrical") {
|
|
1323
|
+
warperCreator = cv::makePtr<cv::CylindricalWarper>();
|
|
1324
|
+
} else if (config.warperType == "spherical") {
|
|
1325
|
+
warperCreator = cv::makePtr<cv::SphericalWarper>();
|
|
1326
|
+
} else {
|
|
1327
|
+
// "plane" is the default — straight verticals/horizontals,
|
|
1328
|
+
// good for close-up subjects. Hourglass shape produced
|
|
1329
|
+
// by partial arcs is removed by the rectangular-crop step
|
|
1330
|
+
// below.
|
|
1331
|
+
warperCreator = cv::makePtr<cv::PlaneWarper>();
|
|
1332
|
+
}
|
|
1333
|
+
// V12.14.3 — FAULT breadcrumbs around each sub-step within
|
|
1334
|
+
// step7 → step7.5. Ram's V12.14.2 trace had the crash here
|
|
1335
|
+
// (last visible log was step7 enter; step7.5 never fired).
|
|
1336
|
+
// These pinpoint which sub-step actually crashes.
|
|
1337
|
+
cv::Ptr<cv::detail::RotationWarper> warper =
|
|
1338
|
+
warperCreator->create(warpedScale);
|
|
1339
|
+
log_info(logFn, "[stitch-bc]",
|
|
1340
|
+
"step7a: warper created (warpedScale=%.2f)", warpedScale);
|
|
1341
|
+
|
|
1342
|
+
// Step 7.5: build composeFrames at COMPOSE_MP from full-res
|
|
1343
|
+
// input. Warp + blend run at this resolution to produce the
|
|
1344
|
+
// sharp final output. Release workFrames first — BA is done,
|
|
1345
|
+
// so we don't need the small set anymore. Sequential release
|
|
1346
|
+
// ensures the two big arrays never coexist at peak.
|
|
1347
|
+
for (auto& wf : workFrames) wf.release();
|
|
1348
|
+
workFrames.clear();
|
|
1349
|
+
log_info(logFn, "[stitch-bc]",
|
|
1350
|
+
"step7b: workFrames released, building composeFrames "
|
|
1351
|
+
"(N=%zu, compose_scale=%.3f)",
|
|
1352
|
+
frames.size(), compose_scale);
|
|
1353
|
+
|
|
1354
|
+
// V12.14.3 — wrap the resize loop in try/catch so a bad input
|
|
1355
|
+
// Mat doesn't terminate the process. Per-frame resize on
|
|
1356
|
+
// bogus/corrupt cv::Mat data has historically been a SIGSEGV
|
|
1357
|
+
// source on consecutive captures.
|
|
1358
|
+
std::vector<cv::Mat> composeFrames;
|
|
1359
|
+
composeFrames.reserve(frames.size());
|
|
1360
|
+
try {
|
|
1361
|
+
for (size_t i = 0; i < frames.size(); i++) {
|
|
1362
|
+
const auto& f = frames[i];
|
|
1363
|
+
log_info(logFn, "[stitch-bc]",
|
|
1364
|
+
"step7c: resize frame %zu (%dx%d, channels=%d, "
|
|
1365
|
+
"data=%p)", i, f.cols, f.rows, f.channels(),
|
|
1366
|
+
(const void*)f.data);
|
|
1367
|
+
|
|
1368
|
+
// V12.14.4 — defensive validation. Skip frames with NULL
|
|
1369
|
+
// data ptr, zero dimensions, or non-positive total — they
|
|
1370
|
+
// would SIGSEGV inside cv::resize regardless of interp mode.
|
|
1371
|
+
if (f.data == nullptr || f.empty() || f.total() == 0
|
|
1372
|
+
|| f.cols <= 0 || f.rows <= 0) {
|
|
1373
|
+
log_error(logFn, "[stitch-bc]",
|
|
1374
|
+
"step7c: SKIPPING frame %zu — invalid Mat "
|
|
1375
|
+
"(data=%p empty=%d total=%zu)",
|
|
1376
|
+
i, (const void*)f.data, (int)f.empty(),
|
|
1377
|
+
(size_t)f.total());
|
|
1378
|
+
continue;
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
// V12.14.4 — original wraps each iteration in
|
|
1382
|
+
// @autoreleasepool so any ObjC temporaries cv::resize
|
|
1383
|
+
// might autorelease internally get drained between
|
|
1384
|
+
// frames. In the pure-C++ port this is a no-op: pure
|
|
1385
|
+
// C++ has no autoreleased temporaries; cv::Mat's RAII
|
|
1386
|
+
// dtor runs at iteration scope exit naturally.
|
|
1387
|
+
cv::Mat scaled;
|
|
1388
|
+
if (std::abs(compose_scale - 1.0) > 1e-3) {
|
|
1389
|
+
// V12.14.4 — pre-allocate `scaled` with explicit dims
|
|
1390
|
+
// BEFORE cv::resize so the internal `dst.create()` is a
|
|
1391
|
+
// no-op. Skips the allocator state corruption Ram's
|
|
1392
|
+
// V12.14.3 trace pointed at: cv::resize crashed on the
|
|
1393
|
+
// 5th consecutive resize when iOS recycled mmap regions
|
|
1394
|
+
// from a prior capture, suggesting cv::resize's internal
|
|
1395
|
+
// allocator path was hitting stale state.
|
|
1396
|
+
//
|
|
1397
|
+
// Plus: switch INTER_AREA → INTER_LINEAR. INTER_AREA
|
|
1398
|
+
// uses precomputed cached interpolation tables that
|
|
1399
|
+
// appear to be the corrupted state. INTER_LINEAR uses
|
|
1400
|
+
// a different code path (no cached table). Slightly
|
|
1401
|
+
// less crisp at extreme downscales but for our 0.538×
|
|
1402
|
+
// shelf-image downscale the visual difference is
|
|
1403
|
+
// negligible — and stability >> sharpness.
|
|
1404
|
+
int newCols = (int)std::round(f.cols * compose_scale);
|
|
1405
|
+
int newRows = (int)std::round(f.rows * compose_scale);
|
|
1406
|
+
scaled.create(newRows, newCols, f.type());
|
|
1407
|
+
cv::resize(f, scaled, scaled.size(), 0, 0, cv::INTER_LINEAR);
|
|
1408
|
+
} else {
|
|
1409
|
+
scaled = f.clone();
|
|
1410
|
+
}
|
|
1411
|
+
composeFrames.push_back(scaled);
|
|
1412
|
+
}
|
|
1413
|
+
} catch (const cv::Exception& e) {
|
|
1414
|
+
// V12.14.7 — %{public}s so the message survives Console.app
|
|
1415
|
+
// privacy redaction. Without this, e.what() shows as "<private>"
|
|
1416
|
+
// and we can't see which assertion fired.
|
|
1417
|
+
log_error(logFn, "[stitch-bc]",
|
|
1418
|
+
"step7c: cv::resize threw cv::Exception: %s",
|
|
1419
|
+
e.what());
|
|
1420
|
+
capturedErrorCode = StitchErrorCode::ComposeResizeFailed;
|
|
1421
|
+
capturedErrorMessage = std::string("Compose-stage resize failed: ") + e.what();
|
|
1422
|
+
// framesIncluded reflects best-known retained count at the
|
|
1423
|
+
// abort site — cameras has been populated by step4 so it's
|
|
1424
|
+
// the most accurate post-prune count.
|
|
1425
|
+
result.framesIncluded = static_cast<int32_t>(cameras.size());
|
|
1426
|
+
failedInsidePool = true;
|
|
1427
|
+
break;
|
|
1428
|
+
} catch (...) {
|
|
1429
|
+
log_error(logFn, "[stitch-bc]",
|
|
1430
|
+
"step7c: cv::resize threw unknown exception");
|
|
1431
|
+
capturedErrorCode = StitchErrorCode::ComposeResizeFailed;
|
|
1432
|
+
capturedErrorMessage = "Compose-stage resize failed (unknown).";
|
|
1433
|
+
result.framesIncluded = static_cast<int32_t>(cameras.size());
|
|
1434
|
+
failedInsidePool = true;
|
|
1435
|
+
break;
|
|
1436
|
+
}
|
|
1437
|
+
log_info(logFn, "[stitch-bc]",
|
|
1438
|
+
"step7d: composeFrames built (N=%zu)",
|
|
1439
|
+
composeFrames.size());
|
|
1440
|
+
|
|
1441
|
+
// Release full-res `frames` now that composeFrames has its
|
|
1442
|
+
// own resized copies. Frees ~50-100 MB for a typical 8-frame
|
|
1443
|
+
// stitch — a critical part of staying under iOS' jetsam
|
|
1444
|
+
// threshold (the ACTUAL cause of the "u != 0" /
|
|
1445
|
+
// WatchdogTermination crashes we were debugging — Sentry
|
|
1446
|
+
// confirmed those were OOM kills, not OpenCV bugs).
|
|
1447
|
+
for (auto& f : frames) f.release();
|
|
1448
|
+
frames.clear();
|
|
1449
|
+
log_info(logFn, "[stitch-bc]",
|
|
1450
|
+
"step7e: full-res frames released mem=%.1fMB",
|
|
1451
|
+
rss_mb());
|
|
1452
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1453
|
+
"step7.5: composeFrames %d×%d "
|
|
1454
|
+
"(compose_scale=%.3f)",
|
|
1455
|
+
composeFrames.empty() ? 0 : composeFrames[0].cols,
|
|
1456
|
+
composeFrames.empty() ? 0 : composeFrames[0].rows,
|
|
1457
|
+
compose_scale);
|
|
1458
|
+
|
|
1459
|
+
// Step 8: warp + (optional) seam finder + blender feed.
|
|
1460
|
+
//
|
|
1461
|
+
// Two paths based on caller's seamFinderType:
|
|
1462
|
+
//
|
|
1463
|
+
// "graphcut" — BATCH path. Warp all frames into memory,
|
|
1464
|
+
// run GraphCutSeamFinder for optimal seams, then feed
|
|
1465
|
+
// the blender. Higher peak memory (all warped frames
|
|
1466
|
+
// coexist during seam finding) but produces clean seams
|
|
1467
|
+
// that pair beautifully with MultiBandBlender. Same
|
|
1468
|
+
// algorithm cv::Stitcher::PANORAMA uses internally.
|
|
1469
|
+
//
|
|
1470
|
+
// "skip" — STREAM path. Warp + feed each frame in the
|
|
1471
|
+
// same loop, releasing immediately. Never holds more
|
|
1472
|
+
// than one warped frame in memory. ~40-50 MB lower peak
|
|
1473
|
+
// at 1.0 MP × 8 frames. Right choice for low-RAM
|
|
1474
|
+
// devices; the host's per-device defaults pick this
|
|
1475
|
+
// path on devices with <2 GB physical RAM.
|
|
1476
|
+
//
|
|
1477
|
+
// Both paths feed the SAME blender (selected per caller's
|
|
1478
|
+
// blenderType). Final blend happens after either path
|
|
1479
|
+
// completes.
|
|
1480
|
+
const bool useSeam = (config.seamFinderType == "graphcut");
|
|
1481
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1482
|
+
"step8: %s",
|
|
1483
|
+
useSeam ? "BATCH (warp-all + seam + feed)"
|
|
1484
|
+
: "STREAM (warp+feed per frame)");
|
|
1485
|
+
log_info(logFn, "[stitch-bc]",
|
|
1486
|
+
"step8 enter: %s", useSeam ? "BATCH" : "STREAM");
|
|
1487
|
+
|
|
1488
|
+
// Build the blender once — both paths feed into it.
|
|
1489
|
+
//
|
|
1490
|
+
// The "u != 0" UMat assertion we previously hit when running
|
|
1491
|
+
// MultiBand or GraphCut was a SYMPTOM of iOS jetsam OOM-kill
|
|
1492
|
+
// (confirmed via Sentry's WatchdogTermination signature),
|
|
1493
|
+
// not a bug in MBB / GraphCut. With the OOM fixes now in
|
|
1494
|
+
// place (autoreleasepool wrapping, camera pause during
|
|
1495
|
+
// stitch, per-frame Mat releases, plus this stream path for
|
|
1496
|
+
// low-mem devices), both should run cleanly.
|
|
1497
|
+
cv::Ptr<cv::detail::Blender> blender;
|
|
1498
|
+
if (config.blenderType == "feather") {
|
|
1499
|
+
blender = cv::detail::Blender::createDefault(
|
|
1500
|
+
cv::detail::Blender::FEATHER, false);
|
|
1501
|
+
auto fb = blender.dynamicCast<cv::detail::FeatherBlender>();
|
|
1502
|
+
if (fb) fb->setSharpness(0.02f);
|
|
1503
|
+
} else {
|
|
1504
|
+
// "multiband" — Laplacian pyramids per fed frame.
|
|
1505
|
+
// More memory than Feather but much sharper seams when
|
|
1506
|
+
// paired with GraphCut.
|
|
1507
|
+
blender = cv::detail::Blender::createDefault(
|
|
1508
|
+
cv::detail::Blender::MULTI_BAND, false);
|
|
1509
|
+
auto mbb = blender.dynamicCast<cv::detail::MultiBandBlender>();
|
|
1510
|
+
if (mbb) mbb->setNumBands(5);
|
|
1511
|
+
}
|
|
1512
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1513
|
+
"step10: blender = %s", config.blenderType.c_str());
|
|
1514
|
+
|
|
1515
|
+
if (useSeam) {
|
|
1516
|
+
// ── BATCH path ─────────────────────────────────────────────
|
|
1517
|
+
const size_t N = composeFrames.size();
|
|
1518
|
+
std::vector<cv::Point> corners(N);
|
|
1519
|
+
std::vector<cv::Mat> imagesWarped(N);
|
|
1520
|
+
std::vector<cv::Mat> masksWarped(N);
|
|
1521
|
+
std::vector<cv::Size> sizes(N);
|
|
1522
|
+
log_info(logFn, "[stitch-bc]",
|
|
1523
|
+
"step8a: BATCH warp loop (N=%zu)", N);
|
|
1524
|
+
// V12.14.6 — defensive measures around the warp loop. Same
|
|
1525
|
+
// recycled-mmap pattern that hit cv::resize in V12.14.3
|
|
1526
|
+
// logs (Ram's 4th-capture crash). cv::PlaneWarper::warp
|
|
1527
|
+
// uses cv::remap internally which has its own cached state
|
|
1528
|
+
// keyed on input addresses.
|
|
1529
|
+
try {
|
|
1530
|
+
for (size_t i = 0; i < N; i++) {
|
|
1531
|
+
log_info(logFn, "[stitch-bc]",
|
|
1532
|
+
"step8b: warp frame %zu (%dx%d, data=%p)", i,
|
|
1533
|
+
composeFrames[i].cols, composeFrames[i].rows,
|
|
1534
|
+
(const void*)composeFrames[i].data);
|
|
1535
|
+
// Per-iteration scope drains any autoreleased temps in
|
|
1536
|
+
// the iOS original; pure C++ does this via RAII.
|
|
1537
|
+
cv::Mat K;
|
|
1538
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1539
|
+
|
|
1540
|
+
// V12.14.6 — clone input to break any recycled-mmap
|
|
1541
|
+
// link to prior captures' allocations. cv::Mat::clone
|
|
1542
|
+
// forces a fresh memcpy into a freshly-allocated buffer.
|
|
1543
|
+
cv::Mat freshInput = composeFrames[i].clone();
|
|
1544
|
+
|
|
1545
|
+
// V12.14.6 — pre-allocate output Mats via warpRoi() so
|
|
1546
|
+
// cv::remap doesn't need to call create() internally
|
|
1547
|
+
// (the suspect path that crashed in cv::resize too).
|
|
1548
|
+
cv::Rect roi = warper->warpRoi(
|
|
1549
|
+
freshInput.size(), K, cameras[i].R);
|
|
1550
|
+
// 2026-05-18 (Issue #1 guard): cv::Stitcher's estimator
|
|
1551
|
+
// + BA can produce wildly wrong camera parameters on
|
|
1552
|
+
// degenerate input (low feature count, near-duplicate
|
|
1553
|
+
// frames, poor texture). warpRoi() then returns an
|
|
1554
|
+
// absurd rectangle (we observed 191 GB allocation on a
|
|
1555
|
+
// standard 4-frame capture). Without this guard the
|
|
1556
|
+
// imagesWarped[i].create() below tries to allocate
|
|
1557
|
+
// hundreds of GB and either OOMs or hard-OOMs the
|
|
1558
|
+
// process. Cap at 100 MP (~400 MB at 3 channels) —
|
|
1559
|
+
// any panorama frame requiring more than 100 MP of
|
|
1560
|
+
// intermediate storage is from a broken estimator,
|
|
1561
|
+
// not a real capture worth completing.
|
|
1562
|
+
constexpr int64_t kMaxWarpPixels = 100ll * 1000ll * 1000ll;
|
|
1563
|
+
const int64_t roiPixels =
|
|
1564
|
+
static_cast<int64_t>(roi.width)
|
|
1565
|
+
* static_cast<int64_t>(roi.height);
|
|
1566
|
+
if (roi.width <= 0 || roi.height <= 0
|
|
1567
|
+
|| roiPixels > kMaxWarpPixels) {
|
|
1568
|
+
log_error(logFn, "[stitch-bc]",
|
|
1569
|
+
"step8b: warpRoi degenerate for frame "
|
|
1570
|
+
"%zu (%dx%d = %lld px > %lld limit) — "
|
|
1571
|
+
"treating as warp failure",
|
|
1572
|
+
i, roi.width, roi.height,
|
|
1573
|
+
(long long)roiPixels,
|
|
1574
|
+
(long long)kMaxWarpPixels);
|
|
1575
|
+
throw cv::Exception(
|
|
1576
|
+
cv::Error::StsOutOfRange,
|
|
1577
|
+
std::string("warpRoi too large (")
|
|
1578
|
+
+ std::to_string(roi.width) + "x"
|
|
1579
|
+
+ std::to_string(roi.height)
|
|
1580
|
+
+ ") — estimator produced degenerate "
|
|
1581
|
+
+ "camera params on this frame",
|
|
1582
|
+
"stitchFramePathsManual",
|
|
1583
|
+
__FILE__, __LINE__);
|
|
1584
|
+
}
|
|
1585
|
+
imagesWarped[i].create(roi.size(), freshInput.type());
|
|
1586
|
+
masksWarped[i].create(roi.size(), CV_8U);
|
|
1587
|
+
|
|
1588
|
+
cv::Mat mask(freshInput.size(), CV_8U, cv::Scalar(255));
|
|
1589
|
+
corners[i] = warper->warp(
|
|
1590
|
+
freshInput, K, cameras[i].R, cv::INTER_LINEAR,
|
|
1591
|
+
cv::BORDER_CONSTANT, imagesWarped[i]);
|
|
1592
|
+
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1593
|
+
cv::BORDER_CONSTANT, masksWarped[i]);
|
|
1594
|
+
sizes[i] = imagesWarped[i].size();
|
|
1595
|
+
// V12.14.7 — release composeFrames[i] inside the loop
|
|
1596
|
+
// (was: released only after the entire loop at step8c).
|
|
1597
|
+
// Frees ~14 MB per frame mid-loop, keeping peak working
|
|
1598
|
+
// set ~50-100 MB lower for an 8-frame batch — directly
|
|
1599
|
+
// targets the jetsam OOM kill that struck V12.14.6
|
|
1600
|
+
// after cv::Exception was caught (process died despite
|
|
1601
|
+
// managed throw). composeFrames[i] is no longer needed
|
|
1602
|
+
// after warp populates imagesWarped[i] / masksWarped[i].
|
|
1603
|
+
composeFrames[i].release();
|
|
1604
|
+
}
|
|
1605
|
+
} catch (const cv::Exception& e) {
|
|
1606
|
+
// V12.14.7 — %{public}s to unredact the message under
|
|
1607
|
+
// Console.app privacy filtering. e.what() was showing as
|
|
1608
|
+
// "<private>" in V12.14.6's caught traces.
|
|
1609
|
+
log_error(logFn, "[stitch-bc]",
|
|
1610
|
+
"step8b: warper->warp threw cv::Exception: %s",
|
|
1611
|
+
e.what());
|
|
1612
|
+
capturedErrorCode = StitchErrorCode::WarpFailed;
|
|
1613
|
+
capturedErrorMessage = std::string("Warp stage failed: ") + e.what();
|
|
1614
|
+
// framesIncluded reflects best-known retained count at
|
|
1615
|
+
// the abort site — cameras is fully populated by step6.
|
|
1616
|
+
result.framesIncluded = static_cast<int32_t>(cameras.size());
|
|
1617
|
+
failedInsidePool = true;
|
|
1618
|
+
break;
|
|
1619
|
+
} catch (...) {
|
|
1620
|
+
log_error(logFn, "[stitch-bc]",
|
|
1621
|
+
"step8b: warper->warp threw unknown exception");
|
|
1622
|
+
capturedErrorCode = StitchErrorCode::WarpFailed;
|
|
1623
|
+
capturedErrorMessage = "Warp stage failed (unknown).";
|
|
1624
|
+
result.framesIncluded = static_cast<int32_t>(cameras.size());
|
|
1625
|
+
failedInsidePool = true;
|
|
1626
|
+
break;
|
|
1627
|
+
}
|
|
1628
|
+
log_info(logFn, "[stitch-bc]",
|
|
1629
|
+
"step8c: warp loop done mem=%.1fMB", rss_mb());
|
|
1630
|
+
// composeFrames has done its job — release before we
|
|
1631
|
+
// allocate the float UMat shadow set for seam finding.
|
|
1632
|
+
// V12.14.7: most/all of these are already released inside
|
|
1633
|
+
// the warp loop above; the .clear() drops the now-empty
|
|
1634
|
+
// Mat headers from the vector.
|
|
1635
|
+
for (auto& cf : composeFrames) cf.release();
|
|
1636
|
+
composeFrames.clear();
|
|
1637
|
+
|
|
1638
|
+
// Step 9: GraphCutSeamFinder at SEAM_MP (~0.1 MP).
|
|
1639
|
+
//
|
|
1640
|
+
// GraphCut's runtime is roughly quadratic in pixel count
|
|
1641
|
+
// because it solves a max-flow on a per-pixel grid graph.
|
|
1642
|
+
// Running it at compose scale (1.0 MP) takes ~100× longer
|
|
1643
|
+
// than at the ~0.1 MP that cv::Stitcher::PANORAMA uses
|
|
1644
|
+
// internally (`seam_est_resol_ = 0.1`). At 1.0 MP we
|
|
1645
|
+
// observed >60s stitch-timeouts in JS; at 0.1 MP it
|
|
1646
|
+
// finishes in <1s. Pattern matches cv::Stitcher's flow:
|
|
1647
|
+
// 1. Downscale imagesWarped + masksWarped + corners to
|
|
1648
|
+
// seam scale.
|
|
1649
|
+
// 2. Run seam finder on the small images.
|
|
1650
|
+
// 3. Upscale the seam-optimised masks back to compose
|
|
1651
|
+
// scale.
|
|
1652
|
+
// 4. Bitwise-AND with the original masks so we don't
|
|
1653
|
+
// include pixels outside each frame's warped region.
|
|
1654
|
+
const double SEAM_MP = (config.seamEstimationResolMP > 0.0)
|
|
1655
|
+
? config.seamEstimationResolMP : 0.1;
|
|
1656
|
+
double seam_scale = std::min(1.0, std::sqrt(SEAM_MP / origMp));
|
|
1657
|
+
// Aspect from compose scale → seam scale (the rescale we
|
|
1658
|
+
// apply to existing compose-scale data, not the original).
|
|
1659
|
+
double seam_compose_aspect = seam_scale / compose_scale;
|
|
1660
|
+
{
|
|
1661
|
+
auto _t = std::chrono::steady_clock::now();
|
|
1662
|
+
double _ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
1663
|
+
_t - t0).count();
|
|
1664
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1665
|
+
"step9: graph-cut seam finder "
|
|
1666
|
+
"(compose→seam aspect = %.3f, t+%.0fms)",
|
|
1667
|
+
seam_compose_aspect, _ms);
|
|
1668
|
+
}
|
|
1669
|
+
auto _seamStart = std::chrono::steady_clock::now();
|
|
1670
|
+
log_info(logFn, "[stitch-bc]",
|
|
1671
|
+
"step9a: seam-scale resize loop (aspect=%.3f)",
|
|
1672
|
+
seam_compose_aspect);
|
|
1673
|
+
std::vector<cv::UMat> imagesWarpedF_seam(N);
|
|
1674
|
+
std::vector<cv::UMat> masksWarpedU_seam(N);
|
|
1675
|
+
std::vector<cv::Point> corners_seam(N);
|
|
1676
|
+
for (size_t i = 0; i < N; i++) {
|
|
1677
|
+
cv::Mat seamImage, seamMask;
|
|
1678
|
+
cv::resize(imagesWarped[i], seamImage, cv::Size(),
|
|
1679
|
+
seam_compose_aspect, seam_compose_aspect,
|
|
1680
|
+
cv::INTER_LINEAR);
|
|
1681
|
+
cv::resize(masksWarped[i], seamMask, cv::Size(),
|
|
1682
|
+
seam_compose_aspect, seam_compose_aspect,
|
|
1683
|
+
cv::INTER_NEAREST);
|
|
1684
|
+
seamImage.convertTo(imagesWarpedF_seam[i], CV_32F);
|
|
1685
|
+
seamMask.copyTo(masksWarpedU_seam[i]);
|
|
1686
|
+
corners_seam[i] = cv::Point(
|
|
1687
|
+
cvRound(corners[i].x * seam_compose_aspect),
|
|
1688
|
+
cvRound(corners[i].y * seam_compose_aspect));
|
|
1689
|
+
}
|
|
1690
|
+
log_info(logFn, "[stitch-bc]",
|
|
1691
|
+
"step9b: seam-scale resize done, GraphCut find starting");
|
|
1692
|
+
cv::Ptr<cv::detail::SeamFinder> seamFinder =
|
|
1693
|
+
cv::makePtr<cv::detail::GraphCutSeamFinder>(
|
|
1694
|
+
cv::detail::GraphCutSeamFinder::COST_COLOR);
|
|
1695
|
+
seamFinder->find(imagesWarpedF_seam, corners_seam,
|
|
1696
|
+
masksWarpedU_seam);
|
|
1697
|
+
log_info(logFn, "[stitch-bc]", "step9c: GraphCut find done");
|
|
1698
|
+
{
|
|
1699
|
+
auto _t = std::chrono::steady_clock::now();
|
|
1700
|
+
double _seamMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
1701
|
+
_t - _seamStart).count();
|
|
1702
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1703
|
+
"step9: graph-cut find took %.0fms", _seamMs);
|
|
1704
|
+
}
|
|
1705
|
+
imagesWarpedF_seam.clear();
|
|
1706
|
+
|
|
1707
|
+
// Upscale seam-optimised masks back to compose scale.
|
|
1708
|
+
//
|
|
1709
|
+
// CRITICAL: dilate each mask before upscaling so adjacent
|
|
1710
|
+
// frames have a small OVERLAP region for the blender to
|
|
1711
|
+
// feather across. Without this, the seam-cut creates a
|
|
1712
|
+
// strict pixel partition with NO overlap — MultiBand then
|
|
1713
|
+
// has nothing to feather, producing visible HARD seams
|
|
1714
|
+
// (the "cuts" we observed in the output). cv::Stitcher
|
|
1715
|
+
// does the same dilation step in its compose pipeline.
|
|
1716
|
+
// A 3×3 default kernel at seam scale becomes ~10px of
|
|
1717
|
+
// overlap at compose scale (since seam_aspect_compose ≈
|
|
1718
|
+
// 0.1 → 10× upscale), which is plenty for MultiBand's
|
|
1719
|
+
// Laplacian pyramids to blend smoothly across.
|
|
1720
|
+
//
|
|
1721
|
+
// The bitwise_and with the original mask keeps each frame's
|
|
1722
|
+
// mask within its actual warped region (seam-cut + dilation
|
|
1723
|
+
// can spill past edges, especially after linear upscale).
|
|
1724
|
+
for (size_t i = 0; i < N; i++) {
|
|
1725
|
+
cv::Mat seamMaskCpu, seamMaskDilated, seamMaskFull;
|
|
1726
|
+
masksWarpedU_seam[i].copyTo(seamMaskCpu);
|
|
1727
|
+
cv::dilate(seamMaskCpu, seamMaskDilated, cv::Mat());
|
|
1728
|
+
cv::resize(seamMaskDilated, seamMaskFull,
|
|
1729
|
+
masksWarped[i].size(), 0, 0, cv::INTER_LINEAR);
|
|
1730
|
+
cv::bitwise_and(seamMaskFull, masksWarped[i], masksWarped[i]);
|
|
1731
|
+
}
|
|
1732
|
+
masksWarpedU_seam.clear();
|
|
1733
|
+
|
|
1734
|
+
// Feed the blender, releasing each frame as we go.
|
|
1735
|
+
log_info(logFn, "[stitch-bc]", "step10a: blender->prepare");
|
|
1736
|
+
blender->prepare(corners, sizes);
|
|
1737
|
+
log_info(logFn, "[stitch-bc]",
|
|
1738
|
+
"step10b: feeding blender (N=%zu)", N);
|
|
1739
|
+
for (size_t i = 0; i < N; i++) {
|
|
1740
|
+
log_info(logFn, "[stitch-bc]", "step10c: feed frame %zu", i);
|
|
1741
|
+
cv::Mat imgS;
|
|
1742
|
+
imagesWarped[i].convertTo(imgS, CV_16S);
|
|
1743
|
+
blender->feed(imgS, masksWarped[i], corners[i]);
|
|
1744
|
+
imagesWarped[i].release();
|
|
1745
|
+
masksWarped[i].release();
|
|
1746
|
+
imgS.release();
|
|
1747
|
+
}
|
|
1748
|
+
imagesWarped.clear();
|
|
1749
|
+
masksWarped.clear();
|
|
1750
|
+
log_info(logFn, "[stitch-bc]", "step10d: feed loop done");
|
|
1751
|
+
} else {
|
|
1752
|
+
// ── STREAM path ────────────────────────────────────────────
|
|
1753
|
+
// Pre-pass: warp masks ONLY (single-channel, cheap) to
|
|
1754
|
+
// compute corners + sizes. blender->prepare() needs both
|
|
1755
|
+
// BEFORE the first feed, so a tiny first pass is unavoidable.
|
|
1756
|
+
const size_t N = composeFrames.size();
|
|
1757
|
+
std::vector<cv::Point> corners(N);
|
|
1758
|
+
std::vector<cv::Size> sizes(N);
|
|
1759
|
+
for (size_t i = 0; i < N; i++) {
|
|
1760
|
+
cv::Mat K;
|
|
1761
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1762
|
+
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1763
|
+
cv::Mat tmpMaskWarped;
|
|
1764
|
+
corners[i] = warper->warp(
|
|
1765
|
+
mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1766
|
+
cv::BORDER_CONSTANT, tmpMaskWarped);
|
|
1767
|
+
sizes[i] = tmpMaskWarped.size();
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
// Main pass: warp + feed + release per frame. Never holds
|
|
1771
|
+
// more than ONE warped image + ONE warped mask in memory.
|
|
1772
|
+
// ~40-50 MB lower peak vs the BATCH path at 1.0 MP × 8
|
|
1773
|
+
// frames — the difference between staying under iOS' jetsam
|
|
1774
|
+
// threshold on a 2 GB device and getting WatchdogTermination.
|
|
1775
|
+
blender->prepare(corners, sizes);
|
|
1776
|
+
for (size_t i = 0; i < N; i++) {
|
|
1777
|
+
cv::Mat K;
|
|
1778
|
+
cameras[i].K().convertTo(K, CV_32F);
|
|
1779
|
+
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1780
|
+
cv::Mat imgWarped, maskWarped;
|
|
1781
|
+
warper->warp(composeFrames[i], K, cameras[i].R,
|
|
1782
|
+
cv::INTER_LINEAR, cv::BORDER_CONSTANT, imgWarped);
|
|
1783
|
+
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1784
|
+
cv::BORDER_CONSTANT, maskWarped);
|
|
1785
|
+
cv::Mat imgS;
|
|
1786
|
+
imgWarped.convertTo(imgS, CV_16S);
|
|
1787
|
+
blender->feed(imgS, maskWarped, corners[i]);
|
|
1788
|
+
// Release the input compose frame too — done with it.
|
|
1789
|
+
composeFrames[i].release();
|
|
1790
|
+
// imgS / imgWarped / maskWarped release at scope exit.
|
|
1791
|
+
}
|
|
1792
|
+
composeFrames.clear();
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
cv::Mat panoramaS, panoramaMask;
|
|
1796
|
+
log_info(logFn, "[stitch-bc]", "step11a: blender->blend starting");
|
|
1797
|
+
blender->blend(panoramaS, panoramaMask);
|
|
1798
|
+
log_info(logFn, "[stitch-bc]",
|
|
1799
|
+
"step11b: blend complete (panoramaS=%dx%d)",
|
|
1800
|
+
panoramaS.cols, panoramaS.rows);
|
|
1801
|
+
panoramaS.convertTo(panorama, CV_8U);
|
|
1802
|
+
{
|
|
1803
|
+
auto _t = std::chrono::steady_clock::now();
|
|
1804
|
+
double _ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
1805
|
+
_t - t0).count();
|
|
1806
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1807
|
+
"step11: blend complete (output %d×%d, t+%.0fms)",
|
|
1808
|
+
panorama.cols, panorama.rows, _ms);
|
|
1809
|
+
}
|
|
1810
|
+
log_info(logFn, "[stitch-bc]",
|
|
1811
|
+
"step11c: panorama 8U conversion done (panorama=%dx%d) mem=%.1fMB",
|
|
1812
|
+
panorama.cols, panorama.rows, rss_mb());
|
|
1813
|
+
|
|
1814
|
+
// Record retained-frame count for telemetry. In the high-level
|
|
1815
|
+
// path this comes from stitcher->component().size() after retry;
|
|
1816
|
+
// in the manual path it's whatever leaveBiggestComponent kept
|
|
1817
|
+
// at the threshold that succeeded.
|
|
1818
|
+
result.framesIncluded = static_cast<int32_t>(cameras.size());
|
|
1819
|
+
// Threshold from the C+D progressive-confidence retry at PRUNE
|
|
1820
|
+
// granularity above. Matches the high-level path's telemetry
|
|
1821
|
+
// semantics: -1.0 means we never ran the prune (shouldn't
|
|
1822
|
+
// happen on success-path), else the threshold that produced
|
|
1823
|
+
// ≥ 2 frames in the biggest component.
|
|
1824
|
+
result.finalConfidenceThresh = (pruneThresholdUsed > 0.0f)
|
|
1825
|
+
? static_cast<double>(pruneThresholdUsed)
|
|
1826
|
+
: 1.0;
|
|
1827
|
+
} catch (const cv::Exception& e) {
|
|
1828
|
+
// Top-level catch: anything inside the pipeline that wasn't
|
|
1829
|
+
// caught by a stage-specific try/catch lands here. Capture
|
|
1830
|
+
// into a strong local + break out of the do/while(0) wrapper.
|
|
1831
|
+
capturedErrorCode = StitchErrorCode::UnknownCvException;
|
|
1832
|
+
capturedErrorMessage = std::string("OpenCV exception during stitch: ") + e.what();
|
|
1833
|
+
failedInsidePool = true;
|
|
1834
|
+
break;
|
|
1835
|
+
} catch (const std::exception& e) {
|
|
1836
|
+
capturedErrorCode = StitchErrorCode::UnknownCvException;
|
|
1837
|
+
capturedErrorMessage = std::string("std exception during stitch: ") + e.what();
|
|
1838
|
+
failedInsidePool = true;
|
|
1839
|
+
break;
|
|
1840
|
+
} catch (...) {
|
|
1841
|
+
capturedErrorCode = StitchErrorCode::UnknownCvException;
|
|
1842
|
+
capturedErrorMessage = "Unknown exception during stitch.";
|
|
1843
|
+
failedInsidePool = true;
|
|
1844
|
+
break;
|
|
1845
|
+
}
|
|
1846
|
+
} while (0);
|
|
1847
|
+
|
|
1848
|
+
// V16 fix-10 — handle failure paths captured from inside the pool.
|
|
1849
|
+
//
|
|
1850
|
+
// HISTORY (V16 fix-10, 2026-05-13): in the iOS original, the
|
|
1851
|
+
// closing @autoreleasepool brace USED to live at the very bottom
|
|
1852
|
+
// of the function, wrapping the return statement as well. ARC
|
|
1853
|
+
// inserts an autorelease for the return value, which then
|
|
1854
|
+
// registered with this @autoreleasepool; the pool drained at the
|
|
1855
|
+
// closing brace, deallocating the return object BEFORE the
|
|
1856
|
+
// caller could `objc_retain` it.
|
|
1857
|
+
//
|
|
1858
|
+
// Fix-10 restructure: every failure path captures its return
|
|
1859
|
+
// value into a STRONG LOCAL declared above the pool
|
|
1860
|
+
// (`result`/`capturedError`) and `break`s out of the do/while(0)
|
|
1861
|
+
// wrapper to fall past the pool's closing brace cleanly. The
|
|
1862
|
+
// strong locals survive the drain.
|
|
1863
|
+
//
|
|
1864
|
+
// In the pure-C++ port there is no @autoreleasepool — the
|
|
1865
|
+
// do/while(0) wrapper is kept purely for control-flow parity with
|
|
1866
|
+
// the iOS original. C++ stack-locals have proper RAII lifetimes
|
|
1867
|
+
// so the drain UAF is impossible.
|
|
1868
|
+
//
|
|
1869
|
+
// See docs/site-content/learnings/react-native.md#autoreleasepool-return-uaf
|
|
1870
|
+
if (sentinelInsidePool || failedInsidePool) {
|
|
1871
|
+
result.errorCode = capturedErrorCode;
|
|
1872
|
+
result.errorMessage = capturedErrorMessage;
|
|
1873
|
+
const auto t1 = std::chrono::steady_clock::now();
|
|
1874
|
+
result.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
1875
|
+
t1 - t0).count();
|
|
1876
|
+
return result;
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
if (panorama.empty()) {
|
|
1880
|
+
result.errorCode = StitchErrorCode::EmptyPanorama;
|
|
1881
|
+
result.errorMessage = "Stitcher produced an empty panorama.";
|
|
1882
|
+
// framesIncluded was already set above (line ~1640 in the
|
|
1883
|
+
// success-path block). Leave it as-is — it reflects the
|
|
1884
|
+
// count of cameras that fed the blender.
|
|
1885
|
+
return result;
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1888
|
+
// Crop the panorama to the bounding box of non-black pixels.
|
|
1889
|
+
//
|
|
1890
|
+
// The default SphericalWarper from PANORAMA mode lays the
|
|
1891
|
+
// captured patch into a much larger sphere-shaped canvas. For
|
|
1892
|
+
// a typical 30-45° shelf-scan arc, that means the actual scene
|
|
1893
|
+
// occupies a small region of a much larger black-bordered
|
|
1894
|
+
// image (the "panorama bowl" effect). Cropping to the
|
|
1895
|
+
// content's bounding box returns the actual stitched scene
|
|
1896
|
+
// without the surrounding empty bowl. Algorithm:
|
|
1897
|
+
// 1. Convert to grayscale.
|
|
1898
|
+
// 2. Threshold > 1 to find any non-black pixel.
|
|
1899
|
+
// 3. boundingRect of all non-zero pixels.
|
|
1900
|
+
// 4. Crop the panorama to that rect.
|
|
1901
|
+
//
|
|
1902
|
+
// V16 Phase 1b.fix3 — maximum-inscribed-rectangle crop (was bbox).
|
|
1903
|
+
// cv::Stitcher's compose stage produces irregular black corners
|
|
1904
|
+
// where the warped frames didn't fill; cv::boundingRect was
|
|
1905
|
+
// including those. MaxInscribedRectFromMask finds the largest
|
|
1906
|
+
// axis-aligned rectangle entirely inside the non-zero region —
|
|
1907
|
+
// clean output with no black corners. Falls back to bbox
|
|
1908
|
+
// (and ultimately the un-cropped panorama) on any OpenCV failure.
|
|
1909
|
+
//
|
|
1910
|
+
// V16 Phase 1b.fix5 — RCA from Ram's first fix3 capture: the
|
|
1911
|
+
// raw inscribed-rect collapsed to a thin sliver in the
|
|
1912
|
+
// landscape output. Cause: cv::Stitcher's compose produces
|
|
1913
|
+
// small scattered zero-pixels INSIDE the content region (graph-
|
|
1914
|
+
// cut seam, exposure-comp rounding, multi-band blend artifacts).
|
|
1915
|
+
// The inscribed-rect algorithm demands a strictly hole-free
|
|
1916
|
+
// rectangle, so a single interior zero forces it to either
|
|
1917
|
+
// avoid that pixel (collapsing to a thin strip) or skip the
|
|
1918
|
+
// affected row entirely. Python simulation on a realistic
|
|
1919
|
+
// 800×200 mask with 0.5% scattered holes:
|
|
1920
|
+
//
|
|
1921
|
+
// raw inscribed-rect → 23×100 = 1.4% of original (BUG)
|
|
1922
|
+
// after 5×5 close → 642×196 = 78.6% of original (clean)
|
|
1923
|
+
// bounding rect → 800×200 = 100%
|
|
1924
|
+
//
|
|
1925
|
+
// Fix: morphologically CLOSE the mask before the inscribed-rect
|
|
1926
|
+
// search — a 5×5 close fills holes ≤5 px (more than enough for
|
|
1927
|
+
// compose artifacts) without bridging across legitimate concave
|
|
1928
|
+
// gaps (which cv::Stitcher panoramas don't really have). Keep
|
|
1929
|
+
// the bbox safety floor: if the inscribed rect still came out
|
|
1930
|
+
// < 50% of bbox area, use bbox — the mask shape is pathological
|
|
1931
|
+
// and shipping bbox-with-corners is better than a sliver.
|
|
1932
|
+
cv::Mat finalImage = panorama;
|
|
1933
|
+
try {
|
|
1934
|
+
cv::Mat gray;
|
|
1935
|
+
cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
|
|
1936
|
+
cv::Mat mask;
|
|
1937
|
+
cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
|
|
1938
|
+
|
|
1939
|
+
// V16 Phase 1b.fix5c — operator-toggleable crop strategy.
|
|
1940
|
+
//
|
|
1941
|
+
// useInscribedRectCrop = NO (default in settings modal):
|
|
1942
|
+
// Final crop is just cv::boundingRect(mask) — preserves all
|
|
1943
|
+
// stitched content at the cost of possible black corners
|
|
1944
|
+
// where cv::Stitcher's projection didn't fill.
|
|
1945
|
+
//
|
|
1946
|
+
// useInscribedRectCrop = YES (operator opt-in):
|
|
1947
|
+
// Run the full inscribed-rect pipeline (morph-close + 50%
|
|
1948
|
+
// safety floor + column-projection second pass) for a clean
|
|
1949
|
+
// -cornered rectangle. Can over-aggressively shrink the
|
|
1950
|
+
// output on lopsided masks (1146×1102 bbox → 602×1102 strip
|
|
1951
|
+
// in one field log).
|
|
1952
|
+
cv::Rect bbox;
|
|
1953
|
+
if (config.useInscribedRectCrop) {
|
|
1954
|
+
cv::Mat closedMask;
|
|
1955
|
+
cv::morphologyEx(
|
|
1956
|
+
mask, closedMask, cv::MORPH_CLOSE,
|
|
1957
|
+
cv::getStructuringElement(cv::MORPH_RECT, cv::Size(5, 5)));
|
|
1958
|
+
bbox = maxInscribedRectFromMask(closedMask);
|
|
1959
|
+
cv::Rect bboxFallback = cv::boundingRect(mask);
|
|
1960
|
+
const long long inscribedArea =
|
|
1961
|
+
(long long)bbox.width * bbox.height;
|
|
1962
|
+
const long long fallbackArea =
|
|
1963
|
+
(long long)bboxFallback.width * bboxFallback.height;
|
|
1964
|
+
if (bbox.width <= 0 || bbox.height <= 0
|
|
1965
|
+
|| inscribedArea * 2 < fallbackArea) {
|
|
1966
|
+
// Either degenerate, or inscribed < 50% of bbox area.
|
|
1967
|
+
// Safety floor: ship bbox so the operator gets *something*
|
|
1968
|
+
// usable (legacy behaviour pre-fix3) rather than a sliver.
|
|
1969
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1970
|
+
"inscribed-rect rejected: "
|
|
1971
|
+
"%dx%d (area=%lld) vs bbox %dx%d (area=%lld); "
|
|
1972
|
+
"using bbox fallback.",
|
|
1973
|
+
bbox.width, bbox.height, inscribedArea,
|
|
1974
|
+
bboxFallback.width, bboxFallback.height, fallbackArea);
|
|
1975
|
+
bbox = bboxFallback;
|
|
1976
|
+
} else {
|
|
1977
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1978
|
+
"inscribed-rect: %dx%d "
|
|
1979
|
+
"(area=%lld, %.0f%% of bbox %dx%d)",
|
|
1980
|
+
bbox.width, bbox.height, inscribedArea,
|
|
1981
|
+
100.0 * (double)inscribedArea / (double)fallbackArea,
|
|
1982
|
+
bboxFallback.width, bboxFallback.height);
|
|
1983
|
+
}
|
|
1984
|
+
} else {
|
|
1985
|
+
bbox = cv::boundingRect(mask);
|
|
1986
|
+
log_info(logFn, "[BatchStitcher]",
|
|
1987
|
+
"crop: bbox-only %dx%d "
|
|
1988
|
+
"(useInscribedRectCrop=NO via setting)",
|
|
1989
|
+
bbox.width, bbox.height);
|
|
1990
|
+
}
|
|
1991
|
+
if (bbox.width > 0 && bbox.height > 0
|
|
1992
|
+
&& bbox.width <= panorama.cols && bbox.height <= panorama.rows) {
|
|
1993
|
+
finalImage = panorama(bbox).clone();
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
// V16 Phase 1b.fix5c — column-projection second pass ALSO gated
|
|
1997
|
+
// on the inscribed-rect toggle. When OFF, skip directly to the
|
|
1998
|
+
// write so the operator sees the full bbox-cropped panorama
|
|
1999
|
+
// without further trimming. When ON, keep the existing
|
|
2000
|
+
// 95%-then-80%-then-skip relaxation chain.
|
|
2001
|
+
if (config.useInscribedRectCrop) {
|
|
2002
|
+
// Second pass: rectangular crop. Find the column range where
|
|
2003
|
+
// ≥95% of rows have content, crop to that × full height.
|
|
2004
|
+
cv::Mat finalGray;
|
|
2005
|
+
cv::cvtColor(finalImage, finalGray, cv::COLOR_BGR2GRAY);
|
|
2006
|
+
cv::Mat finalMask;
|
|
2007
|
+
cv::threshold(finalGray, finalMask, 30, 255, cv::THRESH_BINARY);
|
|
2008
|
+
cv::erode(finalMask, finalMask,
|
|
2009
|
+
cv::getStructuringElement(cv::MORPH_RECT, cv::Size(5, 5)),
|
|
2010
|
+
cv::Point(-1, -1), 1);
|
|
2011
|
+
|
|
2012
|
+
int rows = finalMask.rows, cols = finalMask.cols;
|
|
2013
|
+
// Reduce mask to per-column content count. Mask is 0 or 255,
|
|
2014
|
+
// so column sum / 255 = number of content rows in that column.
|
|
2015
|
+
cv::Mat colSum;
|
|
2016
|
+
cv::reduce(finalMask, colSum, 0, cv::REDUCE_SUM, CV_32S);
|
|
2017
|
+
const int contentThreshold = (int)(0.95 * rows * 255);
|
|
2018
|
+
int cropLeft = -1, cropRight = -1;
|
|
2019
|
+
const int* cs = colSum.ptr<int>(0);
|
|
2020
|
+
for (int c = 0; c < cols; c++) {
|
|
2021
|
+
if (cs[c] >= contentThreshold) {
|
|
2022
|
+
if (cropLeft < 0) cropLeft = c;
|
|
2023
|
+
cropRight = c;
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
2026
|
+
log_info(logFn, "[BatchStitcher]",
|
|
2027
|
+
"rectCrop col-proj: cols=%d rows=%d threshold=%d cropLeft=%d cropRight=%d",
|
|
2028
|
+
cols, rows, contentThreshold, cropLeft, cropRight);
|
|
2029
|
+
// Sanity floor: don't accept a column-projection crop that
|
|
2030
|
+
// shrinks the image to less than 30% of the bbox-cropped width.
|
|
2031
|
+
// Such an aggressive crop usually means the stitch was poorly
|
|
2032
|
+
// aligned and only a tiny vertical band has full multi-frame
|
|
2033
|
+
// coverage — applying it produces the "thin sliver" output
|
|
2034
|
+
// we observed in the field. Better to show the user the full
|
|
2035
|
+
// bounding-box crop (still trims the all-black borders) than
|
|
2036
|
+
// a sliver that's effectively useless.
|
|
2037
|
+
const int minRectWidth = (int)(cols * 0.30);
|
|
2038
|
+
if (cropLeft >= 0 && cropRight > cropLeft + 10
|
|
2039
|
+
&& (cropRight - cropLeft + 1) >= minRectWidth) {
|
|
2040
|
+
cv::Rect rectCrop(cropLeft, 0,
|
|
2041
|
+
cropRight - cropLeft + 1, rows);
|
|
2042
|
+
finalImage = finalImage(rectCrop).clone();
|
|
2043
|
+
log_info(logFn, "[BatchStitcher]",
|
|
2044
|
+
"rectCrop applied: %dx%d → %dx%d",
|
|
2045
|
+
cols, rows, finalImage.cols, finalImage.rows);
|
|
2046
|
+
} else {
|
|
2047
|
+
// No column qualified at 95%, OR the qualifying band is too
|
|
2048
|
+
// narrow to trust. Try a relaxed 80% before giving up.
|
|
2049
|
+
const int relaxedThreshold = (int)(0.80 * rows * 255);
|
|
2050
|
+
cropLeft = -1;
|
|
2051
|
+
cropRight = -1;
|
|
2052
|
+
for (int c = 0; c < cols; c++) {
|
|
2053
|
+
if (cs[c] >= relaxedThreshold) {
|
|
2054
|
+
if (cropLeft < 0) cropLeft = c;
|
|
2055
|
+
cropRight = c;
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
log_info(logFn, "[BatchStitcher]",
|
|
2059
|
+
"rectCrop relaxed (80%%): cropLeft=%d cropRight=%d",
|
|
2060
|
+
cropLeft, cropRight);
|
|
2061
|
+
if (cropLeft >= 0 && cropRight > cropLeft + 10
|
|
2062
|
+
&& (cropRight - cropLeft + 1) >= minRectWidth) {
|
|
2063
|
+
cv::Rect rectCrop(cropLeft, 0,
|
|
2064
|
+
cropRight - cropLeft + 1, rows);
|
|
2065
|
+
finalImage = finalImage(rectCrop).clone();
|
|
2066
|
+
log_info(logFn, "[BatchStitcher]",
|
|
2067
|
+
"rectCrop relaxed applied: %dx%d → %dx%d",
|
|
2068
|
+
cols, rows, finalImage.cols, finalImage.rows);
|
|
2069
|
+
} else {
|
|
2070
|
+
log_info(logFn, "[BatchStitcher]",
|
|
2071
|
+
"rectCrop SKIPPED — best band is "
|
|
2072
|
+
"narrower than 30%% of bbox (%d < %d). Likely poor "
|
|
2073
|
+
"stitch alignment; keeping bbox crop.",
|
|
2074
|
+
cropRight >= 0 ? (cropRight - cropLeft + 1) : 0,
|
|
2075
|
+
minRectWidth);
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
} catch (...) {
|
|
2080
|
+
// Crop failed — fall back to the raw stitched output.
|
|
2081
|
+
finalImage = panorama;
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
// AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
|
|
2085
|
+
//
|
|
2086
|
+
// Bake-rotation driven by the user's phone-hold orientation at
|
|
2087
|
+
// capture start, NOT the output Mat's aspect ratio. Reasoning:
|
|
2088
|
+
//
|
|
2089
|
+
// - fix5d's earlier attempt to key off the same orientation used
|
|
2090
|
+
// `exifOrientation:NSInteger` (an EXIF tag 1/3/6/8), inferred
|
|
2091
|
+
// from frameRotationDegrees, which collapsed landscape-left
|
|
2092
|
+
// and landscape-right to the same value. Ram's reports made
|
|
2093
|
+
// it clear the two landscape variants need OPPOSITE rotations
|
|
2094
|
+
// (they're mirror images of each other w.r.t. the sensor's
|
|
2095
|
+
// world-up direction), so the EXIF-tag intermediary was lossy.
|
|
2096
|
+
//
|
|
2097
|
+
// - fix5e's aspect-ratio approach was correct for 3+ frame
|
|
2098
|
+
// horizontal pans but the threshold "cols > rows" was fragile
|
|
2099
|
+
// at 2 frames (output Mat near-square or even tall) and
|
|
2100
|
+
// conflated "wide because horizontal pan" with "tall because
|
|
2101
|
+
// vertical pan." The user spec explicitly lists two modes;
|
|
2102
|
+
// using their classification is more robust than guessing
|
|
2103
|
+
// from output geometry.
|
|
2104
|
+
//
|
|
2105
|
+
// The two supported modes:
|
|
2106
|
+
//
|
|
2107
|
+
// Mode A — landscape phone + vertical pan from top
|
|
2108
|
+
// landscape-left → ROTATE_90_COUNTERCLOCKWISE
|
|
2109
|
+
// landscape-right → ROTATE_90_CLOCKWISE
|
|
2110
|
+
// (mirror-image directions because world-up sits on opposite
|
|
2111
|
+
// sensor edges between landscape-left and landscape-right;
|
|
2112
|
+
// opposite rotations land world-up at output-top for both)
|
|
2113
|
+
//
|
|
2114
|
+
// Mode B — portrait phone + horizontal pan from left
|
|
2115
|
+
// portrait → no rotation (cv::Stitcher's natural
|
|
2116
|
+
// output already aligns world-up to
|
|
2117
|
+
// output-top for portrait hold)
|
|
2118
|
+
// portrait-upside-down → ROTATE_180
|
|
2119
|
+
//
|
|
2120
|
+
// Anything else: best-effort no rotation. Unsupported combination
|
|
2121
|
+
// (e.g., portrait phone + vertical pan) is treated as Mode B.
|
|
2122
|
+
//
|
|
2123
|
+
// Properties:
|
|
2124
|
+
// - Compose canvas geometry unchanged from baseline 437c763:
|
|
2125
|
+
// cv::imread default applies EXIF rotation at load time,
|
|
2126
|
+
// producing portrait Mats for portrait hold and landscape
|
|
2127
|
+
// Mats for landscape hold. No fix5b-style 6-frame OOM.
|
|
2128
|
+
// - Output JPEG always EXIF=1 in the iOS original (ImageIO
|
|
2129
|
+
// writer with kCGImagePropertyOrientation=1). In the shared
|
|
2130
|
+
// port we use cv::imwrite which doesn't write EXIF — the
|
|
2131
|
+
// pixels are already rotated correctly, so the visual result
|
|
2132
|
+
// matches. See TODO[shared-stitcher-port-part-2] for a
|
|
2133
|
+
// proper EXIF-aware writer if iOS callers report viewers
|
|
2134
|
+
// that ignore the pixel rotation.
|
|
2135
|
+
// - The cv::rotate happens AFTER BA / blend / seam-find when
|
|
2136
|
+
// their working sets are released — incremental memory cost.
|
|
2137
|
+
// - Per-keyframe JPEGs (OpenCVKeyframeCollector) untouched —
|
|
2138
|
+
// they still carry EXIF=6 so LiveFrameStrip thumbnails show
|
|
2139
|
+
// portrait-correct during capture.
|
|
2140
|
+
//
|
|
2141
|
+
// Empirically calibrated (Ram's 2026-05-11 test, iteration 2):
|
|
2142
|
+
// Iteration 1 swapped both the labels AND the directions — net
|
|
2143
|
+
// visual rotation per roll-value was unchanged (output still
|
|
2144
|
+
// looked "landscape-left oriented" to Ram). Iteration 2 flips
|
|
2145
|
+
// ONLY the directions; labels stay where they landed.
|
|
2146
|
+
// landscape-left (roll ≈ -90°, Ram's L-left hold) → 90° CCW
|
|
2147
|
+
// landscape-right (roll ≈ +90°, Ram's L-right hold) → 90° CW
|
|
2148
|
+
// For a roll=-90° capture (what Ram tested), this rotates the
|
|
2149
|
+
// OPPOSITE direction from iteration 1. If iteration 1 put
|
|
2150
|
+
// scene-up on the LEFT of the tall image, iteration 2 will put
|
|
2151
|
+
// scene-up on the RIGHT.
|
|
2152
|
+
// 2026-05-18 (Iss #1 diag): log pre-bake Mat shape so we can
|
|
2153
|
+
// tell, from a device-log dump alone, whether the stitcher output
|
|
2154
|
+
// is landscape-aspect or portrait-aspect BEFORE the rotation is
|
|
2155
|
+
// applied. bake_rotation already logs the rotated path's input
|
|
2156
|
+
// and output dims; the no-rotation branch logs only one pair.
|
|
2157
|
+
// Either way, this line is the source-of-truth for the pre-bake
|
|
2158
|
+
// shape and the captureOrientation that will be matched against.
|
|
2159
|
+
log_info(logFn, "[stitch]",
|
|
2160
|
+
"pre-bake finalImage %dx%d orientation=%s",
|
|
2161
|
+
finalImage.cols, finalImage.rows,
|
|
2162
|
+
config.captureOrientation.c_str());
|
|
2163
|
+
cv::Mat finalImageRotated = bake_rotation(finalImage,
|
|
2164
|
+
config.captureOrientation,
|
|
2165
|
+
logFn);
|
|
2166
|
+
|
|
2167
|
+
// Encode + write the JPEG. Clamp quality into [0, 100] to defend
|
|
2168
|
+
// against caller bugs.
|
|
2169
|
+
//
|
|
2170
|
+
// V16 Phase 1b.fix3 (iOS original) — write via ImageIO so we can
|
|
2171
|
+
// bake the EXIF Orientation tag into the output. cv::imwrite
|
|
2172
|
+
// produces a plain JPEG with no metadata. In the shared port we
|
|
2173
|
+
// rely on `bake_rotation` rotating pixels in-place above, so the
|
|
2174
|
+
// EXIF tag is unnecessary for correct display — kept as a TODO
|
|
2175
|
+
// below in case downstream consumers expect EXIF=1.
|
|
2176
|
+
const int q = std::max(0, std::min(100, config.jpegQuality));
|
|
2177
|
+
std::vector<int> params{cv::IMWRITE_JPEG_QUALITY, q};
|
|
2178
|
+
bool wrote = false;
|
|
2179
|
+
try {
|
|
2180
|
+
wrote = cv::imwrite(outputPath, finalImageRotated, params);
|
|
2181
|
+
} catch (const cv::Exception& e) {
|
|
2182
|
+
result.errorCode = StitchErrorCode::ImageWriteFailed;
|
|
2183
|
+
result.errorMessage = std::string("cv::imwrite threw: ") + e.what();
|
|
2184
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
2185
|
+
return result;
|
|
2186
|
+
}
|
|
2187
|
+
if (!wrote) {
|
|
2188
|
+
result.errorCode = StitchErrorCode::ImageWriteFailed;
|
|
2189
|
+
result.errorMessage = "Stitch succeeded but could not write JPEG to " + outputPath;
|
|
2190
|
+
log_error(logFn, "[stitch]", "%s", result.errorMessage.c_str());
|
|
2191
|
+
return result;
|
|
2192
|
+
}
|
|
2193
|
+
|
|
2194
|
+
// V16 Phase 1b.fix5d — report the dimensions of the bytes we
|
|
2195
|
+
// actually wrote (rotated, if we baked one in above), not the
|
|
2196
|
+
// pre-rotate Mat. JS-side consumers need the displayable shape.
|
|
2197
|
+
const auto t1 = std::chrono::steady_clock::now();
|
|
2198
|
+
result.success = true;
|
|
2199
|
+
result.errorCode = StitchErrorCode::Ok;
|
|
2200
|
+
result.width = finalImageRotated.cols;
|
|
2201
|
+
result.height = finalImageRotated.rows;
|
|
2202
|
+
result.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
2203
|
+
t1 - t0).count();
|
|
2204
|
+
return result;
|
|
2205
|
+
}
|
|
2206
|
+
|
|
2207
|
+
} // namespace retailens
|