react-native-image-stitcher 0.14.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +131 -0
- package/README.md +35 -0
- package/RNImageStitcher.podspec +8 -7
- package/android/build.gradle +0 -16
- package/android/src/main/cpp/CMakeLists.txt +2 -63
- package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
- package/cpp/keyframe_gate.cpp +82 -23
- package/cpp/keyframe_gate.hpp +31 -2
- package/cpp/stitcher.cpp +208 -28
- package/cpp/tests/CMakeLists.txt +18 -12
- package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
- package/cpp/tests/warp_guard_test.cpp +48 -0
- package/cpp/warp_guard.hpp +41 -0
- package/dist/camera/Camera.d.ts +31 -16
- package/dist/camera/Camera.js +10 -2
- package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
- package/dist/camera/CaptureStitchStatsToast.js +27 -7
- package/dist/camera/PanoramaSettings.d.ts +10 -223
- package/dist/camera/PanoramaSettings.js +6 -28
- package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
- package/dist/camera/PanoramaSettingsBridge.js +3 -102
- package/dist/camera/PanoramaSettingsModal.js +7 -1
- package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
- package/dist/camera/buildPanoramaInitialSettings.js +4 -0
- package/dist/camera/cameraErrorMessages.d.ts +32 -0
- package/dist/camera/cameraErrorMessages.js +53 -0
- package/dist/camera/selectCaptureDevice.d.ts +5 -1
- package/dist/camera/selectCaptureDevice.js +22 -2
- package/dist/camera/useCapture.js +38 -0
- package/dist/index.d.ts +5 -8
- package/dist/index.js +11 -34
- package/dist/stitching/incremental.d.ts +1 -117
- package/dist/stitching/stitchVideo.d.ts +0 -35
- package/dist/types.d.ts +0 -87
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
- package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
- package/package.json +3 -2
- package/src/camera/Camera.tsx +43 -22
- package/src/camera/CaptureStitchStatsToast.tsx +58 -14
- package/src/camera/PanoramaSettings.ts +16 -289
- package/src/camera/PanoramaSettingsBridge.ts +3 -114
- package/src/camera/PanoramaSettingsModal.tsx +14 -1
- package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
- package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
- package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
- package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
- package/src/camera/buildPanoramaInitialSettings.ts +17 -0
- package/src/camera/cameraErrorMessages.ts +84 -0
- package/src/camera/selectCaptureDevice.ts +28 -3
- package/src/camera/useCapture.ts +44 -1
- package/src/index.ts +11 -40
- package/src/stitching/incremental.ts +3 -140
- package/src/stitching/stitchVideo.ts +0 -26
- package/src/types.ts +0 -95
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
- package/cpp/stitcher_frame_jsi.cpp +0 -214
- package/cpp/stitcher_frame_jsi.hpp +0 -108
- package/cpp/stitcher_proxy_jsi.cpp +0 -109
- package/cpp/stitcher_proxy_jsi.hpp +0 -46
- package/cpp/stitcher_worklet_dispatch.cpp +0 -103
- package/cpp/stitcher_worklet_dispatch.hpp +0 -71
- package/cpp/stitcher_worklet_registry.cpp +0 -91
- package/cpp/stitcher_worklet_registry.hpp +0 -146
- package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
- package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
- package/dist/stitching/IncrementalStitcherView.js +0 -157
- package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
- package/dist/stitching/StitcherWorkletRegistry.js +0 -78
- package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
- package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
- package/dist/stitching/useFrameProcessor.d.ts +0 -119
- package/dist/stitching/useFrameProcessor.js +0 -196
- package/dist/stitching/useFrameStream.d.ts +0 -34
- package/dist/stitching/useFrameStream.js +0 -234
- package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
- package/dist/stitching/useThrottledFrameProcessor.js +0 -132
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
- package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
- package/src/stitching/IncrementalStitcherView.tsx +0 -198
- package/src/stitching/StitcherWorkletRegistry.ts +0 -156
- package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
- package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
- package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
- package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
- package/src/stitching/useFrameProcessor.ts +0 -226
- package/src/stitching/useFrameStream.ts +0 -271
- package/src/stitching/useThrottledFrameProcessor.ts +0 -145
|
@@ -169,6 +169,28 @@ static cv::Rect MaxInscribedRectFromMask(const cv::Mat &mask) {
|
|
|
169
169
|
return bestRect;
|
|
170
170
|
}
|
|
171
171
|
|
|
172
|
+
// v0.15 — fill interior holes of a content mask so that ONLY black
|
|
173
|
+
// connected to the image BORDER (the never-covered projection wedges)
|
|
174
|
+
// stays excluded. Dark image content (unlit furniture, shadow) forms
|
|
175
|
+
// INTERIOR holes surrounded by content — those are filled back in.
|
|
176
|
+
// This is the pixel-based proxy for true frame coverage, which the
|
|
177
|
+
// shared high-level cv::Stitcher path doesn't expose.
|
|
178
|
+
static cv::Mat FillBorderConnectedHoles(const cv::Mat &mask) {
|
|
179
|
+
// Pad a 1px black border so the exterior is one connected region,
|
|
180
|
+
// then flood the border-connected black to white from the corner.
|
|
181
|
+
cv::Mat padded;
|
|
182
|
+
cv::copyMakeBorder(mask, padded, 1, 1, 1, 1, cv::BORDER_CONSTANT, cv::Scalar(0));
|
|
183
|
+
cv::floodFill(padded, cv::Point(0, 0), cv::Scalar(255));
|
|
184
|
+
cv::Mat exterior = padded(cv::Rect(1, 1, mask.cols, mask.rows));
|
|
185
|
+
// Pixels still 0 after the flood are interior holes (never reached
|
|
186
|
+
// from the border) → real content to keep.
|
|
187
|
+
cv::Mat holes;
|
|
188
|
+
cv::bitwise_not(exterior, holes);
|
|
189
|
+
cv::Mat filled;
|
|
190
|
+
cv::bitwise_or(mask, holes, filled);
|
|
191
|
+
return filled;
|
|
192
|
+
}
|
|
193
|
+
|
|
172
194
|
|
|
173
195
|
// V16 Phase 1b.fix3 — write a cv::Mat (BGR) as a JPEG with an EXIF
|
|
174
196
|
// Orientation tag, via ImageIO. iOS image renderers (UIImage,
|
|
@@ -304,7 +326,7 @@ NSString *normalizeImagePath(NSString *path) {
|
|
|
304
326
|
// - error mapping: the explicit StitchErrorCode → NSError.code
|
|
305
327
|
// switch in this file at the new wrapper (lines ~528-595)
|
|
306
328
|
// Removed to keep the anonymous namespace tight; sibling methods
|
|
307
|
-
// (
|
|
329
|
+
// (stitchFramePaths, stitchVideoAtPath) don't need them.
|
|
308
330
|
|
|
309
331
|
// Phase 5: build a cv::detail::CameraParams from an ARKit pose.
|
|
310
332
|
//
|
|
@@ -421,10 +443,16 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
|
|
|
421
443
|
if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
|
|
422
444
|
if (captureOrientation == nil || captureOrientation.length == 0) captureOrientation = @"portrait";
|
|
423
445
|
|
|
424
|
-
// Build the shared-C++ config.
|
|
425
|
-
//
|
|
426
|
-
//
|
|
446
|
+
// Build the shared-C++ config.
|
|
447
|
+
//
|
|
448
|
+
// 2026-06-06 (parity — see docs/stitch-pipeline-architecture.md §3/§7):
|
|
449
|
+
// explicitly match the high-level / Android resolution budget instead of
|
|
450
|
+
// leaving the sentinel -1.0, which made the manual entry point fall back
|
|
451
|
+
// to its LOW defaults (registration 0.3 MP / compose 0.6 MP — half /
|
|
452
|
+
// 0.6x Android's 0.6 / 1.0 MP, a major reason iOS output looked softer).
|
|
427
453
|
retailens::StitchConfig cfg;
|
|
454
|
+
cfg.registrationResolMP = 0.6; // cv::Stitcher default
|
|
455
|
+
cfg.compositingResolMP = 1.0; // high-level default (manual was 0.6)
|
|
428
456
|
cfg.warperType = warperType.UTF8String;
|
|
429
457
|
cfg.blenderType = blenderType.UTF8String;
|
|
430
458
|
cfg.seamFinderType = seamFinderType.UTF8String;
|
|
@@ -848,998 +876,196 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
|
|
|
848
876
|
|
|
849
877
|
|
|
850
878
|
// ─────────────────────────────────────────────────────────────────────
|
|
851
|
-
//
|
|
879
|
+
// Photo orientation normalisation
|
|
852
880
|
// ─────────────────────────────────────────────────────────────────────
|
|
853
|
-
//
|
|
854
|
-
//
|
|
855
|
-
//
|
|
856
|
-
//
|
|
857
|
-
// → BundleAdjuster steps that the feature-matched path runs.
|
|
858
|
-
// The compose stage (warp + seam + blend + crop) is duplicated
|
|
859
|
-
// from `stitchFramePaths` rather than refactored — keeps the
|
|
860
|
-
// hard-won existing pipeline untouched while we field-test the
|
|
861
|
-
// pose path; both paths can be DRY'd into a shared helper once
|
|
862
|
-
// the new code is proven on real shelf captures.
|
|
863
|
-
|
|
864
|
-
+ (nullable RNStitchResult *)stitchVideoAtPath:(NSString *)videoPath
|
|
865
|
-
outputPath:(NSString *)outputPath
|
|
866
|
-
maxFrames:(NSInteger)maxFrames
|
|
867
|
-
jpegQuality:(NSInteger)quality
|
|
868
|
-
warperType:(NSString *)warperType
|
|
869
|
-
blenderType:(NSString *)blenderType
|
|
870
|
-
seamFinderType:(NSString *)seamFinderType
|
|
871
|
-
poses:(NSArray<NSDictionary *> *)poses
|
|
872
|
-
error:(NSError **)error {
|
|
873
|
-
if (warperType == nil || warperType.length == 0) warperType = @"plane";
|
|
874
|
-
if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
|
|
875
|
-
if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
|
|
876
|
-
if (poses.count < 2) {
|
|
877
|
-
if (error) {
|
|
878
|
-
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
879
|
-
code:1030
|
|
880
|
-
userInfo:@{
|
|
881
|
-
NSLocalizedDescriptionKey:
|
|
882
|
-
@"Pose-driven stitch needs at least 2 poses; got fewer.",
|
|
883
|
-
}];
|
|
884
|
-
}
|
|
885
|
-
return nil;
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
NSString *tmpDir =
|
|
889
|
-
[NSTemporaryDirectory() stringByAppendingPathComponent:
|
|
890
|
-
[NSString stringWithFormat:@"RNImageStitcherStitchAR-%@",
|
|
891
|
-
[[NSUUID UUID] UUIDString]]];
|
|
892
|
-
|
|
893
|
-
// Extract evenly-spaced frames from the video (same helper the
|
|
894
|
-
// feature-matched path uses). Returns paths only; we'll compute
|
|
895
|
-
// each frame's timestamp ourselves to match against `poses`.
|
|
896
|
-
NSError *extractErr = nil;
|
|
897
|
-
NSArray<NSString *> *framePaths =
|
|
898
|
-
[self extractFramesFromVideoAtPath:videoPath
|
|
899
|
-
outputDir:tmpDir
|
|
900
|
-
maxFrames:maxFrames
|
|
901
|
-
jpegQuality:quality
|
|
902
|
-
error:&extractErr];
|
|
903
|
-
if (!framePaths) {
|
|
904
|
-
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
905
|
-
if (error) *error = extractErr;
|
|
906
|
-
return nil;
|
|
907
|
-
}
|
|
908
|
-
|
|
909
|
-
// Compute total video duration so frame timestamps match what
|
|
910
|
-
// the AR session captured. Pose timestamps are in absolute ms;
|
|
911
|
-
// we normalise against poses[0] so they align with the mp4
|
|
912
|
-
// timeline (which AVAssetWriter wrote starting at 0).
|
|
913
|
-
NSURL *videoURL = [NSURL fileURLWithPath:
|
|
914
|
-
([videoPath hasPrefix:@"file://"]
|
|
915
|
-
? [videoPath substringFromIndex:[@"file://" length]]
|
|
916
|
-
: videoPath)];
|
|
917
|
-
AVURLAsset *asset = [AVURLAsset assetWithURL:videoURL];
|
|
918
|
-
Float64 totalSeconds = CMTimeGetSeconds(asset.duration);
|
|
919
|
-
if (!isfinite(totalSeconds) || totalSeconds <= 0) {
|
|
920
|
-
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
921
|
-
if (error) {
|
|
922
|
-
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
923
|
-
code:1031
|
|
924
|
-
userInfo:@{
|
|
925
|
-
NSLocalizedDescriptionKey:
|
|
926
|
-
@"Could not read video duration for pose-time alignment.",
|
|
927
|
-
}];
|
|
928
|
-
}
|
|
929
|
-
return nil;
|
|
930
|
-
}
|
|
931
|
-
double baseMs = [poses[0][@"timestampMs"] doubleValue];
|
|
932
|
-
|
|
933
|
-
// Match each extracted frame to its closest pose by timestamp.
|
|
934
|
-
// Tolerance is 100 ms — at 60 Hz pose log + 30 fps frame extract,
|
|
935
|
-
// worst case is ~17 ms drift, plenty of headroom.
|
|
936
|
-
NSInteger N = (NSInteger)framePaths.count;
|
|
937
|
-
std::vector<cv::Mat> frames;
|
|
938
|
-
std::vector<cv::detail::CameraParams> cameras;
|
|
939
|
-
frames.reserve(N);
|
|
940
|
-
cameras.reserve(N);
|
|
941
|
-
int matched = 0, dropped = 0;
|
|
942
|
-
for (NSInteger i = 0; i < N; i++) {
|
|
943
|
-
Float64 fraction = (N == 1) ? 0.0 : ((Float64)i / (Float64)(N - 1));
|
|
944
|
-
Float64 frameTimeMs = fraction * totalSeconds * 1000.0;
|
|
945
|
-
|
|
946
|
-
NSDictionary *bestPose = nil;
|
|
947
|
-
double bestDelta = INFINITY;
|
|
948
|
-
for (NSDictionary *pose in poses) {
|
|
949
|
-
double poseMs = [pose[@"timestampMs"] doubleValue] - baseMs;
|
|
950
|
-
double delta = fabs(poseMs - frameTimeMs);
|
|
951
|
-
if (delta < bestDelta) {
|
|
952
|
-
bestDelta = delta;
|
|
953
|
-
bestPose = pose;
|
|
954
|
-
}
|
|
955
|
-
}
|
|
956
|
-
if (!bestPose || bestDelta > 100.0) {
|
|
957
|
-
dropped++;
|
|
958
|
-
continue;
|
|
959
|
-
}
|
|
960
|
-
// V16 Phase 1.fix3 — IMREAD_IGNORE_ORIENTATION parity with the
|
|
961
|
-
// batch-keyframe path. AVAssetImageGenerator writes JPEGs with
|
|
962
|
-
// EXIF Orientation tags; cv::imread defaults (OpenCV 4.5+) apply
|
|
963
|
-
// them, returning rotated pixels that don't match the pose's
|
|
964
|
-
// intrinsics (which describe the unrotated landscape sensor).
|
|
965
|
-
// Force raw landscape pixels for the stitcher.
|
|
966
|
-
cv::Mat img = cv::imread([framePaths[i] UTF8String],
|
|
967
|
-
cv::IMREAD_COLOR | cv::IMREAD_IGNORE_ORIENTATION);
|
|
968
|
-
if (img.empty()) {
|
|
969
|
-
dropped++;
|
|
970
|
-
continue;
|
|
971
|
-
}
|
|
972
|
-
frames.push_back(img);
|
|
973
|
-
cameras.push_back(cameraParamsFromPose(bestPose));
|
|
974
|
-
matched++;
|
|
975
|
-
}
|
|
976
|
-
NSLog(@"[BatchStitcher] pose-driven: matched=%d dropped=%d",
|
|
977
|
-
matched, dropped);
|
|
978
|
-
|
|
979
|
-
if (frames.size() < 2) {
|
|
980
|
-
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
981
|
-
if (error) {
|
|
982
|
-
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
983
|
-
code:1032
|
|
984
|
-
userInfo:@{
|
|
985
|
-
NSLocalizedDescriptionKey:
|
|
986
|
-
@"Fewer than 2 frames matched a pose within tolerance — "
|
|
987
|
-
"AR tracking may have been lost during the pan.",
|
|
988
|
-
}];
|
|
989
|
-
}
|
|
990
|
-
return nil;
|
|
991
|
-
}
|
|
992
|
-
|
|
993
|
-
auto t0 = std::chrono::steady_clock::now();
|
|
994
|
-
cv::Mat panorama;
|
|
995
|
-
|
|
996
|
-
@autoreleasepool {
|
|
997
|
-
try {
|
|
998
|
-
// Pose-driven path: cameras already populated. intrinsics are
|
|
999
|
-
// at the source frame's native resolution, so work_scale = 1.0.
|
|
1000
|
-
int origCols = frames[0].cols;
|
|
1001
|
-
int origRows = frames[0].rows;
|
|
1002
|
-
double origMp = (double)origCols * origRows / 1e6;
|
|
1003
|
-
constexpr double COMPOSE_MP = 1.0;
|
|
1004
|
-
double compose_scale = (origMp > COMPOSE_MP)
|
|
1005
|
-
? std::sqrt(COMPOSE_MP / origMp)
|
|
1006
|
-
: 1.0;
|
|
1007
|
-
double compose_work_aspect = compose_scale; // work_scale == 1
|
|
1008
|
-
|
|
1009
|
-
// No camera-0 normalisation in the pose-driven path.
|
|
1010
|
-
//
|
|
1011
|
-
// I added one previously thinking it matched cv::Stitcher's BA
|
|
1012
|
-
// convention. In fact it BROKE the natural orientation: BA
|
|
1013
|
-
// normalises into a frame where camera 0's "up" is the panorama
|
|
1014
|
-
// up; for pose-driven, the cameras already live in ARKit's
|
|
1015
|
-
// gravity-aligned world (Y-up = scene up regardless of phone
|
|
1016
|
-
// orientation), so passing R values in ARKit's world frame is
|
|
1017
|
-
// exactly what cv::detail::SphericalWarper wants — it unwraps
|
|
1018
|
-
// the sphere with world's +Y as up, giving correct orientation
|
|
1019
|
-
// for any phone pose + any pan direction. Normalising rotated
|
|
1020
|
-
// the panorama 90° (the user's left-to-right pan in portrait
|
|
1021
|
-
// came out with natural-up on the side).
|
|
1022
|
-
//
|
|
1023
|
-
// waveCorrect below provides the per-camera fine alignment that
|
|
1024
|
-
// BA would have done in the feature-matched path.
|
|
1025
|
-
|
|
1026
|
-
// Optional waveCorrect — uses HORIZ to match the feature-
|
|
1027
|
-
// matched path. Operators may pan in any direction; HORIZ
|
|
1028
|
-
// aligns each camera's "up" to the world Y axis (gravity),
|
|
1029
|
-
// which is what we want for both portrait+horizontal and
|
|
1030
|
-
// landscape+vertical pans (assuming the user keeps the phone
|
|
1031
|
-
// oriented to gravity, which is the typical handheld case).
|
|
1032
|
-
std::vector<cv::Mat> rmats;
|
|
1033
|
-
rmats.reserve(cameras.size());
|
|
1034
|
-
for (const auto &cam : cameras) rmats.push_back(cam.R.clone());
|
|
1035
|
-
try {
|
|
1036
|
-
cv::detail::waveCorrect(rmats, cv::detail::WAVE_CORRECT_HORIZ);
|
|
1037
|
-
for (size_t i = 0; i < cameras.size(); i++) {
|
|
1038
|
-
cameras[i].R = rmats[i];
|
|
1039
|
-
}
|
|
1040
|
-
} catch (const cv::Exception &e) {
|
|
1041
|
-
NSLog(@"[BatchStitcher] pose: wave correction skipped: %s", e.what());
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
// Rescale intrinsics for compose-scale warping.
|
|
1045
|
-
for (auto &cam : cameras) {
|
|
1046
|
-
cam.focal *= compose_work_aspect;
|
|
1047
|
-
cam.ppx *= compose_work_aspect;
|
|
1048
|
-
cam.ppy *= compose_work_aspect;
|
|
1049
|
-
}
|
|
1050
|
-
|
|
1051
|
-
std::vector<double> focals;
|
|
1052
|
-
for (const auto &cam : cameras) focals.push_back(cam.focal);
|
|
1053
|
-
std::sort(focals.begin(), focals.end());
|
|
1054
|
-
float warpedScale = focals.empty() ? 1.0f
|
|
1055
|
-
: (float)focals[focals.size() / 2];
|
|
1056
|
-
|
|
1057
|
-
cv::Ptr<cv::WarperCreator> warperCreator;
|
|
1058
|
-
if ([warperType isEqualToString:@"cylindrical"]) {
|
|
1059
|
-
warperCreator = cv::makePtr<cv::CylindricalWarper>();
|
|
1060
|
-
} else if ([warperType isEqualToString:@"spherical"]) {
|
|
1061
|
-
warperCreator = cv::makePtr<cv::SphericalWarper>();
|
|
1062
|
-
} else {
|
|
1063
|
-
warperCreator = cv::makePtr<cv::PlaneWarper>();
|
|
1064
|
-
}
|
|
1065
|
-
cv::Ptr<cv::detail::RotationWarper> warper =
|
|
1066
|
-
warperCreator->create(warpedScale);
|
|
1067
|
-
|
|
1068
|
-
// Build composeFrames at COMPOSE_MP from full-res input.
|
|
1069
|
-
std::vector<cv::Mat> composeFrames;
|
|
1070
|
-
composeFrames.reserve(frames.size());
|
|
1071
|
-
for (const auto &f : frames) {
|
|
1072
|
-
cv::Mat scaled;
|
|
1073
|
-
if (std::abs(compose_scale - 1.0) > 1e-3) {
|
|
1074
|
-
cv::resize(f, scaled, cv::Size(), compose_scale, compose_scale,
|
|
1075
|
-
cv::INTER_AREA);
|
|
1076
|
-
} else {
|
|
1077
|
-
scaled = f.clone();
|
|
1078
|
-
}
|
|
1079
|
-
composeFrames.push_back(scaled);
|
|
1080
|
-
}
|
|
1081
|
-
for (auto &f : frames) f.release();
|
|
1082
|
-
frames.clear();
|
|
1083
|
-
|
|
1084
|
-
// Build the blender (same selection logic as the feature-matched
|
|
1085
|
-
// path). The "u != 0" UMat assertion the original feature-matched
|
|
1086
|
-
// builds hit was OOM-induced; with the per-frame Mat releases
|
|
1087
|
-
// and @autoreleasepool from that path's stabilisation, MultiBand
|
|
1088
|
-
// + GraphCut are safe here too.
|
|
1089
|
-
BOOL useSeam = [seamFinderType isEqualToString:@"graphcut"];
|
|
1090
|
-
cv::Ptr<cv::detail::Blender> blender;
|
|
1091
|
-
if ([blenderType isEqualToString:@"feather"]) {
|
|
1092
|
-
blender = cv::detail::Blender::createDefault(
|
|
1093
|
-
cv::detail::Blender::FEATHER, false);
|
|
1094
|
-
auto fb = blender.dynamicCast<cv::detail::FeatherBlender>();
|
|
1095
|
-
if (fb) fb->setSharpness(0.02f);
|
|
1096
|
-
} else {
|
|
1097
|
-
blender = cv::detail::Blender::createDefault(
|
|
1098
|
-
cv::detail::Blender::MULTI_BAND, false);
|
|
1099
|
-
auto mbb = blender.dynamicCast<cv::detail::MultiBandBlender>();
|
|
1100
|
-
if (mbb) mbb->setNumBands(5);
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
|
-
if (useSeam) {
|
|
1104
|
-
const size_t M = composeFrames.size();
|
|
1105
|
-
std::vector<cv::Point> corners(M);
|
|
1106
|
-
std::vector<cv::Mat> imagesWarped(M);
|
|
1107
|
-
std::vector<cv::Mat> masksWarped(M);
|
|
1108
|
-
std::vector<cv::Size> sizes(M);
|
|
1109
|
-
for (size_t i = 0; i < M; i++) {
|
|
1110
|
-
cv::Mat K;
|
|
1111
|
-
cameras[i].K().convertTo(K, CV_32F);
|
|
1112
|
-
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1113
|
-
corners[i] = warper->warp(
|
|
1114
|
-
composeFrames[i], K, cameras[i].R, cv::INTER_LINEAR,
|
|
1115
|
-
cv::BORDER_CONSTANT, imagesWarped[i]);
|
|
1116
|
-
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1117
|
-
cv::BORDER_CONSTANT, masksWarped[i]);
|
|
1118
|
-
sizes[i] = imagesWarped[i].size();
|
|
1119
|
-
}
|
|
1120
|
-
for (auto &cf : composeFrames) cf.release();
|
|
1121
|
-
composeFrames.clear();
|
|
1122
|
-
|
|
1123
|
-
// Seam finder at SEAM_MP scale (same downscale-find-upscale
|
|
1124
|
-
// pattern as the feature-matched path).
|
|
1125
|
-
const double SEAM_MP = 0.1;
|
|
1126
|
-
double seam_scale = std::min(1.0, std::sqrt(SEAM_MP / origMp));
|
|
1127
|
-
double seam_compose_aspect = seam_scale / compose_scale;
|
|
1128
|
-
std::vector<cv::UMat> imagesWarpedF_seam(M);
|
|
1129
|
-
std::vector<cv::UMat> masksWarpedU_seam(M);
|
|
1130
|
-
std::vector<cv::Point> corners_seam(M);
|
|
1131
|
-
for (size_t i = 0; i < M; i++) {
|
|
1132
|
-
cv::Mat seamImage, seamMask;
|
|
1133
|
-
cv::resize(imagesWarped[i], seamImage, cv::Size(),
|
|
1134
|
-
seam_compose_aspect, seam_compose_aspect,
|
|
1135
|
-
cv::INTER_LINEAR);
|
|
1136
|
-
cv::resize(masksWarped[i], seamMask, cv::Size(),
|
|
1137
|
-
seam_compose_aspect, seam_compose_aspect,
|
|
1138
|
-
cv::INTER_NEAREST);
|
|
1139
|
-
seamImage.convertTo(imagesWarpedF_seam[i], CV_32F);
|
|
1140
|
-
seamMask.copyTo(masksWarpedU_seam[i]);
|
|
1141
|
-
corners_seam[i] = cv::Point(
|
|
1142
|
-
cvRound(corners[i].x * seam_compose_aspect),
|
|
1143
|
-
cvRound(corners[i].y * seam_compose_aspect));
|
|
1144
|
-
}
|
|
1145
|
-
cv::Ptr<cv::detail::SeamFinder> seamFinder =
|
|
1146
|
-
cv::makePtr<cv::detail::GraphCutSeamFinder>(
|
|
1147
|
-
cv::detail::GraphCutSeamFinder::COST_COLOR);
|
|
1148
|
-
seamFinder->find(imagesWarpedF_seam, corners_seam, masksWarpedU_seam);
|
|
1149
|
-
imagesWarpedF_seam.clear();
|
|
1150
|
-
for (size_t i = 0; i < M; i++) {
|
|
1151
|
-
cv::Mat seamMaskCpu, seamMaskDilated, seamMaskFull;
|
|
1152
|
-
masksWarpedU_seam[i].copyTo(seamMaskCpu);
|
|
1153
|
-
cv::dilate(seamMaskCpu, seamMaskDilated, cv::Mat());
|
|
1154
|
-
cv::resize(seamMaskDilated, seamMaskFull,
|
|
1155
|
-
masksWarped[i].size(), 0, 0, cv::INTER_LINEAR);
|
|
1156
|
-
cv::bitwise_and(seamMaskFull, masksWarped[i], masksWarped[i]);
|
|
1157
|
-
}
|
|
1158
|
-
masksWarpedU_seam.clear();
|
|
1159
|
-
|
|
1160
|
-
blender->prepare(corners, sizes);
|
|
1161
|
-
for (size_t i = 0; i < M; i++) {
|
|
1162
|
-
cv::Mat imgS;
|
|
1163
|
-
imagesWarped[i].convertTo(imgS, CV_16S);
|
|
1164
|
-
blender->feed(imgS, masksWarped[i], corners[i]);
|
|
1165
|
-
imagesWarped[i].release();
|
|
1166
|
-
masksWarped[i].release();
|
|
1167
|
-
imgS.release();
|
|
1168
|
-
}
|
|
1169
|
-
imagesWarped.clear();
|
|
1170
|
-
masksWarped.clear();
|
|
1171
|
-
} else {
|
|
1172
|
-
// STREAM path
|
|
1173
|
-
const size_t M = composeFrames.size();
|
|
1174
|
-
std::vector<cv::Point> corners(M);
|
|
1175
|
-
std::vector<cv::Size> sizes(M);
|
|
1176
|
-
for (size_t i = 0; i < M; i++) {
|
|
1177
|
-
cv::Mat K;
|
|
1178
|
-
cameras[i].K().convertTo(K, CV_32F);
|
|
1179
|
-
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1180
|
-
cv::Mat tmpMaskWarped;
|
|
1181
|
-
corners[i] = warper->warp(
|
|
1182
|
-
mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1183
|
-
cv::BORDER_CONSTANT, tmpMaskWarped);
|
|
1184
|
-
sizes[i] = tmpMaskWarped.size();
|
|
1185
|
-
}
|
|
1186
|
-
blender->prepare(corners, sizes);
|
|
1187
|
-
for (size_t i = 0; i < M; i++) {
|
|
1188
|
-
cv::Mat K;
|
|
1189
|
-
cameras[i].K().convertTo(K, CV_32F);
|
|
1190
|
-
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1191
|
-
cv::Mat imgWarped, maskWarped;
|
|
1192
|
-
warper->warp(composeFrames[i], K, cameras[i].R,
|
|
1193
|
-
cv::INTER_LINEAR, cv::BORDER_CONSTANT, imgWarped);
|
|
1194
|
-
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1195
|
-
cv::BORDER_CONSTANT, maskWarped);
|
|
1196
|
-
cv::Mat imgS;
|
|
1197
|
-
imgWarped.convertTo(imgS, CV_16S);
|
|
1198
|
-
blender->feed(imgS, maskWarped, corners[i]);
|
|
1199
|
-
composeFrames[i].release();
|
|
1200
|
-
}
|
|
1201
|
-
composeFrames.clear();
|
|
1202
|
-
}
|
|
881
|
+
// Round-trip through cv::imread / cv::imwrite to bake the EXIF
|
|
882
|
+
// rotation into the pixel buffer, then write a plain JPEG with no
|
|
883
|
+
// orientation metadata. Cheap (~ms for a typical iPhone JPEG) and
|
|
884
|
+
// idempotent on already-normalised files.
|
|
1203
885
|
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
1209
|
-
if (error) {
|
|
1210
|
-
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1211
|
-
code:1100
|
|
1212
|
-
userInfo:@{
|
|
1213
|
-
NSLocalizedDescriptionKey:
|
|
1214
|
-
[NSString stringWithFormat:
|
|
1215
|
-
@"OpenCV exception during pose-driven stitch: %s", e.what()],
|
|
1216
|
-
}];
|
|
1217
|
-
}
|
|
1218
|
-
return nil;
|
|
1219
|
-
} catch (...) {
|
|
1220
|
-
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
886
|
+
+ (NSDictionary<NSString *, NSNumber *> *)normaliseImageAtPath:(NSString *)imagePath
|
|
887
|
+
error:(NSError **)error {
|
|
888
|
+
NSString *cleaned = normalizeImagePath(imagePath);
|
|
889
|
+
if (![[NSFileManager defaultManager] fileExistsAtPath:cleaned]) {
|
|
1221
890
|
if (error) {
|
|
1222
891
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1223
|
-
code:
|
|
892
|
+
code:1020
|
|
1224
893
|
userInfo:@{
|
|
1225
894
|
NSLocalizedDescriptionKey:
|
|
1226
|
-
|
|
895
|
+
[NSString stringWithFormat:@"Image not found: %@", imagePath],
|
|
1227
896
|
}];
|
|
1228
897
|
}
|
|
1229
898
|
return nil;
|
|
1230
899
|
}
|
|
1231
|
-
} // end @autoreleasepool
|
|
1232
900
|
|
|
1233
|
-
|
|
1234
|
-
|
|
901
|
+
std::string nativePath(cleaned.UTF8String);
|
|
902
|
+
cv::Mat img = cv::imread(nativePath, cv::IMREAD_COLOR);
|
|
903
|
+
if (img.empty()) {
|
|
1235
904
|
if (error) {
|
|
1236
905
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1237
|
-
code:
|
|
906
|
+
code:1021
|
|
1238
907
|
userInfo:@{
|
|
1239
908
|
NSLocalizedDescriptionKey:
|
|
1240
|
-
|
|
909
|
+
[NSString stringWithFormat:@"Could not decode image at %@", imagePath],
|
|
1241
910
|
}];
|
|
1242
911
|
}
|
|
1243
912
|
return nil;
|
|
1244
913
|
}
|
|
1245
914
|
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
// plane-warper feature-matched panoramas produce).
|
|
1249
|
-
cv::Mat finalImage = panorama;
|
|
1250
|
-
try {
|
|
1251
|
-
cv::Mat gray;
|
|
1252
|
-
cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
|
|
1253
|
-
cv::Mat mask;
|
|
1254
|
-
cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
|
|
1255
|
-
cv::Rect bbox = cv::boundingRect(mask);
|
|
1256
|
-
if (bbox.width > 0 && bbox.height > 0
|
|
1257
|
-
&& bbox.width <= panorama.cols && bbox.height <= panorama.rows) {
|
|
1258
|
-
finalImage = panorama(bbox).clone();
|
|
1259
|
-
}
|
|
1260
|
-
} catch (...) {
|
|
1261
|
-
finalImage = panorama;
|
|
1262
|
-
}
|
|
1263
|
-
|
|
1264
|
-
auto t1 = std::chrono::steady_clock::now();
|
|
1265
|
-
double durationMs =
|
|
1266
|
-
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
|
|
1267
|
-
|
|
1268
|
-
NSInteger clampedQuality = MAX(0, MIN(100, quality));
|
|
1269
|
-
std::vector<int> params = {
|
|
1270
|
-
cv::IMWRITE_JPEG_QUALITY, static_cast<int>(clampedQuality),
|
|
915
|
+
std::vector<int> writeParams = {
|
|
916
|
+
cv::IMWRITE_JPEG_QUALITY, 92,
|
|
1271
917
|
};
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
: outputPath);
|
|
1275
|
-
bool wrote = cv::imwrite([cleanedOutPath UTF8String], finalImage, params);
|
|
1276
|
-
|
|
1277
|
-
// Cleanup the tmp dir always.
|
|
1278
|
-
[[NSFileManager defaultManager] removeItemAtPath:tmpDir error:nil];
|
|
1279
|
-
|
|
1280
|
-
if (!wrote) {
|
|
918
|
+
bool ok = cv::imwrite(nativePath, img, writeParams);
|
|
919
|
+
if (!ok) {
|
|
1281
920
|
if (error) {
|
|
1282
921
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1283
|
-
code:
|
|
922
|
+
code:1022
|
|
1284
923
|
userInfo:@{
|
|
1285
924
|
NSLocalizedDescriptionKey:
|
|
1286
925
|
[NSString stringWithFormat:
|
|
1287
|
-
@"
|
|
1288
|
-
outputPath],
|
|
926
|
+
@"Could not rewrite image at %@", imagePath],
|
|
1289
927
|
}];
|
|
1290
928
|
}
|
|
1291
929
|
return nil;
|
|
1292
930
|
}
|
|
1293
931
|
|
|
1294
|
-
return
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
durationMs:durationMs];
|
|
932
|
+
return @{
|
|
933
|
+
@"width": @((NSInteger)img.cols),
|
|
934
|
+
@"height": @((NSInteger)img.rows),
|
|
935
|
+
};
|
|
1299
936
|
}
|
|
1300
937
|
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
//
|
|
1306
|
-
// Same compose stage as the video-driven pose path above, minus the
|
|
1307
|
-
// AVAssetImageGenerator extract + timestamp-matching step. Frames
|
|
1308
|
-
// arrive as already-on-disk JPEGs from the AR-keyframe capture flow;
|
|
1309
|
-
// poses are 1:1 with frames (KeyframeGate saved both as the user
|
|
1310
|
-
// panned). Compose code is duplicated per the convention noted
|
|
1311
|
-
// above ("DRY when the new path is proven on real shelf captures").
|
|
1312
|
-
//
|
|
1313
|
-
// AUDIT NOTE (2026-05-15, sibling @autoreleasepool-return audit)
|
|
1314
|
-
// ──────────────────────────────────────────────────────────────
|
|
1315
|
-
//
|
|
1316
|
-
// This method (and the pose-driven `stitchVideoAtPath:withPoses:`
|
|
1317
|
-
// variant earlier in this file at ~line 2162) BOTH have the same
|
|
1318
|
-
// @autoreleasepool-return-UAF pattern that V16 fix-10 closed in
|
|
1319
|
-
// `stitchFramePaths:` at line 597 — autoreleased NSError* assigned
|
|
1320
|
-
// to the `error` outparameter from inside an @autoreleasepool, then
|
|
1321
|
-
// the function returns, the pool drains, the NSError dangles, the
|
|
1322
|
-
// caller crashes dereferencing. See:
|
|
1323
|
-
// docs/site-content/design/2026-05-12-finalize-crash-investigation.md
|
|
1324
|
-
//
|
|
1325
|
-
// CURRENT REACHABILITY: BOTH methods are dead code as of 2026-05-15.
|
|
1326
|
-
// Confirmed by grep — only referenced in dSYM debug symbols + comments,
|
|
1327
|
-
// never actually called from Swift/Obj-C/Kotlin source paths. V16
|
|
1328
|
-
// batch-keyframe uses `stitchFramePaths:` exclusively; this method
|
|
1329
|
-
// was the earlier per-keyframe-with-pose design that was superseded.
|
|
1330
|
-
//
|
|
1331
|
-
// IF/WHEN RE-ENABLED, apply fix-10's pattern (also in this file
|
|
1332
|
-
// around `stitchFramePaths:` lines 562-571 + 1519-1527):
|
|
1333
|
-
//
|
|
1334
|
-
// NSError *capturedError = nil;
|
|
1335
|
-
// RNStitchResult *result = nil;
|
|
1336
|
-
// @autoreleasepool {
|
|
1337
|
-
// do {
|
|
1338
|
-
// try { ... ; result = [[RNStitchResult alloc] init...]; break; }
|
|
1339
|
-
// catch (cv::Exception &e) { capturedError = [NSError ...]; break; }
|
|
1340
|
-
// catch (...) { capturedError = [NSError ...]; break; }
|
|
1341
|
-
// } while (0);
|
|
1342
|
-
// }
|
|
1343
|
-
// if (capturedError) { if (error) *error = capturedError; return nil; }
|
|
1344
|
-
// return result;
|
|
1345
|
-
//
|
|
1346
|
-
// Strong locals (`capturedError`, `result`) are declared OUTSIDE the
|
|
1347
|
-
// @autoreleasepool so their refcount survives the pool drain. Both
|
|
1348
|
-
// success + failure paths exit the pool via `break` rather than
|
|
1349
|
-
// `return nil;` so the pool drains cleanly before the function
|
|
1350
|
-
// returns.
|
|
1351
|
-
//
|
|
1352
|
-
// Not applied now because the methods aren't called; risk is latent
|
|
1353
|
-
// not active. Refactoring dead code carries its own risk (subtle
|
|
1354
|
-
// behaviour changes) without active testing.
|
|
1355
|
-
|
|
1356
|
-
+ (nullable RNStitchResult *)stitchKeyframePaths:(NSArray<NSString *> *)framePaths
|
|
1357
|
-
outputPath:(NSString *)outputPath
|
|
1358
|
-
jpegQuality:(NSInteger)quality
|
|
1359
|
-
warperType:(NSString *)warperType
|
|
1360
|
-
blenderType:(NSString *)blenderType
|
|
1361
|
-
seamFinderType:(NSString *)seamFinderType
|
|
1362
|
-
poses:(NSArray<NSDictionary *> *)poses
|
|
1363
|
-
error:(NSError **)error {
|
|
1364
|
-
if (warperType == nil || warperType.length == 0) warperType = @"plane";
|
|
1365
|
-
if (blenderType == nil || blenderType.length == 0) blenderType = @"multiband";
|
|
1366
|
-
if (seamFinderType == nil || seamFinderType.length == 0) seamFinderType = @"graphcut";
|
|
1367
|
-
if (framePaths.count < 2) {
|
|
938
|
+
+ (NSDictionary<NSString *, NSNumber *> *)computeInscribedRectAtPath:(NSString *)imagePath
|
|
939
|
+
error:(NSError **)error {
|
|
940
|
+
NSString *cleaned = normalizeImagePath(imagePath);
|
|
941
|
+
if (![[NSFileManager defaultManager] fileExistsAtPath:cleaned]) {
|
|
1368
942
|
if (error) {
|
|
1369
943
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1370
|
-
code:
|
|
944
|
+
code:1020
|
|
1371
945
|
userInfo:@{
|
|
1372
946
|
NSLocalizedDescriptionKey:
|
|
1373
|
-
|
|
947
|
+
[NSString stringWithFormat:@"Image not found: %@", imagePath],
|
|
1374
948
|
}];
|
|
1375
949
|
}
|
|
1376
950
|
return nil;
|
|
1377
951
|
}
|
|
1378
|
-
|
|
952
|
+
|
|
953
|
+
std::string nativePath(cleaned.UTF8String);
|
|
954
|
+
cv::Mat img = cv::imread(nativePath, cv::IMREAD_COLOR);
|
|
955
|
+
if (img.empty()) {
|
|
1379
956
|
if (error) {
|
|
1380
957
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1381
|
-
code:
|
|
958
|
+
code:1021
|
|
1382
959
|
userInfo:@{
|
|
1383
960
|
NSLocalizedDescriptionKey:
|
|
1384
|
-
[NSString stringWithFormat
|
|
1385
|
-
@"Keyframe stitch requires 1:1 paths/poses; "
|
|
1386
|
-
"got %lu paths, %lu poses.",
|
|
1387
|
-
(unsigned long)framePaths.count,
|
|
1388
|
-
(unsigned long)poses.count],
|
|
961
|
+
[NSString stringWithFormat:@"Could not decode image at %@", imagePath],
|
|
1389
962
|
}];
|
|
1390
963
|
}
|
|
1391
964
|
return nil;
|
|
1392
965
|
}
|
|
1393
966
|
|
|
1394
|
-
//
|
|
1395
|
-
//
|
|
1396
|
-
//
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
// to load (corrupt JPEG, missing file) — but require ≥2 to
|
|
1404
|
-
// succeed for a panorama to be possible.
|
|
1405
|
-
//
|
|
1406
|
-
// V16 Phase 1.fix2 — IMREAD_IGNORE_ORIENTATION: collector saves
|
|
1407
|
-
// JPEGs with an EXIF Orientation tag so iOS Image renderers (e.g.
|
|
1408
|
-
// LiveFrameStrip) display correctly. cv::imread defaults (since
|
|
1409
|
-
// OpenCV 4.5+) APPLY the EXIF rotation; that would re-introduce
|
|
1410
|
-
// the image-vs-intrinsics mismatch fix1 was meant to remove. Pass
|
|
1411
|
-
// IMREAD_IGNORE_ORIENTATION explicitly to get raw landscape sensor
|
|
1412
|
-
// pixels for the stitcher.
|
|
1413
|
-
std::vector<cv::Mat> frames;
|
|
1414
|
-
std::vector<cv::detail::CameraParams> cameras;
|
|
1415
|
-
frames.reserve(framePaths.count);
|
|
1416
|
-
cameras.reserve(framePaths.count);
|
|
1417
|
-
int loaded = 0, dropped = 0;
|
|
1418
|
-
for (NSInteger i = 0; i < (NSInteger)framePaths.count; i++) {
|
|
1419
|
-
NSString *path = framePaths[i];
|
|
1420
|
-
NSString *cleaned = ([path hasPrefix:@"file://"]
|
|
1421
|
-
? [path substringFromIndex:[@"file://" length]]
|
|
1422
|
-
: path);
|
|
1423
|
-
cv::Mat img = cv::imread([cleaned UTF8String],
|
|
1424
|
-
cv::IMREAD_COLOR | cv::IMREAD_IGNORE_ORIENTATION);
|
|
1425
|
-
if (img.empty()) {
|
|
1426
|
-
dropped++;
|
|
1427
|
-
continue;
|
|
967
|
+
// Prefer the TRUE coverage sidecar the stitch writes next to the
|
|
968
|
+
// panorama (<path>.coverage.png); fall back to the hole-fill brightness
|
|
969
|
+
// proxy when it's absent (e.g. a non-stitch image).
|
|
970
|
+
NSString *coveragePath = [cleaned stringByAppendingString:@".coverage.png"];
|
|
971
|
+
cv::Mat mask;
|
|
972
|
+
if ([[NSFileManager defaultManager] fileExistsAtPath:coveragePath]) {
|
|
973
|
+
cv::Mat cov = cv::imread(std::string(coveragePath.UTF8String), cv::IMREAD_GRAYSCALE);
|
|
974
|
+
if (!cov.empty() && cov.cols == img.cols && cov.rows == img.rows) {
|
|
975
|
+
cv::threshold(cov, mask, 0, 255, cv::THRESH_BINARY);
|
|
1428
976
|
}
|
|
1429
|
-
frames.push_back(img);
|
|
1430
|
-
cameras.push_back(cameraParamsFromPose(poses[i]));
|
|
1431
|
-
loaded++;
|
|
1432
|
-
}
|
|
1433
|
-
NSLog(@"[BatchStitcher] keyframe-stitch: loaded=%d dropped=%d",
|
|
1434
|
-
loaded, dropped);
|
|
1435
|
-
if (!frames.empty()) {
|
|
1436
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1437
|
-
"[V16-stitch-mem] AFTER imread N=%d size=%dx%d totalMB=%.1f phys=%.1fMB",
|
|
1438
|
-
(int)frames.size(),
|
|
1439
|
-
frames[0].cols, frames[0].rows,
|
|
1440
|
-
(double)frames.size() * frames[0].cols * frames[0].rows * 3
|
|
1441
|
-
/ (1024.0 * 1024.0),
|
|
1442
|
-
StitcherResidentMB());
|
|
1443
977
|
}
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
userInfo:@{
|
|
1450
|
-
NSLocalizedDescriptionKey:
|
|
1451
|
-
@"Fewer than 2 keyframes loaded successfully — JPEGs may "
|
|
1452
|
-
"have been corrupted or removed before stitch ran.",
|
|
1453
|
-
}];
|
|
1454
|
-
}
|
|
1455
|
-
return nil;
|
|
978
|
+
if (mask.empty()) {
|
|
979
|
+
cv::Mat gray, raw;
|
|
980
|
+
cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY);
|
|
981
|
+
cv::threshold(gray, raw, 1, 255, cv::THRESH_BINARY);
|
|
982
|
+
mask = FillBorderConnectedHoles(raw);
|
|
1456
983
|
}
|
|
984
|
+
cv::Rect r = MaxInscribedRectFromMask(mask);
|
|
1457
985
|
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
double compose_scale = (origMp > COMPOSE_MP)
|
|
1468
|
-
? std::sqrt(COMPOSE_MP / origMp)
|
|
1469
|
-
: 1.0;
|
|
1470
|
-
double compose_work_aspect = compose_scale; // work_scale == 1
|
|
1471
|
-
|
|
1472
|
-
// V16 Phase 1.fix2 — auto-detect pan axis from camera rotation
|
|
1473
|
-
// spread. Compute the std-dev of camera "forward" vectors
|
|
1474
|
-
// projected onto each world axis; the axis with the smallest
|
|
1475
|
-
// spread is the pan-rotation axis (i.e. rotation about that
|
|
1476
|
-
// axis is what differs across frames most). HORIZ_PAN means
|
|
1477
|
-
// rotation about world Y (yaw): use WAVE_CORRECT_HORIZ.
|
|
1478
|
-
// VERT_PAN means rotation about world X (pitch): use WAVE_CORRECT_VERT.
|
|
1479
|
-
//
|
|
1480
|
-
// Earlier hardcoded HORIZ produced misaligned panoramas for
|
|
1481
|
-
// Ram's top-to-bottom landscape pan (no yaw spread; pitch
|
|
1482
|
-
// spread). Picking the right axis lets waveCorrect actually
|
|
1483
|
-
// help instead of being a no-op (or flipping the panorama).
|
|
1484
|
-
cv::detail::WaveCorrectKind waveKind = cv::detail::WAVE_CORRECT_HORIZ;
|
|
1485
|
-
if (cameras.size() >= 2) {
|
|
1486
|
-
// forward[i] = -3rd-column of R (camera looks along -Z in cv)
|
|
1487
|
-
double minF[3] = { 1e9, 1e9, 1e9};
|
|
1488
|
-
double maxF[3] = {-1e9,-1e9,-1e9};
|
|
1489
|
-
for (const auto &cam : cameras) {
|
|
1490
|
-
for (int axis = 0; axis < 3; axis++) {
|
|
1491
|
-
double v = -cam.R.at<float>(2, axis);
|
|
1492
|
-
if (v < minF[axis]) minF[axis] = v;
|
|
1493
|
-
if (v > maxF[axis]) maxF[axis] = v;
|
|
1494
|
-
}
|
|
1495
|
-
}
|
|
1496
|
-
double rangeX = maxF[0] - minF[0];
|
|
1497
|
-
double rangeY = maxF[1] - minF[1];
|
|
1498
|
-
// Larger Y-range of forward => more vertical (pitch) variation
|
|
1499
|
-
// => vertical pan => WAVE_CORRECT_VERT.
|
|
1500
|
-
if (rangeY > rangeX) {
|
|
1501
|
-
waveKind = cv::detail::WAVE_CORRECT_VERT;
|
|
1502
|
-
}
|
|
1503
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1504
|
-
"[V16-stitch-mem] waveKind=%{public}s "
|
|
1505
|
-
"rangeForwardX=%.3f rangeForwardY=%.3f",
|
|
1506
|
-
(waveKind == cv::detail::WAVE_CORRECT_VERT)
|
|
1507
|
-
? "VERT (vertical pan)"
|
|
1508
|
-
: "HORIZ (horizontal pan)",
|
|
1509
|
-
rangeX, rangeY);
|
|
1510
|
-
}
|
|
1511
|
-
std::vector<cv::Mat> rmats;
|
|
1512
|
-
rmats.reserve(cameras.size());
|
|
1513
|
-
for (const auto &cam : cameras) rmats.push_back(cam.R.clone());
|
|
1514
|
-
try {
|
|
1515
|
-
cv::detail::waveCorrect(rmats, waveKind);
|
|
1516
|
-
for (size_t i = 0; i < cameras.size(); i++) {
|
|
1517
|
-
cameras[i].R = rmats[i];
|
|
1518
|
-
}
|
|
1519
|
-
} catch (const cv::Exception &e) {
|
|
1520
|
-
NSLog(@"[BatchStitcher] keyframe: wave correction skipped: %s",
|
|
1521
|
-
e.what());
|
|
1522
|
-
}
|
|
1523
|
-
|
|
1524
|
-
// Rescale intrinsics for compose-scale warping.
|
|
1525
|
-
for (auto &cam : cameras) {
|
|
1526
|
-
cam.focal *= compose_work_aspect;
|
|
1527
|
-
cam.ppx *= compose_work_aspect;
|
|
1528
|
-
cam.ppy *= compose_work_aspect;
|
|
1529
|
-
}
|
|
1530
|
-
|
|
1531
|
-
std::vector<double> focals;
|
|
1532
|
-
for (const auto &cam : cameras) focals.push_back(cam.focal);
|
|
1533
|
-
std::sort(focals.begin(), focals.end());
|
|
1534
|
-
float warpedScale = focals.empty() ? 1.0f
|
|
1535
|
-
: (float)focals[focals.size() / 2];
|
|
1536
|
-
|
|
1537
|
-
cv::Ptr<cv::WarperCreator> warperCreator;
|
|
1538
|
-
if ([warperType isEqualToString:@"cylindrical"]) {
|
|
1539
|
-
warperCreator = cv::makePtr<cv::CylindricalWarper>();
|
|
1540
|
-
} else if ([warperType isEqualToString:@"spherical"]) {
|
|
1541
|
-
warperCreator = cv::makePtr<cv::SphericalWarper>();
|
|
1542
|
-
} else {
|
|
1543
|
-
warperCreator = cv::makePtr<cv::PlaneWarper>();
|
|
1544
|
-
}
|
|
1545
|
-
cv::Ptr<cv::detail::RotationWarper> warper =
|
|
1546
|
-
warperCreator->create(warpedScale);
|
|
1547
|
-
|
|
1548
|
-
// Build composeFrames at COMPOSE_MP from full-res input.
|
|
1549
|
-
std::vector<cv::Mat> composeFrames;
|
|
1550
|
-
composeFrames.reserve(frames.size());
|
|
1551
|
-
for (const auto &f : frames) {
|
|
1552
|
-
cv::Mat scaled;
|
|
1553
|
-
if (std::abs(compose_scale - 1.0) > 1e-3) {
|
|
1554
|
-
cv::resize(f, scaled, cv::Size(), compose_scale, compose_scale,
|
|
1555
|
-
cv::INTER_AREA);
|
|
1556
|
-
} else {
|
|
1557
|
-
scaled = f.clone();
|
|
1558
|
-
}
|
|
1559
|
-
composeFrames.push_back(scaled);
|
|
1560
|
-
}
|
|
1561
|
-
for (auto &f : frames) f.release();
|
|
1562
|
-
frames.clear();
|
|
1563
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1564
|
-
"[V16-stitch-mem] AFTER composeFrames built+frames cleared "
|
|
1565
|
-
"compose_scale=%.3f compose_size=%dx%d phys=%.1fMB",
|
|
1566
|
-
compose_scale,
|
|
1567
|
-
composeFrames.empty() ? 0 : composeFrames[0].cols,
|
|
1568
|
-
composeFrames.empty() ? 0 : composeFrames[0].rows,
|
|
1569
|
-
StitcherResidentMB());
|
|
1570
|
-
|
|
1571
|
-
BOOL useSeam = [seamFinderType isEqualToString:@"graphcut"];
|
|
1572
|
-
cv::Ptr<cv::detail::Blender> blender;
|
|
1573
|
-
if ([blenderType isEqualToString:@"feather"]) {
|
|
1574
|
-
blender = cv::detail::Blender::createDefault(
|
|
1575
|
-
cv::detail::Blender::FEATHER, false);
|
|
1576
|
-
auto fb = blender.dynamicCast<cv::detail::FeatherBlender>();
|
|
1577
|
-
if (fb) fb->setSharpness(0.02f);
|
|
1578
|
-
} else {
|
|
1579
|
-
blender = cv::detail::Blender::createDefault(
|
|
1580
|
-
cv::detail::Blender::MULTI_BAND, false);
|
|
1581
|
-
auto mbb = blender.dynamicCast<cv::detail::MultiBandBlender>();
|
|
1582
|
-
if (mbb) mbb->setNumBands(5);
|
|
1583
|
-
}
|
|
1584
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1585
|
-
"[V16-stitch-mem] config blender=%{public}@ seam=%{public}@ warper=%{public}@ phys=%.1fMB",
|
|
1586
|
-
blenderType, seamFinderType, warperType, StitcherResidentMB());
|
|
1587
|
-
|
|
1588
|
-
if (useSeam) {
|
|
1589
|
-
const size_t M = composeFrames.size();
|
|
1590
|
-
std::vector<cv::Point> corners(M);
|
|
1591
|
-
std::vector<cv::Mat> imagesWarped(M);
|
|
1592
|
-
std::vector<cv::Mat> masksWarped(M);
|
|
1593
|
-
std::vector<cv::Size> sizes(M);
|
|
1594
|
-
for (size_t i = 0; i < M; i++) {
|
|
1595
|
-
cv::Mat K;
|
|
1596
|
-
cameras[i].K().convertTo(K, CV_32F);
|
|
1597
|
-
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1598
|
-
corners[i] = warper->warp(
|
|
1599
|
-
composeFrames[i], K, cameras[i].R, cv::INTER_LINEAR,
|
|
1600
|
-
cv::BORDER_CONSTANT, imagesWarped[i]);
|
|
1601
|
-
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1602
|
-
cv::BORDER_CONSTANT, masksWarped[i]);
|
|
1603
|
-
sizes[i] = imagesWarped[i].size();
|
|
1604
|
-
}
|
|
1605
|
-
// Compute panorama bbox so we can see if the warped span is
|
|
1606
|
-
// unexpectedly large (drives MultiBand pyramid memory).
|
|
1607
|
-
int minX = INT_MAX, minY = INT_MAX, maxX = INT_MIN, maxY = INT_MIN;
|
|
1608
|
-
for (size_t i = 0; i < M; i++) {
|
|
1609
|
-
minX = std::min(minX, corners[i].x);
|
|
1610
|
-
minY = std::min(minY, corners[i].y);
|
|
1611
|
-
maxX = std::max(maxX, corners[i].x + (int)sizes[i].width);
|
|
1612
|
-
maxY = std::max(maxY, corners[i].y + (int)sizes[i].height);
|
|
1613
|
-
}
|
|
1614
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1615
|
-
"[V16-stitch-mem] AFTER warps M=%d bbox=%dx%d "
|
|
1616
|
-
"warpedTotalMB=%.1f phys=%.1fMB",
|
|
1617
|
-
(int)M,
|
|
1618
|
-
(maxX > minX ? maxX - minX : 0),
|
|
1619
|
-
(maxY > minY ? maxY - minY : 0),
|
|
1620
|
-
(double)M * (M ? sizes[0].width : 0)
|
|
1621
|
-
* (M ? sizes[0].height : 0) * 3 / (1024.0 * 1024.0),
|
|
1622
|
-
StitcherResidentMB());
|
|
1623
|
-
const int panBboxW = (maxX > minX ? maxX - minX : 0);
|
|
1624
|
-
const int panBboxH = (maxY > minY ? maxY - minY : 0);
|
|
1625
|
-
// Quiet `unused variable` warnings if the inner os_log calls
|
|
1626
|
-
// are stripped by the compiler in release builds.
|
|
1627
|
-
(void)panBboxW; (void)panBboxH;
|
|
1628
|
-
for (auto &cf : composeFrames) cf.release();
|
|
1629
|
-
composeFrames.clear();
|
|
1630
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1631
|
-
"[V16-stitch-mem] AFTER composeFrames cleared (warps held) phys=%.1fMB",
|
|
1632
|
-
StitcherResidentMB());
|
|
1633
|
-
|
|
1634
|
-
const double SEAM_MP = 0.1;
|
|
1635
|
-
double seam_scale = std::min(1.0, std::sqrt(SEAM_MP / origMp));
|
|
1636
|
-
double seam_compose_aspect = seam_scale / compose_scale;
|
|
1637
|
-
std::vector<cv::UMat> imagesWarpedF_seam(M);
|
|
1638
|
-
std::vector<cv::UMat> masksWarpedU_seam(M);
|
|
1639
|
-
std::vector<cv::Point> corners_seam(M);
|
|
1640
|
-
for (size_t i = 0; i < M; i++) {
|
|
1641
|
-
cv::Mat seamImage, seamMask;
|
|
1642
|
-
cv::resize(imagesWarped[i], seamImage, cv::Size(),
|
|
1643
|
-
seam_compose_aspect, seam_compose_aspect,
|
|
1644
|
-
cv::INTER_LINEAR);
|
|
1645
|
-
cv::resize(masksWarped[i], seamMask, cv::Size(),
|
|
1646
|
-
seam_compose_aspect, seam_compose_aspect,
|
|
1647
|
-
cv::INTER_NEAREST);
|
|
1648
|
-
seamImage.convertTo(imagesWarpedF_seam[i], CV_32F);
|
|
1649
|
-
seamMask.copyTo(masksWarpedU_seam[i]);
|
|
1650
|
-
corners_seam[i] = cv::Point(
|
|
1651
|
-
cvRound(corners[i].x * seam_compose_aspect),
|
|
1652
|
-
cvRound(corners[i].y * seam_compose_aspect));
|
|
1653
|
-
}
|
|
1654
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1655
|
-
"[V16-stitch-mem] BEFORE GraphCutSeamFinder seam_scale=%.3f phys=%.1fMB",
|
|
1656
|
-
seam_scale, StitcherResidentMB());
|
|
1657
|
-
cv::Ptr<cv::detail::SeamFinder> seamFinder =
|
|
1658
|
-
cv::makePtr<cv::detail::GraphCutSeamFinder>(
|
|
1659
|
-
cv::detail::GraphCutSeamFinder::COST_COLOR);
|
|
1660
|
-
seamFinder->find(imagesWarpedF_seam, corners_seam, masksWarpedU_seam);
|
|
1661
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1662
|
-
"[V16-stitch-mem] AFTER GraphCutSeamFinder phys=%.1fMB",
|
|
1663
|
-
StitcherResidentMB());
|
|
1664
|
-
imagesWarpedF_seam.clear();
|
|
1665
|
-
for (size_t i = 0; i < M; i++) {
|
|
1666
|
-
cv::Mat seamMaskCpu, seamMaskDilated, seamMaskFull;
|
|
1667
|
-
masksWarpedU_seam[i].copyTo(seamMaskCpu);
|
|
1668
|
-
cv::dilate(seamMaskCpu, seamMaskDilated, cv::Mat());
|
|
1669
|
-
cv::resize(seamMaskDilated, seamMaskFull,
|
|
1670
|
-
masksWarped[i].size(), 0, 0, cv::INTER_LINEAR);
|
|
1671
|
-
cv::bitwise_and(seamMaskFull, masksWarped[i], masksWarped[i]);
|
|
1672
|
-
}
|
|
1673
|
-
masksWarpedU_seam.clear();
|
|
1674
|
-
|
|
1675
|
-
blender->prepare(corners, sizes);
|
|
1676
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1677
|
-
"[V16-stitch-mem] AFTER blender->prepare() phys=%.1fMB",
|
|
1678
|
-
StitcherResidentMB());
|
|
1679
|
-
for (size_t i = 0; i < M; i++) {
|
|
1680
|
-
cv::Mat imgS;
|
|
1681
|
-
imagesWarped[i].convertTo(imgS, CV_16S);
|
|
1682
|
-
blender->feed(imgS, masksWarped[i], corners[i]);
|
|
1683
|
-
imagesWarped[i].release();
|
|
1684
|
-
masksWarped[i].release();
|
|
1685
|
-
imgS.release();
|
|
1686
|
-
}
|
|
1687
|
-
imagesWarped.clear();
|
|
1688
|
-
masksWarped.clear();
|
|
1689
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1690
|
-
"[V16-stitch-mem] AFTER blender->feed() loop (graphcut) phys=%.1fMB",
|
|
1691
|
-
StitcherResidentMB());
|
|
1692
|
-
} else {
|
|
1693
|
-
// STREAM path
|
|
1694
|
-
const size_t M = composeFrames.size();
|
|
1695
|
-
std::vector<cv::Point> corners(M);
|
|
1696
|
-
std::vector<cv::Size> sizes(M);
|
|
1697
|
-
for (size_t i = 0; i < M; i++) {
|
|
1698
|
-
cv::Mat K;
|
|
1699
|
-
cameras[i].K().convertTo(K, CV_32F);
|
|
1700
|
-
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1701
|
-
cv::Mat tmpMaskWarped;
|
|
1702
|
-
corners[i] = warper->warp(
|
|
1703
|
-
mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1704
|
-
cv::BORDER_CONSTANT, tmpMaskWarped);
|
|
1705
|
-
sizes[i] = tmpMaskWarped.size();
|
|
1706
|
-
}
|
|
1707
|
-
blender->prepare(corners, sizes);
|
|
1708
|
-
for (size_t i = 0; i < M; i++) {
|
|
1709
|
-
cv::Mat K;
|
|
1710
|
-
cameras[i].K().convertTo(K, CV_32F);
|
|
1711
|
-
cv::Mat mask(composeFrames[i].size(), CV_8U, cv::Scalar(255));
|
|
1712
|
-
cv::Mat imgWarped, maskWarped;
|
|
1713
|
-
warper->warp(composeFrames[i], K, cameras[i].R,
|
|
1714
|
-
cv::INTER_LINEAR, cv::BORDER_CONSTANT, imgWarped);
|
|
1715
|
-
warper->warp(mask, K, cameras[i].R, cv::INTER_NEAREST,
|
|
1716
|
-
cv::BORDER_CONSTANT, maskWarped);
|
|
1717
|
-
cv::Mat imgS;
|
|
1718
|
-
imgWarped.convertTo(imgS, CV_16S);
|
|
1719
|
-
blender->feed(imgS, maskWarped, corners[i]);
|
|
1720
|
-
composeFrames[i].release();
|
|
1721
|
-
}
|
|
1722
|
-
composeFrames.clear();
|
|
1723
|
-
}
|
|
986
|
+
return @{
|
|
987
|
+
@"x": @((NSInteger)r.x),
|
|
988
|
+
@"y": @((NSInteger)r.y),
|
|
989
|
+
@"width": @((NSInteger)r.width),
|
|
990
|
+
@"height": @((NSInteger)r.height),
|
|
991
|
+
@"imageWidth": @((NSInteger)img.cols),
|
|
992
|
+
@"imageHeight": @((NSInteger)img.rows),
|
|
993
|
+
};
|
|
994
|
+
}
|
|
1724
995
|
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1735
|
-
"[V16-stitch-mem] AFTER 16S->8U convert phys=%.1fMB",
|
|
1736
|
-
StitcherResidentMB());
|
|
1737
|
-
} catch (const cv::Exception &e) {
|
|
1738
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1739
|
-
"[V16-stitch-mem] cv::Exception: %{public}s phys=%.1fMB",
|
|
1740
|
-
e.what(), StitcherResidentMB());
|
|
1741
|
-
if (error) {
|
|
1742
|
-
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1743
|
-
code:1100
|
|
1744
|
-
userInfo:@{
|
|
1745
|
-
NSLocalizedDescriptionKey:
|
|
1746
|
-
[NSString stringWithFormat:
|
|
1747
|
-
@"OpenCV exception during keyframe stitch: %s", e.what()],
|
|
1748
|
-
}];
|
|
1749
|
-
}
|
|
1750
|
-
return nil;
|
|
1751
|
-
} catch (...) {
|
|
996
|
+
+ (NSDictionary<NSString *, NSNumber *> *)cropToRectAtPath:(NSString *)imagePath
|
|
997
|
+
x:(NSInteger)x
|
|
998
|
+
y:(NSInteger)y
|
|
999
|
+
width:(NSInteger)width
|
|
1000
|
+
height:(NSInteger)height
|
|
1001
|
+
quality:(NSInteger)quality
|
|
1002
|
+
error:(NSError **)error {
|
|
1003
|
+
NSString *cleaned = normalizeImagePath(imagePath);
|
|
1004
|
+
if (![[NSFileManager defaultManager] fileExistsAtPath:cleaned]) {
|
|
1752
1005
|
if (error) {
|
|
1753
1006
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1754
|
-
code:
|
|
1007
|
+
code:1020
|
|
1755
1008
|
userInfo:@{
|
|
1756
1009
|
NSLocalizedDescriptionKey:
|
|
1757
|
-
|
|
1010
|
+
[NSString stringWithFormat:@"Image not found: %@", imagePath],
|
|
1758
1011
|
}];
|
|
1759
1012
|
}
|
|
1760
1013
|
return nil;
|
|
1761
1014
|
}
|
|
1762
|
-
} // end @autoreleasepool
|
|
1763
1015
|
|
|
1764
|
-
|
|
1016
|
+
std::string nativePath(cleaned.UTF8String);
|
|
1017
|
+
cv::Mat img = cv::imread(nativePath, cv::IMREAD_COLOR);
|
|
1018
|
+
if (img.empty()) {
|
|
1765
1019
|
if (error) {
|
|
1766
1020
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1767
|
-
code:
|
|
1021
|
+
code:1021
|
|
1768
1022
|
userInfo:@{
|
|
1769
1023
|
NSLocalizedDescriptionKey:
|
|
1770
|
-
|
|
1024
|
+
[NSString stringWithFormat:@"Could not decode image at %@", imagePath],
|
|
1771
1025
|
}];
|
|
1772
1026
|
}
|
|
1773
1027
|
return nil;
|
|
1774
1028
|
}
|
|
1775
1029
|
|
|
1776
|
-
//
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
}
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
auto t1 = std::chrono::steady_clock::now();
|
|
1796
|
-
double durationMs =
|
|
1797
|
-
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
|
|
1798
|
-
|
|
1799
|
-
NSInteger clampedQuality = MAX(0, MIN(100, quality));
|
|
1800
|
-
std::vector<int> params = {
|
|
1801
|
-
cv::IMWRITE_JPEG_QUALITY, static_cast<int>(clampedQuality),
|
|
1802
|
-
};
|
|
1803
|
-
NSString *cleanedOutPath = ([outputPath hasPrefix:@"file://"]
|
|
1804
|
-
? [outputPath substringFromIndex:[@"file://" length]]
|
|
1805
|
-
: outputPath);
|
|
1806
|
-
bool wrote = cv::imwrite([cleanedOutPath UTF8String], finalImage, params);
|
|
1807
|
-
os_log_with_type(StitcherDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1808
|
-
"[V16-stitch-mem] AFTER cv::imwrite ok=%d total=%.0fms phys=%.1fMB",
|
|
1809
|
-
wrote ? 1 : 0, durationMs, StitcherResidentMB());
|
|
1810
|
-
|
|
1811
|
-
if (!wrote) {
|
|
1030
|
+
// Clamp the requested rect to the image bounds (defensive — the JS
|
|
1031
|
+
// side derives it from computeInscribedRect, but never trust input).
|
|
1032
|
+
int rx = (int)x; if (rx < 0) { rx = 0; }
|
|
1033
|
+
int ry = (int)y; if (ry < 0) { ry = 0; }
|
|
1034
|
+
if (rx > img.cols - 1) { rx = img.cols - 1; }
|
|
1035
|
+
if (ry > img.rows - 1) { ry = img.rows - 1; }
|
|
1036
|
+
int rw = (int)width; if (rw < 1) { rw = 1; }
|
|
1037
|
+
int rh = (int)height; if (rh < 1) { rh = 1; }
|
|
1038
|
+
if (rx + rw > img.cols) { rw = img.cols - rx; }
|
|
1039
|
+
if (ry + rh > img.rows) { rh = img.rows - ry; }
|
|
1040
|
+
|
|
1041
|
+
cv::Mat cropped = img(cv::Rect(rx, ry, rw, rh)).clone();
|
|
1042
|
+
|
|
1043
|
+
int q = (int)quality;
|
|
1044
|
+
if (q < 1) { q = 1; }
|
|
1045
|
+
if (q > 100) { q = 100; }
|
|
1046
|
+
std::vector<int> writeParams = { cv::IMWRITE_JPEG_QUALITY, q };
|
|
1047
|
+
bool ok = cv::imwrite(nativePath, cropped, writeParams);
|
|
1048
|
+
if (!ok) {
|
|
1812
1049
|
if (error) {
|
|
1813
1050
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1814
|
-
code:
|
|
1051
|
+
code:1022
|
|
1815
1052
|
userInfo:@{
|
|
1816
1053
|
NSLocalizedDescriptionKey:
|
|
1817
|
-
[NSString stringWithFormat
|
|
1818
|
-
@"Keyframe stitch succeeded but could not write JPEG to %@",
|
|
1819
|
-
outputPath],
|
|
1054
|
+
[NSString stringWithFormat:@"Could not rewrite image at %@", imagePath],
|
|
1820
1055
|
}];
|
|
1821
1056
|
}
|
|
1822
1057
|
return nil;
|
|
1823
1058
|
}
|
|
1824
1059
|
|
|
1825
|
-
return
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
durationMs:durationMs];
|
|
1060
|
+
return @{
|
|
1061
|
+
@"width": @((NSInteger)cropped.cols),
|
|
1062
|
+
@"height": @((NSInteger)cropped.rows),
|
|
1063
|
+
};
|
|
1830
1064
|
}
|
|
1831
1065
|
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
1836
|
-
// Round-trip through cv::imread / cv::imwrite to bake the EXIF
|
|
1837
|
-
// rotation into the pixel buffer, then write a plain JPEG with no
|
|
1838
|
-
// orientation metadata. Cheap (~ms for a typical iPhone JPEG) and
|
|
1839
|
-
// idempotent on already-normalised files.
|
|
1840
|
-
|
|
1841
|
-
+ (NSDictionary<NSString *, NSNumber *> *)normaliseImageAtPath:(NSString *)imagePath
|
|
1842
|
-
error:(NSError **)error {
|
|
1066
|
+
+ (NSDictionary *)debugMaskOverlayAtPath:(NSString *)imagePath
|
|
1067
|
+
threshold:(NSInteger)threshold
|
|
1068
|
+
error:(NSError **)error {
|
|
1843
1069
|
NSString *cleaned = normalizeImagePath(imagePath);
|
|
1844
1070
|
if (![[NSFileManager defaultManager] fileExistsAtPath:cleaned]) {
|
|
1845
1071
|
if (error) {
|
|
@@ -1867,26 +1093,59 @@ cv::detail::CameraParams cameraParamsFromPose(NSDictionary *pose) {
|
|
|
1867
1093
|
return nil;
|
|
1868
1094
|
}
|
|
1869
1095
|
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
if (
|
|
1096
|
+
// Prefer the TRUE coverage sidecar (<path>.coverage.png) the stitch
|
|
1097
|
+
// writes; else the hole-fill brightness proxy at threshold `t`.
|
|
1098
|
+
NSString *coveragePath = [cleaned stringByAppendingString:@".coverage.png"];
|
|
1099
|
+
cv::Mat mask;
|
|
1100
|
+
if ([[NSFileManager defaultManager] fileExistsAtPath:coveragePath]) {
|
|
1101
|
+
cv::Mat cov = cv::imread(std::string(coveragePath.UTF8String), cv::IMREAD_GRAYSCALE);
|
|
1102
|
+
if (!cov.empty() && cov.cols == img.cols && cov.rows == img.rows) {
|
|
1103
|
+
cv::threshold(cov, mask, 0, 255, cv::THRESH_BINARY);
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
if (mask.empty()) {
|
|
1107
|
+
int t = (int)threshold;
|
|
1108
|
+
if (t < 0) { t = 0; }
|
|
1109
|
+
cv::Mat gray, raw;
|
|
1110
|
+
cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY);
|
|
1111
|
+
cv::threshold(gray, raw, t, 255, cv::THRESH_BINARY);
|
|
1112
|
+
mask = FillBorderConnectedHoles(raw);
|
|
1113
|
+
}
|
|
1114
|
+
cv::Mat excluded;
|
|
1115
|
+
cv::bitwise_not(mask, excluded); // 255 = dropped pixels
|
|
1116
|
+
|
|
1117
|
+
// Blend red (BGR 0,0,255) over the dropped pixels so they stand out.
|
|
1118
|
+
cv::Mat overlay = img.clone();
|
|
1119
|
+
cv::Mat red(img.size(), img.type(), cv::Scalar(0, 0, 255));
|
|
1120
|
+
cv::Mat blended;
|
|
1121
|
+
cv::addWeighted(img, 0.35, red, 0.65, 0.0, blended);
|
|
1122
|
+
blended.copyTo(overlay, excluded);
|
|
1123
|
+
|
|
1124
|
+
std::string outPath = std::string(cleaned.UTF8String) + ".mask.jpg";
|
|
1125
|
+
std::vector<int> writeParams = { cv::IMWRITE_JPEG_QUALITY, 90 };
|
|
1126
|
+
if (!cv::imwrite(outPath, overlay, writeParams)) {
|
|
1875
1127
|
if (error) {
|
|
1876
1128
|
*error = [NSError errorWithDomain:RNImageStitcherErrorDomain
|
|
1877
1129
|
code:1022
|
|
1878
1130
|
userInfo:@{
|
|
1879
1131
|
NSLocalizedDescriptionKey:
|
|
1880
|
-
[NSString stringWithFormat
|
|
1881
|
-
@"Could not rewrite image at %@", imagePath],
|
|
1132
|
+
[NSString stringWithFormat:@"Could not write mask overlay for %@", imagePath],
|
|
1882
1133
|
}];
|
|
1883
1134
|
}
|
|
1884
1135
|
return nil;
|
|
1885
1136
|
}
|
|
1886
1137
|
|
|
1138
|
+
long total = (long)mask.rows * (long)mask.cols;
|
|
1139
|
+
long content = (long)cv::countNonZero(mask);
|
|
1140
|
+
int excludedPct = (total > 0)
|
|
1141
|
+
? (int)((double)(total - content) * 100.0 / (double)total)
|
|
1142
|
+
: 0;
|
|
1143
|
+
|
|
1887
1144
|
return @{
|
|
1888
|
-
@"
|
|
1889
|
-
@"
|
|
1145
|
+
@"maskPath": [NSString stringWithUTF8String:outPath.c_str()],
|
|
1146
|
+
@"width": @((NSInteger)img.cols),
|
|
1147
|
+
@"height": @((NSInteger)img.rows),
|
|
1148
|
+
@"excludedPercent": @((NSInteger)excludedPct),
|
|
1890
1149
|
};
|
|
1891
1150
|
}
|
|
1892
1151
|
|