react-native-image-stitcher 0.14.1 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +160 -0
  2. package/README.md +35 -0
  3. package/RNImageStitcher.podspec +8 -1
  4. package/android/build.gradle +0 -16
  5. package/android/src/main/cpp/CMakeLists.txt +2 -63
  6. package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
  7. package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
  8. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
  9. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
  10. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
  11. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
  12. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
  13. package/cpp/keyframe_gate.cpp +82 -23
  14. package/cpp/keyframe_gate.hpp +31 -2
  15. package/cpp/stitcher.cpp +208 -28
  16. package/cpp/tests/CMakeLists.txt +18 -12
  17. package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
  18. package/cpp/tests/warp_guard_test.cpp +48 -0
  19. package/cpp/warp_guard.hpp +41 -0
  20. package/dist/ar/useARSession.d.ts +9 -0
  21. package/dist/ar/useARSession.js +24 -2
  22. package/dist/camera/Camera.d.ts +31 -16
  23. package/dist/camera/Camera.js +27 -4
  24. package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
  25. package/dist/camera/CaptureStitchStatsToast.js +27 -7
  26. package/dist/camera/PanoramaSettings.d.ts +10 -223
  27. package/dist/camera/PanoramaSettings.js +6 -28
  28. package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
  29. package/dist/camera/PanoramaSettingsBridge.js +3 -102
  30. package/dist/camera/PanoramaSettingsModal.js +7 -1
  31. package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
  32. package/dist/camera/buildPanoramaInitialSettings.js +4 -0
  33. package/dist/camera/cameraErrorMessages.d.ts +32 -0
  34. package/dist/camera/cameraErrorMessages.js +53 -0
  35. package/dist/camera/selectCaptureDevice.d.ts +5 -1
  36. package/dist/camera/selectCaptureDevice.js +22 -2
  37. package/dist/camera/useCapture.js +38 -0
  38. package/dist/index.d.ts +5 -8
  39. package/dist/index.js +11 -34
  40. package/dist/stitching/incremental.d.ts +1 -117
  41. package/dist/stitching/stitchVideo.d.ts +0 -35
  42. package/dist/types.d.ts +0 -87
  43. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
  44. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
  45. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
  46. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
  47. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
  48. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
  49. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
  50. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
  51. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
  52. package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
  53. package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
  54. package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
  55. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
  56. package/package.json +3 -2
  57. package/src/ar/useARSession.ts +35 -5
  58. package/src/camera/Camera.tsx +63 -24
  59. package/src/camera/CaptureStitchStatsToast.tsx +58 -14
  60. package/src/camera/PanoramaSettings.ts +16 -289
  61. package/src/camera/PanoramaSettingsBridge.ts +3 -114
  62. package/src/camera/PanoramaSettingsModal.tsx +14 -1
  63. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
  64. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
  65. package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
  66. package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
  67. package/src/camera/buildPanoramaInitialSettings.ts +17 -0
  68. package/src/camera/cameraErrorMessages.ts +84 -0
  69. package/src/camera/selectCaptureDevice.ts +28 -3
  70. package/src/camera/useCapture.ts +44 -1
  71. package/src/index.ts +11 -40
  72. package/src/stitching/incremental.ts +3 -140
  73. package/src/stitching/stitchVideo.ts +0 -26
  74. package/src/types.ts +0 -95
  75. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
  76. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
  77. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
  78. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
  79. package/cpp/stitcher_frame_jsi.cpp +0 -214
  80. package/cpp/stitcher_frame_jsi.hpp +0 -108
  81. package/cpp/stitcher_proxy_jsi.cpp +0 -109
  82. package/cpp/stitcher_proxy_jsi.hpp +0 -46
  83. package/cpp/stitcher_worklet_dispatch.cpp +0 -103
  84. package/cpp/stitcher_worklet_dispatch.hpp +0 -71
  85. package/cpp/stitcher_worklet_registry.cpp +0 -91
  86. package/cpp/stitcher_worklet_registry.hpp +0 -146
  87. package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
  88. package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
  89. package/dist/stitching/IncrementalStitcherView.js +0 -157
  90. package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
  91. package/dist/stitching/StitcherWorkletRegistry.js +0 -78
  92. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
  93. package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
  94. package/dist/stitching/useFrameProcessor.d.ts +0 -119
  95. package/dist/stitching/useFrameProcessor.js +0 -196
  96. package/dist/stitching/useFrameStream.d.ts +0 -34
  97. package/dist/stitching/useFrameStream.js +0 -234
  98. package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
  99. package/dist/stitching/useThrottledFrameProcessor.js +0 -132
  100. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
  101. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
  102. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
  104. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
  105. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
  106. package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
  107. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  108. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
  109. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
  110. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
  111. package/src/stitching/IncrementalStitcherView.tsx +0 -198
  112. package/src/stitching/StitcherWorkletRegistry.ts +0 -156
  113. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
  114. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
  115. package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
  116. package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
  117. package/src/stitching/useFrameProcessor.ts +0 -226
  118. package/src/stitching/useFrameStream.ts +0 -271
  119. package/src/stitching/useThrottledFrameProcessor.ts +0 -145
@@ -1,1328 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // OpenCVIncrementalStitcher.mm
4
- //
5
- // See OpenCVIncrementalStitcher.h for the API contract.
6
- //
7
- // Algorithm summary (per addPixelBuffer call):
8
- // 1. Lock + read NV12 planes from CVPixelBuffer
9
- // 2. Convert + downscale to compose-size BGR cv::Mat
10
- // 3. Pose-delta gate: skip if overlap > maxOverlap or < minOverlap
11
- // 4. ORB.detectAndCompute (1000 features cap)
12
- // 5. BFMatcher.knnMatch + Lowe's ratio test (0.75)
13
- // 6. cv::findHomography(src=newPts, dst=lastPts, RANSAC, 5.0)
14
- // 7. Inlier ratio + match count + det(H) → confidence
15
- // 8. Compose worldH = lastFrameToWorldH * H_newToLast
16
- // 9. warpPerspective + distance-transform feather blend onto canvas
17
- // 10. Update state: lastFrameToWorldH, lastDescriptors, lastKeypoints,
18
- // lastAcceptedYaw/Pitch, acceptedCount++
19
- //
20
- // What this file deliberately does NOT do:
21
- // - Bundle adjustment. We accumulate pair-wise homography only;
22
- // drift is accepted as the trade for live preview + Android parity
23
- // (the Android prebuilt OpenCV ships without `cv::Stitcher`'s BA
24
- // helpers). Long-pan drift is documented as future work in the
25
- // design doc's open questions.
26
- // - Multi-band blending. Same Android-parity reason. Distance-
27
- // transform feather over a 20px band gives clean-enough seams for
28
- // live preview; a final-pass MultiBand is possible at finalize
29
- // time on iOS only if drift becomes the dominant artefact.
30
- // - Exposure compensation. Auto-exposure on the camera handles
31
- // gross brightness changes; the feather hides residual mismatches.
32
-
33
- // OpenCV's stitching headers contain `enum { NO, ... }` and `enum { YES, ... }`
34
- // declarations. Objective-C's `<objc/objc.h>` (transitively imported by every
35
- // Cocoapods prefix.pch) #defines `NO` and `YES` as boolean macros — by the
36
- // time OpenCV's enums are parsed, the preprocessor has already eaten those
37
- // identifiers and the build dies with "expected identifier". Undef both
38
- // BEFORE importing opencv2/*; restore after. Same pattern as OpenCVStitcher.mm.
39
- #ifdef NO
40
- #undef NO
41
- #endif
42
- #ifdef YES
43
- #undef YES
44
- #endif
45
-
46
- #import <opencv2/opencv.hpp>
47
- #import <opencv2/core.hpp>
48
- #import <opencv2/imgproc.hpp>
49
- #import <opencv2/imgcodecs.hpp>
50
- #import <opencv2/features2d.hpp>
51
- #import <opencv2/calib3d.hpp>
52
- #import <opencv2/stitching/detail/warpers.hpp> // V14.0pre — cv::detail::CylindricalWarper
53
-
54
- #import <vector>
55
- #import <chrono>
56
-
57
- #define NO ((BOOL)0)
58
- #define YES ((BOOL)1)
59
-
60
- #import "OpenCVIncrementalStitcher.h"
61
-
62
- #import <UIKit/UIKit.h>
63
-
64
- NSString *const RNImageStitcherIncrementalErrorDomain =
65
- @"RNImageStitcherIncrementalErrorDomain";
66
-
67
- // ── Private telemetry result class ──────────────────────────────────
68
-
69
- @interface RLISFrameTelemetry ()
70
- @property (nonatomic, readwrite) RLISFrameOutcome outcome;
71
- @property (nonatomic, readwrite) double overlapPercent;
72
- @property (nonatomic, readwrite) NSInteger matchCount;
73
- @property (nonatomic, readwrite) double inlierRatio;
74
- @property (nonatomic, readwrite) double confidence;
75
- @property (nonatomic, readwrite) double processingMs;
76
- @property (nonatomic, readwrite) BOOL isLandscape;
77
- // V12.14.9 — see header doc for `paintedExtent` / `panExtent` semantics.
78
- @property (nonatomic, readwrite) NSInteger paintedExtent;
79
- @property (nonatomic, readwrite) NSInteger panExtent;
80
- @end
81
-
82
- @implementation RLISFrameTelemetry
83
- @end
84
-
85
- @interface RLISSnapshot ()
86
- @property (nonatomic, copy, readwrite) NSString *panoramaPath;
87
- @property (nonatomic, readwrite) NSInteger width;
88
- @property (nonatomic, readwrite) NSInteger height;
89
- @property (nonatomic, readwrite) NSInteger acceptedCount;
90
- @end
91
-
92
- @implementation RLISSnapshot
93
- @end
94
-
95
-
96
- // ── V15 — RLISStitcherConfig ────────────────────────────────────────
97
-
98
- @implementation RLISStitcherConfig
99
-
100
- + (instancetype)configForMode:(NSString *)mode {
101
- RLISStitcherConfig *c = [[RLISStitcherConfig alloc] init];
102
-
103
- NSString *m = mode ?: @"slitscan-both";
104
- // Backward-compat translation.
105
- if ([m isEqualToString:@"firstwins-rectilinear"]) {
106
- m = @"slitscan-rotate";
107
- } else if ([m isEqualToString:@"firstwins"] ||
108
- [m isEqualToString:@"firstwins-zoomed"]) {
109
- NSLog(@"[V15-config] DEPRECATED engine mode '%@' — falling "
110
- @"back to 'slitscan-both'", mode);
111
- m = @"slitscan-both";
112
- }
113
-
114
- // V15.0d — defaults shared by all engine modes for the new knobs.
115
- // Set before the per-mode overrides so per-mode only overrides
116
- // the fields that genuinely differ from this baseline.
117
- c.nccSearchMargin2d = 12; // was hardcoded V15.0c.4
118
- // V15.0i.1 — default raised to 0.99. At 0.75 the NCC was applying
119
- // corrections on weak matches in plane-projected mode (Ram observed
120
- // mid-capture wobble). 0.99 means we only apply corrections on
121
- // near-perfect overlap matches; ambiguous matches are skipped.
122
- c.nccConfidenceThreshold2d = 0.99;
123
- c.enableNcc2dEmaSmoothing = NO; // 1B opt-in
124
- c.ncc2dEmaAlpha = 0.4; // 60% prev / 40% current
125
- c.enableNcc2dPanAxisLock = NO; // 1C opt-in
126
- c.ncc2dCrossAxisLockPx = 5;
127
- c.planeSource = RLISPlaneSourceDisabled;
128
- c.virtualPlaneDepthMeters = 1.5;
129
- c.arkitPlaneAlignmentThreshold = 0.6; // ~53° max off-camera
130
- // V15.0g — Rectified is the default since Trapezoidal's
131
- // tilt-induced distortion was the field-blocker on V15.0e/f.
132
- // Operators can flip back to Trapezoidal for A/B comparison.
133
- c.planeProjectionStyle = RLISPlaneProjectionStyleRectified;
134
-
135
- if ([m isEqualToString:@"hybrid"]) {
136
- // n/a slit-shaping; hybrid uses whole-frame projection.
137
- c.kPanAxisFractionRect = 0.30; // unused for hybrid
138
- c.kMinAcceptDeltaPx = 50;
139
- c.enableTriangulation = NO;
140
- c.enableTriAccumulator = NO;
141
- c.enable1dNcc = NO;
142
- c.nccSearchRadius1d = 15;
143
- c.enable2dNcc = NO;
144
- c.enableRansacHomography = NO;
145
- c.paintMode = RLISPaintModeFeatherBlend; // V12.x feather
146
- c.hybridProjection = RLISHybridProjectionPlanar; // V15: planar default
147
- c.useDetectedPlane = NO;
148
- c.sliverPosition = RLISSliverPositionCenter;
149
- c.firstFrameFullFrame = NO;
150
- } else if ([m isEqualToString:@"slitscan-rotate"]) {
151
- // V13.0a baseline + 1D NCC. No tri, no 2D NCC, no homography.
152
- c.kPanAxisFractionRect = 0.30;
153
- c.kMinAcceptDeltaPx = 0; // accept on every frame
154
- c.enableTriangulation = NO;
155
- c.enableTriAccumulator = NO;
156
- c.enable1dNcc = YES; // wobble correction
157
- c.nccSearchRadius1d = 15;
158
- c.enable2dNcc = NO;
159
- c.enableRansacHomography = NO;
160
- c.paintMode = RLISPaintModeFirstPaintedWins;
161
- c.hybridProjection = RLISHybridProjectionPlanar; // unused
162
- c.useDetectedPlane = NO;
163
- c.sliverPosition = RLISSliverPositionCenter;
164
- c.firstFrameFullFrame = NO;
165
- } else {
166
- // slitscan-both (V15.0c default). V13.0a baseline + no gate +
167
- // first-painted-wins (Ram observation: feather often introduces
168
- // ghosting; first-painted-wins is consistently the best for our
169
- // typical retail fixture pans). Iterate via settings UI:
170
- // enable tri / 2D NCC / RANSAC as needed.
171
- c.kPanAxisFractionRect = 0.30;
172
- c.kMinAcceptDeltaPx = 0; // accept on every frame
173
- c.enableTriangulation = NO;
174
- c.enableTriAccumulator = NO;
175
- c.enable1dNcc = NO;
176
- c.nccSearchRadius1d = 15;
177
- c.enable2dNcc = NO;
178
- c.enableRansacHomography = NO;
179
- c.paintMode = RLISPaintModeFirstPaintedWins; // V15.0c default
180
- c.hybridProjection = RLISHybridProjectionPlanar; // unused
181
- c.useDetectedPlane = NO;
182
- c.sliverPosition = RLISSliverPositionCenter;
183
- c.firstFrameFullFrame = NO;
184
- }
185
-
186
- return c;
187
- }
188
-
189
- @end
190
-
191
-
192
- // ── Acceptance thresholds ───────────────────────────────────────────
193
- //
194
- // All values empirical seeds from the design doc. Documented here
195
- // alongside the code so a tuning pass during field testing can adjust
196
- // them without hunting through the .h.
197
-
198
- namespace {
199
-
200
- // FoV gate window — slightly more permissive than the design doc's
201
- // 30-50% sweet spot to handle pose noise + slow pans without
202
- // rejecting valid candidates.
203
- constexpr double kMinOverlapPct = 10.0; // below this → moved too far
204
- constexpr double kMaxOverlapPct = 75.0; // above this → too close, wait
205
- // Match-quality gates — relaxed from the design doc seeds because
206
- // shelf scenes with light textures (white walls behind shelves,
207
- // uniform packaging) produce fewer matches than the 80-feature
208
- // "ideal". Field tuning in v3.
209
- constexpr int kMinMatchesAccept = 10;
210
- constexpr double kMinInlierRatioAccept = 0.18;
211
- constexpr double kHighConfidenceMatches = 60;
212
- constexpr double kHighConfidenceInlierRatio = 0.55;
213
- constexpr int kOrbMaxFeatures = 1000;
214
- constexpr float kLoweRatio = 0.75f;
215
- constexpr double kRansacReprojThresh = 5.0;
216
- // Similarity (4-DOF: scale, rotation, tx, ty) keeps the determinant
217
- // equal to scale². Tight bounds reject degenerate fits aggressively
218
- // while leaving slack for the natural ~0.9-1.1 scale that hand-held
219
- // pans introduce (parallax + lens distortion residuals).
220
- // V11 Gap #30: deleted dead kHomDetMin/kHomDetMax — pose-driven path
221
- // doesn't fit a homography, so the determinant bounds were dead.
222
-
223
- } // namespace
224
-
225
- // ── Engine impl ─────────────────────────────────────────────────────
226
-
227
- @implementation OpenCVIncrementalStitcher {
228
- NSInteger _composeWidth;
229
- NSInteger _composeHeight;
230
- NSInteger _canvasWidth;
231
- NSInteger _canvasHeight;
232
- NSInteger _featherPx;
233
- NSInteger _frameRotationDegrees;
234
- /// V12.3 orientation-aware cylinder axis. Derived from
235
- /// frameRotationDegrees: 0/180 → landscape (axis = pan_X,
236
- /// transverse cylinder; pan direction is vertical world); 90/270 →
237
- /// portrait (axis = pan_Y, vertical-axis cylinder; pan direction
238
- /// is horizontal world). Apple's pano follows the same rule:
239
- /// pan along the device's longer side, projection wraps around
240
- /// the shorter side.
241
- BOOL _isLandscape;
242
-
243
- cv::Mat _canvas; // CV_8UC3 BGR — the running panorama
244
- cv::Mat _canvasMask; // CV_8UC1 — 255 where canvas has been written
245
-
246
- /// V9 pose-driven state — hand-rolled cylindrical projection.
247
- /// Replaces v7's planar `H = K · R · K⁻¹` with cylindrical
248
- /// remap into a gravity-aligned panorama frame. The panorama
249
- /// frame is defined at first-frame time:
250
- /// +Y = world-gravity-up (cylinder axis is vertical)
251
- /// +Z = horizontal projection of first camera's forward
252
- /// (theta=0 sits at first frame's centre; no wraparound)
253
- /// +X = +Y × +Z (right-handed)
254
- /// This avoids the v8 bug where cv::detail::CylindricalWarper
255
- /// placed the cylinder seam directly in front of the camera.
256
- cv::Mat _firstRotationArkit; // 3x3 CV_64F, ARKit camera-to-world
257
- cv::Mat _K_compose; // 3x3 CV_64F, intrinsics scaled to compose dims
258
- cv::Mat _M_arkitToCv; // diag(1, -1, -1) basis flip
259
- cv::Mat _T_canvas; // (legacy from v7; unused in v9)
260
- cv::Mat _R_panToWorld; // 3x3 CV_64F, panorama-to-world (cached at first frame)
261
- double _focalCompose; // cylinder radius in compose pixels
262
- int _canvasOriginCylX; // canvas (0,0) in cylindrical pixel space (origin offset)
263
- int _canvasOriginCylY;
264
- /// V11 Gap #5: last-accepted rotation matrix (ARKit cam-to-world).
265
- /// The pose-delta gate computes the relative rotation between
266
- /// frames in SENSOR FRAME (axes fixed to the device hardware),
267
- /// not in world frame. World-frame yaw/pitch were being compared
268
- /// against camera-frame FoV, which broke landscape pitch pans
269
- /// (the dominant pan axis is rotation about world-Y, but in
270
- /// landscape the sensor sees that as pitch-equivalent motion).
271
- cv::Mat _lastAcceptedR;
272
-
273
- double _lastAcceptedYaw;
274
- double _lastAcceptedPitch;
275
- bool _hasFirstFrame;
276
-
277
- NSInteger _accepted;
278
- /// Monotonic snapshot sequence — used to mint a unique path per
279
- /// live snapshot. RN's <Image> caches `file://` URIs by path
280
- /// alone and ignores cache-bust query strings, so writing to the
281
- /// SAME path each accept made the live preview show the FIRST
282
- /// frame forever. Bumping the path each snapshot side-steps
283
- /// the cache.
284
- NSInteger _snapshotSeq;
285
-
286
- /// V15 — runtime config controlling projection (cylindrical vs
287
- /// planar) and other hybrid-specific knobs. Set via -setConfig:
288
- /// after init; defaults to hybrid factory config (planar) if
289
- /// never set.
290
- RLISStitcherConfig *_config;
291
- }
292
-
293
- - (instancetype)initWithComposeWidth:(NSInteger)composeWidth
294
- composeHeight:(NSInteger)composeHeight
295
- canvasWidth:(NSInteger)canvasWidth
296
- canvasHeight:(NSInteger)canvasHeight
297
- featherPx:(NSInteger)featherPx
298
- frameRotationDegrees:(NSInteger)frameRotationDegrees
299
- {
300
- if (self = [super init]) {
301
- // V7: frameRotationDegrees is now an OUTPUT-ONLY rotation —
302
- // it's applied at snapshot/finalize time to orient the saved
303
- // JPEG for display. The compute pipeline always works in
304
- // sensor-native landscape compose space.
305
- _frameRotationDegrees = frameRotationDegrees;
306
- // V12.6 Step C: _isLandscape is no longer derived from the
307
- // JS-passed frameRotationDegrees. V12.5 telemetry proved
308
- // JS was sending the wrong value under portrait-locked
309
- // hosts (the lock suppresses RN's rotation event so
310
- // `useWindowDimensions` always reported portrait even in
311
- // landscape). We now detect at first-frame init from
312
- // R_panToCam directly — see the cylindricalWarp's
313
- // first-frame branch. Pose-derived detection works for
314
- // both portrait-locked and non-locked R2-lite hosts.
315
- // Default false here is just a safe initialiser; it WILL
316
- // be overwritten before any warping happens.
317
- _isLandscape = NO;
318
- // Default compose dims preserve the 4:3 sensor aspect
319
- // (1920x1440 → 960x720 at scale 0.5). Always landscape
320
- // because we no longer rotate input; the canvas geometry
321
- // matches the sensor's pixel-shift direction for either
322
- // yaw or pitch pan, in either device orientation.
323
- _composeWidth = composeWidth > 0 ? composeWidth : 960;
324
- _composeHeight = composeHeight > 0 ? composeHeight : 720;
325
- // Canvas: wide-landscape because for the typical shelf-scan
326
- // use case (portrait phone, left-right yaw pan), the sensor
327
- // sees content shifting along its X axis (the wide 1920
328
- // axis). Canvas-X covers ~3 frame-widths of pan; canvas-Y
329
- // covers one frame plus pitch-wobble headroom.
330
- // V11 Gap #4: square canvas so EITHER pan axis fits. The
331
- // primary use case (top-to-bottom landscape pan) needs canvas-Y
332
- // ≥ 3000 px to cover ~90° pitch at typical compose focal — the
333
- // earlier 2200 px clipped the pan after 4-5 frames. 5000² is
334
- // ~88 MB (canvas + mask), comfortable on iPhone 13+ where the
335
- // app is targeted. Real auto-grow is deferred — flat over-
336
- // allocation is simpler and works for both use cases.
337
- _canvasWidth = canvasWidth > 0 ? canvasWidth : 5000;
338
- _canvasHeight = canvasHeight > 0 ? canvasHeight : 5000;
339
- _featherPx = featherPx > 0 ? featherPx : 20;
340
-
341
- // ARKit camera frame (Y-up, -Z forward) → OpenCV camera frame
342
- // (Y-down, +Z forward). Pre-multiplying ARKit-rotation by
343
- // M (and post-multiplying by M, since M is its own inverse)
344
- // converts the rotation into OpenCV camera-frame conventions
345
- // before we plug it into the pinhole projection K.
346
- _M_arkitToCv = (cv::Mat_<double>(3, 3) <<
347
- 1, 0, 0,
348
- 0, -1, 0,
349
- 0, 0, -1);
350
-
351
- // V15 — default config (hybrid mode → planar projection).
352
- // Caller should override via -setConfig: after init.
353
- _config = [RLISStitcherConfig configForMode:@"hybrid"];
354
-
355
- [self reset];
356
- }
357
- return self;
358
- }
359
-
360
- - (void)setConfig:(RLISStitcherConfig *)config {
361
- if (config == nil) return;
362
- _config = config;
363
- NSLog(@"[V15-config] hybrid config applied: hybridProjection=%@",
364
- _config.hybridProjection == RLISHybridProjectionPlanar
365
- ? @"Planar" : @"Cylindrical");
366
- }
367
-
368
- - (void)reset {
369
- _canvas = cv::Mat::zeros((int)_canvasHeight, (int)_canvasWidth, CV_8UC3);
370
- _canvasMask = cv::Mat::zeros((int)_canvasHeight, (int)_canvasWidth, CV_8UC1);
371
- _firstRotationArkit = cv::Mat();
372
- _K_compose = cv::Mat();
373
- _T_canvas = cv::Mat::eye(3, 3, CV_64F);
374
- _lastAcceptedYaw = 0.0;
375
- _lastAcceptedPitch = 0.0;
376
- _lastAcceptedR = cv::Mat(); // V11 Gap #5: clear sensor-frame gate state
377
- _hasFirstFrame = false;
378
- _accepted = 0;
379
- _snapshotSeq = 0;
380
- }
381
-
382
- - (NSInteger)acceptedCount { return _accepted; }
383
-
384
- // ── Public: ingestPixelBuffer (V6 pose-driven) ─────────────────────
385
-
386
- - (RLISFrameTelemetry *)ingestPixelBuffer:(CVPixelBufferRef)pixelBuffer
387
- qx:(double)qx
388
- qy:(double)qy
389
- qz:(double)qz
390
- qw:(double)qw
391
- tx:(double)tx
392
- ty:(double)ty
393
- tz:(double)tz
394
- fx:(double)fx
395
- fy:(double)fy
396
- cx:(double)cx
397
- cy:(double)cy
398
- imageWidth:(NSInteger)imageWidth
399
- imageHeight:(NSInteger)imageHeight
400
- yaw:(double)yaw
401
- pitch:(double)pitch
402
- fovHorizDegrees:(double)fovHorizDegrees
403
- fovVertDegrees:(double)fovVertDegrees
404
- trackingPoor:(BOOL)trackingPoor
405
- {
406
- // V13.0e — hybrid engine accepts tx/ty/tz for API symmetry with the
407
- // slit-scan engine but does not (yet) use them. The Samsung-style
408
- // hybrid path's robustness comes from feature-matching its overlap
409
- // each frame; pose translation correction layered on top would be
410
- // redundant. Suppress unused-warning explicitly so the call stays
411
- // semantically tied to the slit engine.
412
- (void)tx; (void)ty; (void)tz;
413
-
414
- auto t0 = std::chrono::steady_clock::now();
415
-
416
- RLISFrameTelemetry *tele = [[RLISFrameTelemetry alloc] init];
417
- tele.overlapPercent = -1;
418
-
419
- if (trackingPoor) {
420
- tele.outcome = RLISFrameOutcomeSkippedTrackingPoor;
421
- tele.processingMs = msSince(t0);
422
- return tele;
423
- }
424
-
425
- cv::Mat frameBGR;
426
- if (![self convertPixelBuffer:pixelBuffer toMat:frameBGR]) {
427
- tele.outcome = RLISFrameOutcomeSkippedTrackingPoor;
428
- tele.processingMs = msSince(t0);
429
- return tele;
430
- }
431
-
432
- // Build R_new (3x3, ARKit-camera-to-world rotation) from the
433
- // quaternion.
434
- cv::Mat R_new = quaternionToRotationMat(qx, qy, qz, qw);
435
-
436
- // First frame: place at canvas centre, store the reference pose
437
- // and intrinsics, accept unconditionally. All subsequent frames
438
- // compute their pose-driven homography RELATIVE to this first
439
- // frame.
440
- if (!_hasFirstFrame) {
441
- _firstRotationArkit = R_new.clone();
442
- // V11 Gap #1: per-axis intrinsics scaling (was averaging into
443
- // a single scalar — silently distorts whenever compose dims
444
- // ratio ≠ sensor dims ratio). K_compose = diag(sx,sy,1)·K.
445
- double sx = (double)frameBGR.cols / std::max((NSInteger)1, imageWidth);
446
- double sy = (double)frameBGR.rows / std::max((NSInteger)1, imageHeight);
447
- _K_compose = (cv::Mat_<double>(3, 3) <<
448
- fx * sx, 0, cx * sx,
449
- 0, fy * sy, cy * sy,
450
- 0, 0, 1);
451
- // V11 Gap #2: cylinder radius = geometric mean of compose
452
- // focals. Was just `fx * s`, which made canvas h-spacing
453
- // inconsistent with theta-spacing for any anisotropic-pixel
454
- // intrinsic.
455
- _focalCompose = std::sqrt((fx * sx) * (fy * sy));
456
-
457
- // V9: build the panorama-to-world rotation from the first
458
- // ARKit pose. Panorama +Z = horizontal projection of first-
459
- // camera forward. For PORTRAIT (cylinder axis = vertical):
460
- // panorama +Y = world up. For LANDSCAPE (cylinder axis =
461
- // horizontal pan rotation axis): V14.0pre — panorama +Y =
462
- // first-camera +X (sensor X = phone long edge in landscape =
463
- // pan rotation axis). This makes pano-Y the actual rotation
464
- // axis, eliminating the V12.x landscape-projection roll-
465
- // sensitivity bug (see 2026-05-07-v14-stitcher-plan.md).
466
- cv::Mat fwdArkitCam = (cv::Mat_<double>(3, 1) << 0, 0, -1);
467
- cv::Mat fwdWorld = _firstRotationArkit * fwdArkitCam;
468
- double fwx = fwdWorld.at<double>(0);
469
- double fwz = fwdWorld.at<double>(2);
470
- double horiz = std::sqrt(fwx * fwx + fwz * fwz);
471
- // V11 Gap #3: reject if camera looking nearly vertical. The
472
- // panorama frame needs a horizontal +Z anchor; if camera
473
- // forward is gravity-aligned, the horizontal projection is
474
- // degenerate.
475
- if (horiz < 0.1) {
476
- tele.outcome = RLISFrameOutcomeRejectedAlignmentLost;
477
- tele.processingMs = msSince(t0);
478
- return tele;
479
- }
480
- double pzx = fwx / horiz;
481
- double pzz = fwz / horiz;
482
-
483
- // V14.0pre — orientation detection BEFORE _R_panToWorld
484
- // construction (was V12.6 detection done AFTER, using
485
- // R_panToCam_first which itself depended on _R_panToWorld —
486
- // chicken-and-egg).
487
- //
488
- // V14.0pre.1 — comparison INVERTED after V14.0pre field test
489
- // showed it firing backwards. Hardware geometry: the phone's
490
- // sensor-Y axis (cam-Y) is along the SHORT edge of the phone.
491
- // In LANDSCAPE the phone is held long-edge horizontal, so
492
- // cam-Y points UP in the user's view (= along world-up =
493
- // gravity). In PORTRAIT it points sideways (horizontal).
494
- // V14.0pre had max(|X|,|Z|) > |Y| firing as "landscape" — that
495
- // pattern actually identifies PORTRAIT. Field log showed
496
- // |camY.worldY|=0.937 (clearly landscape) firing isLandscape=0.
497
- // Inverted comparison matches V12.6 slit-scan detection's
498
- // direction (absR11 > absR01).
499
- cv::Mat camYInWorld = _firstRotationArkit *
500
- (cv::Mat_<double>(3, 1) << 0, 1, 0);
501
- const double absCamYInWorldX = std::fabs(camYInWorld.at<double>(0));
502
- const double absCamYInWorldY = std::fabs(camYInWorld.at<double>(1));
503
- const double absCamYInWorldZ = std::fabs(camYInWorld.at<double>(2));
504
- _isLandscape = (absCamYInWorldY
505
- > std::max(absCamYInWorldX, absCamYInWorldZ));
506
- NSLog(@"[V14.0pre-orient] engine=hybrid isLandscape=%d "
507
- @"|camY.worldX|=%.4f |camY.worldY|=%.4f |camY.worldZ|=%.4f",
508
- (int)_isLandscape, absCamYInWorldX, absCamYInWorldY,
509
- absCamYInWorldZ);
510
-
511
- if (_isLandscape) {
512
- // V14.0pre LANDSCAPE: pano-Y = first-cam +X axis (the pan
513
- // rotation axis for vertical pan). Cylinder axis = pano-Y
514
- // = cam-X → roll around cam-Z just slides pixels along
515
- // theta with no asymmetric distortion. Pano-Z = horizontal
516
- // projection of first-cam forward (already computed above).
517
- // Pano-X = pano-Y × pano-Z (right-handed completion;
518
- // approximately gravity-up for level first frame).
519
- //
520
- // pano-Y = R_first · (1,0,0) (cam-X in world coords)
521
- cv::Mat camXInWorld = _firstRotationArkit *
522
- (cv::Mat_<double>(3, 1) << 1, 0, 0);
523
- const double pyx = camXInWorld.at<double>(0);
524
- const double pyy = camXInWorld.at<double>(1);
525
- const double pyz = camXInWorld.at<double>(2);
526
- // pano-Z = (pzx, 0, pzz) by construction above.
527
- // pano-X = pano-Y × pano-Z; with pano-Z having zero Y comp:
528
- // px.x = pyy*pzz - pyz*0 = pyy*pzz
529
- // px.y = pyz*pzx - pyx*pzz
530
- // px.z = pyx*0 - pyy*pzx = -pyy*pzx
531
- const double pxx = pyy * pzz;
532
- const double pxy = pyz * pzx - pyx * pzz;
533
- const double pxz = -pyy * pzx;
534
- // Columns of _R_panToWorld are pano-X, pano-Y, pano-Z.
535
- _R_panToWorld = (cv::Mat_<double>(3, 3) <<
536
- pxx, pyx, pzx,
537
- pxy, pyy, 0.0,
538
- pxz, pyz, pzz);
539
- } else {
540
- // PORTRAIT (unchanged from V9): pano-Y = world up.
541
- // pano-X = pano-Y × pano-Z = (0,1,0) × (pzx,0,pzz) = (pzz, 0, -pzx)
542
- _R_panToWorld = (cv::Mat_<double>(3, 3) <<
543
- pzz, 0, pzx,
544
- 0, 1, 0,
545
- -pzx, 0, pzz);
546
- }
547
-
548
- // V14.0pre — sanity check the constructed pano frame is
549
- // orthonormal (det ≈ 1, R · R^T ≈ I). Logs once per capture
550
- // (first-frame). If det ≈ -1 or off-diagonals are large, the
551
- // pano-X cross-product computation has a sign or transcription
552
- // bug — fix BEFORE proceeding.
553
- const double pano_det = cv::determinant(_R_panToWorld);
554
- cv::Mat pano_RRT = _R_panToWorld * _R_panToWorld.t();
555
- NSLog(@"[V14.0pre-pano] det=%.4f I[0,0]=%.4f I[1,1]=%.4f I[2,2]=%.4f "
556
- @"I[0,1]=%.4f I[0,2]=%.4f I[1,2]=%.4f",
557
- pano_det,
558
- pano_RRT.at<double>(0,0), pano_RRT.at<double>(1,1),
559
- pano_RRT.at<double>(2,2), pano_RRT.at<double>(0,1),
560
- pano_RRT.at<double>(0,2), pano_RRT.at<double>(1,2));
561
-
562
- // Place first frame onto canvas via cylindrical warp. R for
563
- // the warp is panorama→camera in OpenCV cam frame; for the
564
- // first frame this is approximately identity (camera-forward
565
- // = panorama +Z). The cylindrical warp gives us a warped
566
- // image + a corner in cylindrical-pixel (theta, h)·f space.
567
- cv::Mat warpedFirst, warpedFirstMask;
568
- cv::Point firstCornerCyl =
569
- [self cylindricalWarp:frameBGR rArkit:R_new
570
- outImage:warpedFirst outMask:warpedFirstMask];
571
-
572
- // Anchor the first frame at canvas centre.
573
- int dstX = (int)(_canvas.cols - warpedFirst.cols) / 2;
574
- int dstY = (int)(_canvas.rows - warpedFirst.rows) / 2;
575
- cv::Rect roi(dstX, dstY, warpedFirst.cols, warpedFirst.rows);
576
- roi &= cv::Rect(0, 0, _canvas.cols, _canvas.rows);
577
- cv::Rect srcRoi(0, 0, roi.width, roi.height);
578
- warpedFirst(srcRoi).copyTo(_canvas(roi), warpedFirstMask(srcRoi));
579
- warpedFirstMask(srcRoi).copyTo(_canvasMask(roi),
580
- warpedFirstMask(srcRoi));
581
-
582
- // Track the cylindrical pixel that lives at canvas (0, 0).
583
- // Subsequent frames' cylindrical corners → canvas position
584
- // by subtracting this origin.
585
- _canvasOriginCylX = firstCornerCyl.x - dstX;
586
- _canvasOriginCylY = firstCornerCyl.y - dstY;
587
-
588
- _lastAcceptedYaw = yaw;
589
- _lastAcceptedPitch = pitch;
590
- _lastAcceptedR = R_new.clone();
591
- _hasFirstFrame = true;
592
- _accepted = 1;
593
- tele.outcome = RLISFrameOutcomeAcceptedHigh;
594
- tele.confidence = 1.0;
595
- tele.overlapPercent = 0;
596
- tele.processingMs = msSince(t0);
597
- return tele;
598
- }
599
-
600
- // V11 Gap #5: pose-delta gate in SENSOR FRAME.
601
- //
602
- // Compute the relative rotation between the last-accepted frame
603
- // and this frame, expressed in the previous frame's CAMERA-LOCAL
604
- // axes (these are fixed to the device sensor hardware regardless
605
- // of how the user is holding the phone).
606
- //
607
- // R_relative_in_prev_cam = R_prev⁻¹ · R_new (column-vector convention)
608
- //
609
- // For ARKit camera axes (+X right, +Y up, −Z forward):
610
- // rotation about sensor +Y → scene shifts horizontally on screen
611
- // → compare against fovH
612
- // rotation about sensor +X → scene shifts vertically on screen
613
- // → compare against fovV
614
- //
615
- // For small-angle rotations (typical accept window 5-25°),
616
- // cv::Rodrigues' axis-angle vector ≈ (rotX, rotY, rotZ) with
617
- // each component being the rotation about that sensor axis.
618
- cv::Mat R_relSensor = _lastAcceptedR.t() * R_new;
619
- cv::Mat rvec;
620
- cv::Rodrigues(R_relSensor, rvec);
621
- double sensorRotX = std::fabs(rvec.at<double>(0));
622
- double sensorRotY = std::fabs(rvec.at<double>(1));
623
- double overlap = computeOverlapPctSensor(
624
- sensorRotX,
625
- sensorRotY,
626
- fovHorizDegrees,
627
- fovVertDegrees
628
- );
629
- tele.overlapPercent = overlap;
630
-
631
- if (overlap > kMaxOverlapPct) {
632
- tele.outcome = RLISFrameOutcomeSkippedTooClose;
633
- tele.processingMs = msSince(t0);
634
- return tele;
635
- }
636
- if (overlap < kMinOverlapPct) {
637
- tele.outcome = RLISFrameOutcomeRejectedTooFar;
638
- tele.processingMs = msSince(t0);
639
- return tele;
640
- }
641
-
642
- // V12.2 cylindrical warp (with V12 mirror fix) + feather blend.
643
- cv::Mat warpedNew, warpedNewMask;
644
- cv::Point newCornerCyl =
645
- [self cylindricalWarp:frameBGR rArkit:R_new
646
- outImage:warpedNew outMask:warpedNewMask];
647
- if (warpedNew.empty()) {
648
- tele.outcome = RLISFrameOutcomeRejectedAlignmentLost;
649
- tele.processingMs = msSince(t0);
650
- return tele;
651
- }
652
-
653
- // Map cylindrical-pixel corner to canvas-pixel corner.
654
- cv::Point newCornerCanvas(newCornerCyl.x - _canvasOriginCylX,
655
- newCornerCyl.y - _canvasOriginCylY);
656
-
657
- // V9b: optical-flow refinement. ARKit pose accuracy is ~1-2°,
658
- // which translates to ~25-50 px residual misalignment at typical
659
- // focal lengths. KLT flow on a sparse grid in the overlap
660
- // region recovers sub-pixel accuracy without needing the full
661
- // ORB+RANSAC machinery from the v1-v3 path. The result is a
662
- // single (dx, dy) translation applied to the canvas placement.
663
- cv::Point2f shift = [self refineWithOpticalFlow:warpedNew
664
- newMask:warpedNewMask
665
- canvasOrigin:newCornerCanvas];
666
- newCornerCanvas.x += (int)std::round(shift.x);
667
- newCornerCanvas.y += (int)std::round(shift.y);
668
-
669
- cv::Rect dstRoi(newCornerCanvas.x, newCornerCanvas.y,
670
- warpedNew.cols, warpedNew.rows);
671
- cv::Rect canvasBounds(0, 0, _canvas.cols, _canvas.rows);
672
- cv::Rect dstClipped = dstRoi & canvasBounds;
673
- if (dstClipped.width <= 0 || dstClipped.height <= 0) {
674
- tele.outcome = RLISFrameOutcomeRejectedAlignmentLost;
675
- tele.processingMs = msSince(t0);
676
- return tele;
677
- }
678
- cv::Rect srcRoi(dstClipped.x - dstRoi.x, dstClipped.y - dstRoi.y,
679
- dstClipped.width, dstClipped.height);
680
-
681
- [self featherBlendWarped:warpedNew(srcRoi)
682
- mask:warpedNewMask(srcRoi)
683
- intoCanvas:_canvas(dstClipped)
684
- canvasMask:_canvasMask(dstClipped)];
685
-
686
- _lastAcceptedYaw = yaw;
687
- _lastAcceptedPitch = pitch;
688
- _lastAcceptedR = R_new.clone(); // V11 Gap #5: sensor-frame gate state
689
- _accepted += 1;
690
-
691
- // Pose-driven path is geometrically exact when ARKit tracking is
692
- // good (which we already gated on `trackingPoor`). Confidence
693
- // is a function of the FoV-overlap quality: high near 50%, lower
694
- // at the edges of the [10, 75]% acceptance window.
695
- double midOverlap = 0.5 * (kMinOverlapPct + kMaxOverlapPct);
696
- double overlapDistance = std::fabs(overlap - midOverlap)
697
- / (kMaxOverlapPct - midOverlap);
698
- double confidence = std::max(0.0, 1.0 - overlapDistance);
699
- tele.confidence = confidence;
700
- tele.matchCount = -1; // not applicable in pose-driven path
701
- tele.inlierRatio = -1;
702
- tele.outcome = (confidence >= 0.6)
703
- ? RLISFrameOutcomeAcceptedHigh
704
- : RLISFrameOutcomeAcceptedMedium;
705
- tele.processingMs = msSince(t0);
706
- return tele;
707
- }
708
-
709
- // ── Snapshot / finalize ─────────────────────────────────────────────
710
-
711
- - (nullable RLISSnapshot *)snapshotWithJpegQuality:(NSInteger)quality
712
- error:(NSError **)error
713
- {
714
- _snapshotSeq += 1;
715
- return [self writeSnapshotToPath:[self currentSnapshotPath]
716
- jpegQuality:quality
717
- tightCrop:YES
718
- applyExposureComp:NO
719
- error:error];
720
- }
721
-
722
- - (nullable RLISSnapshot *)finalizeAtPath:(NSString *)outputPath
723
- jpegQuality:(NSInteger)quality
724
- error:(NSError **)error
725
- {
726
- // Final output: bounding-box crop + CLAHE. Runs off the AR
727
- // delegate thread because the Swift wrapper dispatches finalize
728
- // on workQueue. V12 Gap #2: dropped the O(W·H) inscribed-rect
729
- // search — it produced a far thinner output than the actual
730
- // painted region for any non-rectangular pan footprint, and the
731
- // mask edges are clean (no per-pixel artefacts to crop away).
732
- RLISSnapshot *snap = [self writeSnapshotToPath:outputPath
733
- jpegQuality:quality
734
- tightCrop:YES
735
- applyExposureComp:YES
736
- error:error];
737
- [self reset];
738
- return snap;
739
- }
740
-
741
- - (NSString *)currentSnapshotPath {
742
- // Cycle through 4 filenames so RN's image cache sees a new URI
743
- // on every snapshot but tmp dir doesn't grow unbounded over a
744
- // long pan. 4 is enough to outpace RN's most aggressive
745
- // image cache lifetimes; the OS reclaims tmp at app launch
746
- // anyway.
747
- NSString *tmpDir = NSTemporaryDirectory();
748
- NSInteger slot = _snapshotSeq % 4;
749
- NSString *filename = [NSString stringWithFormat:@"rlis-live-%ld.jpg", (long)slot];
750
- return [tmpDir stringByAppendingPathComponent:filename];
751
- }
752
-
753
- - (nullable RLISSnapshot *)writeSnapshotToPath:(NSString *)outputPath
754
- jpegQuality:(NSInteger)quality
755
- tightCrop:(BOOL)tightCrop
756
- applyExposureComp:(BOOL)applyExposureComp
757
- error:(NSError **)error
758
- {
759
- if (_accepted == 0) {
760
- if (error) {
761
- *error = [NSError errorWithDomain:RNImageStitcherIncrementalErrorDomain
762
- code:1
763
- userInfo:@{NSLocalizedDescriptionKey:
764
- @"No frames have been accepted yet."}];
765
- }
766
- return nil;
767
- }
768
-
769
- cv::Rect cropRect(0, 0, _canvas.cols, _canvas.rows);
770
- if (tightCrop) {
771
- cv::Rect bbox = cv::boundingRect(_canvasMask);
772
- if (bbox.width > 0 && bbox.height > 0) {
773
- cropRect = bbox;
774
- }
775
- }
776
- cv::Mat cropped = _canvas(cropRect).clone();
777
-
778
- // V11 Gap #14: NO output rotation needed.
779
- //
780
- // The earlier (v7) sensor-native canvas needed a gravity-derived
781
- // rotation because the canvas was the camera buffer's pixel
782
- // layout — buffer-Y was image-down for portrait, image-right for
783
- // landscape, etc. V8+ switched to a panorama-frame canvas
784
- // (gravity-up = +panorama-Y; the warp Y-flip puts world-up at
785
- // image-top by construction). V12 briefly tried spherical but
786
- // V12.2 reverted to cylindrical — the gravity-alignment guarantee
787
- // is the same. The canvas IS already correctly oriented for
788
- // any device hold.
789
- cv::Mat out = cropped;
790
-
791
- if (applyExposureComp && !out.empty()) {
792
- // CLAHE on the L channel of Lab. Preserves colour, evens
793
- // out luminance variation across the panorama. Conservative
794
- // clipLimit=2.0 — enough to even out auto-exposure bands,
795
- // not so much that it crushes highlight/shadow detail.
796
- cv::Mat lab;
797
- cv::cvtColor(out, lab, cv::COLOR_BGR2Lab);
798
- std::vector<cv::Mat> labChannels(3);
799
- cv::split(lab, labChannels);
800
- cv::Ptr<cv::CLAHE> clahe = cv::createCLAHE(2.0, cv::Size(8, 8));
801
- clahe->apply(labChannels[0], labChannels[0]);
802
- cv::merge(labChannels, lab);
803
- cv::cvtColor(lab, out, cv::COLOR_Lab2BGR);
804
- }
805
-
806
- int q = (int)std::clamp((long long)quality, 0LL, 100LL);
807
- std::vector<int> params = {cv::IMWRITE_JPEG_QUALITY, q};
808
- NSString *cleanPath = [outputPath hasPrefix:@"file://"]
809
- ? [outputPath substringFromIndex:7]
810
- : outputPath;
811
- bool ok = cv::imwrite(std::string([cleanPath UTF8String]), out, params);
812
- NSLog(@"[RLIS-PIP] imwrite path=%@ size=%dx%d quality=%d ok=%d",
813
- cleanPath, out.cols, out.rows, q, (int)ok);
814
- if (!ok) {
815
- if (error) {
816
- *error = [NSError errorWithDomain:RNImageStitcherIncrementalErrorDomain
817
- code:2
818
- userInfo:@{NSLocalizedDescriptionKey:
819
- [NSString stringWithFormat:
820
- @"Failed to write JPEG to %@", outputPath]}];
821
- }
822
- return nil;
823
- }
824
-
825
- RLISSnapshot *snap = [[RLISSnapshot alloc] init];
826
- snap.panoramaPath = cleanPath;
827
- snap.width = out.cols;
828
- snap.height = out.rows;
829
- snap.acceptedCount = _accepted;
830
- return snap;
831
- }
832
-
833
- // ── Internals ───────────────────────────────────────────────────────
834
-
835
- static double msSince(std::chrono::steady_clock::time_point t0) {
836
- auto dt = std::chrono::steady_clock::now() - t0;
837
- return std::chrono::duration_cast<std::chrono::microseconds>(dt).count() / 1000.0;
838
- }
839
-
840
- /// Quaternion (x, y, z, w) → 3x3 rotation matrix (CV_64F).
841
- /// Defensive: normalises if the input isn't unit-length.
842
- static cv::Mat quaternionToRotationMat(double qx, double qy, double qz, double qw) {
843
- double n = std::sqrt(qx*qx + qy*qy + qz*qz + qw*qw);
844
- if (n > 1e-9) { qx /= n; qy /= n; qz /= n; qw /= n; }
845
- return (cv::Mat_<double>(3, 3) <<
846
- 1 - 2*(qy*qy + qz*qz), 2*(qx*qy - qw*qz), 2*(qx*qz + qw*qy),
847
- 2*(qx*qy + qw*qz), 1 - 2*(qx*qx + qz*qz), 2*(qy*qz - qw*qx),
848
- 2*(qx*qz - qw*qy), 2*(qy*qz + qw*qx), 1 - 2*(qx*qx + qy*qy));
849
- }
850
-
851
- // `sensorRotationMatrix` was removed in V7 — the rotation chain it
852
- // powered is no longer in the homography path. See the v7 commit
853
- // for the architectural fix that replaced it with sensor-native
854
- // compute + output-only rotation.
855
-
856
- /// Compute fractional overlap between consecutive frames assuming the
857
- /// camera is rotated about its centre by (deltaYaw, deltaPitch) in
858
- /// radians. Output is in percent. We take the dominant axis (the
859
- /// one with larger angular delta) as the pan axis — overlap on that
860
- /// axis is what determines whether the frames have moved enough.
861
- /// V11 Gap #5: sensor-frame gate.
862
- ///
863
- /// Inputs are absolute rotation magnitudes (radians) about the
864
- /// sensor's +X and +Y axes — these axes are FIXED to the device
865
- /// hardware regardless of how the operator is holding the phone.
866
- ///
867
- /// sensorRotXRad: rotation about sensor +X → vertical scene shift
868
- /// → compare against fovV (sensor's vertical FoV)
869
- /// sensorRotYRad: rotation about sensor +Y → horizontal scene shift
870
- /// → compare against fovH (sensor's horizontal FoV)
871
- ///
872
- /// The "dominant axis = pan direction" heuristic still applies — pick
873
- /// whichever rotation is larger and use the matching FoV to compute
874
- /// the per-axis overlap. Output is overlap percent [0, 100].
875
- static double computeOverlapPctSensor(double sensorRotXRad,
876
- double sensorRotYRad,
877
- double fovHorizDegrees,
878
- double fovVertDegrees)
879
- {
880
- double fovH = fovHorizDegrees * M_PI / 180.0;
881
- double fovV = fovVertDegrees * M_PI / 180.0;
882
- if (fovH <= 1e-6) fovH = 65.0 * M_PI / 180.0;
883
- if (fovV <= 1e-6) fovV = 50.0 * M_PI / 180.0;
884
-
885
- double overlap;
886
- if (sensorRotYRad >= sensorRotXRad) {
887
- overlap = 1.0 - sensorRotYRad / fovH;
888
- } else {
889
- overlap = 1.0 - sensorRotXRad / fovV;
890
- }
891
- return std::clamp(overlap, 0.0, 1.0) * 100.0;
892
- }
893
-
894
- - (BOOL)convertPixelBuffer:(CVPixelBufferRef)pixelBuffer toMat:(cv::Mat &)outBGR {
895
- if (pixelBuffer == NULL) return NO;
896
- OSType pf = CVPixelBufferGetPixelFormatType(pixelBuffer);
897
-
898
- CVReturn lockResult = CVPixelBufferLockBaseAddress(pixelBuffer,
899
- kCVPixelBufferLock_ReadOnly);
900
- if (lockResult != kCVReturnSuccess) return NO;
901
-
902
- cv::Mat frame;
903
- BOOL ok = NO;
904
-
905
- if (pf == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
906
- pf == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
907
- // ARKit's NV12 — Y plane in plane 0, interleaved CbCr in plane 1.
908
- // OpenCV exposes a direct NV12 → BGR conversion.
909
- size_t w = CVPixelBufferGetWidth(pixelBuffer);
910
- size_t h = CVPixelBufferGetHeight(pixelBuffer);
911
- size_t yStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
912
- size_t cStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
913
- uint8_t *yPlane = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
914
- uint8_t *cPlane = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
915
-
916
- // Build a contiguous YUV buffer expected by cvtColorTwoPlane.
917
- // OpenCV provides cvtColorTwoPlane which takes Y and UV planes
918
- // separately — perfect for NV12 with potentially-different
919
- // strides between planes.
920
- cv::Mat yMat((int)h, (int)w, CV_8UC1, yPlane, yStride);
921
- cv::Mat cMat((int)h / 2, (int)w / 2, CV_8UC2, cPlane, cStride);
922
- int code = (pf == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
923
- ? cv::COLOR_YUV2BGR_NV12 : cv::COLOR_YUV2BGR_NV12;
924
- cv::cvtColorTwoPlane(yMat, cMat, frame, code);
925
- ok = YES;
926
- } else if (pf == kCVPixelFormatType_32BGRA) {
927
- size_t w = CVPixelBufferGetWidth(pixelBuffer);
928
- size_t h = CVPixelBufferGetHeight(pixelBuffer);
929
- size_t stride = CVPixelBufferGetBytesPerRow(pixelBuffer);
930
- uint8_t *base = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer);
931
- cv::Mat bgra((int)h, (int)w, CV_8UC4, base, stride);
932
- cv::cvtColor(bgra, frame, cv::COLOR_BGRA2BGR);
933
- ok = YES;
934
- }
935
- CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
936
- if (!ok) return NO;
937
-
938
- // V7: NO input rotation. ARKit delivers sensor-native landscape
939
- // pixels and we keep them that way through the entire compute
940
- // pipeline. This is the architectural fix that resolves the
941
- // v6 rotation-vs-canvas mismatch — we no longer need an `R2S`
942
- // chain in the homography to undo a rotation we shouldn't have
943
- // applied in the first place.
944
- //
945
- // The sensor's native orientation is what the ARKit pose +
946
- // intrinsics describe. Working directly in that frame keeps
947
- // `H = K · R_rel · K⁻¹` clean and bug-free. Output rotation
948
- // for display happens AT SNAPSHOT/FINALIZE time only.
949
- //
950
- // Uniform-scale downsample preserves the 4:3 sensor aspect ratio
951
- // (no non-uniform stretch). Picks whichever dimension hits the
952
- // compose budget first; the other comes out proportional.
953
- double scale = std::min(
954
- (double)_composeWidth / (double)frame.cols,
955
- (double)_composeHeight / (double)frame.rows
956
- );
957
- if (scale > 1.0) scale = 1.0; // never upscale
958
- int outW = std::max(1, (int)std::round(frame.cols * scale));
959
- int outH = std::max(1, (int)std::round(frame.rows * scale));
960
- cv::Size target(outW, outH);
961
- if (frame.cols == outW && frame.rows == outH) {
962
- outBGR = frame;
963
- } else {
964
- cv::resize(frame, outBGR, target, 0, 0, cv::INTER_AREA);
965
- }
966
- return YES;
967
- }
968
-
969
- // `placeFirstFrame` was removed in v6 — the first-frame logic is now
970
- // inlined in `ingestPixelBuffer:` so the engine can capture the
971
- // reference pose + intrinsics in the same place it positions the
972
- // frame on the canvas.
973
-
974
- /// V12.2 hand-rolled CYLINDRICAL projection (with V12 mirror fix
975
- /// kept). V12 had tried spherical to handle extreme pitch, but
976
- /// spherical bulges both axes and made every level frame look
977
- /// fisheye. Reverted to cylindrical here; pitch-axis flexibility
978
- /// will come from making the cylinder axis orientation-aware
979
- /// (Step 3) rather than from changing the projection itself.
980
- ///
981
- /// Cylinder parameterised by:
982
- /// theta = horizontal angle around panorama-Y, atan2(-wx, wz)
983
- /// (the −wx is the V12 mirror fix —
984
- /// panorama-X is "user's left" in our
985
- /// right-handed setup, so we flip X before
986
- /// atan2 to put user's-right at canvas-right)
987
- /// h = wy / sqrt(wx² + wz²) (height up the cylinder)
988
- /// pixel = (focal · theta, -focal · h)
989
- /// (the −h is the Y-flip — panorama +Y is
990
- /// gravity-up, image +Y is image-down)
991
- ///
992
- /// Inverse map:
993
- /// theta = canvas_x / focal
994
- /// h = -canvas_y / focal
995
- /// ray = (-sin(theta), h, cos(theta))
996
- ///
997
- /// Vertical lines (perpendicular to the cylinder axis) stay
998
- /// straight in the projection — that's why cylindrical produces a
999
- /// natural-looking panorama for level scenes. Pitch close to ±90°
1000
- /// (looking straight up/down) makes h = wy/sqrt(wx²+wz²) blow up
1001
- /// and the bbox grows unbounded; the canvas-x2 sanity check rejects
1002
- /// those frames.
1003
- ///
1004
- /// Returns the bbox's top-left in cylindrical-pixel coords; the
1005
- /// caller adds the canvas origin offset to land it on the canvas.
1006
- - (cv::Point)cylindricalWarp:(const cv::Mat &)src
1007
- rArkit:(const cv::Mat &)rArkit
1008
- outImage:(cv::Mat &)outImage
1009
- outMask:(cv::Mat &)outMask
1010
- {
1011
- if (_R_panToWorld.empty() || _focalCompose <= 0) {
1012
- outImage = cv::Mat();
1013
- outMask = cv::Mat();
1014
- return cv::Point(0, 0);
1015
- }
1016
-
1017
- // V14.0pre — cv::detail::CylindricalWarper does the projection.
1018
- // R = camera-to-panorama rotation in OpenCV camera frame. K is
1019
- // intrinsics in CV_32F (warper requires float). Cylinder radius
1020
- // = focal length.
1021
- //
1022
- // Replaces ~240 lines of hand-rolled per-corner forward projection
1023
- // + bbox computation + inverse-map remap. Same K + R inputs, same
1024
- // cv::Point corner output (top-left of warped image in cylindrical-
1025
- // pixel space). Battle-tested edge handling, antialiased remap.
1026
- cv::Mat R_panToCam = _M_arkitToCv * rArkit.t() * _R_panToWorld;
1027
- cv::Mat R_camToPan = R_panToCam.t();
1028
-
1029
- cv::Mat K32, R32;
1030
- _K_compose.convertTo(K32, CV_32F);
1031
- R_camToPan.convertTo(R32, CV_32F);
1032
-
1033
- // V15 — projection selectable via _config.hybridProjection.
1034
- // Default is Planar (cv::detail::PlaneWarper) for V15 hybrid mode,
1035
- // because cylindrical projection has the V12.x roll-asymmetry bug
1036
- // that's been documented in the V14 spec. Planar is well-behaved
1037
- // for pans <60°, which is the typical retail use case.
1038
- cv::Point corner;
1039
- cv::Mat whiteFrame(src.size(), CV_8UC1, cv::Scalar(255));
1040
-
1041
- if (_config.hybridProjection == RLISHybridProjectionPlanar) {
1042
- cv::detail::PlaneWarper warper((float)_focalCompose);
1043
- corner = warper.warp(src, K32, R32,
1044
- cv::INTER_LINEAR,
1045
- cv::BORDER_REFLECT,
1046
- outImage);
1047
- warper.warp(whiteFrame, K32, R32,
1048
- cv::INTER_NEAREST,
1049
- cv::BORDER_CONSTANT,
1050
- outMask);
1051
- } else {
1052
- cv::detail::CylindricalWarper warper((float)_focalCompose);
1053
- corner = warper.warp(src, K32, R32,
1054
- cv::INTER_LINEAR,
1055
- cv::BORDER_REFLECT,
1056
- outImage);
1057
- warper.warp(whiteFrame, K32, R32,
1058
- cv::INTER_NEAREST,
1059
- cv::BORDER_CONSTANT,
1060
- outMask);
1061
- }
1062
-
1063
- static bool _v14LoggedFirstWarp = false;
1064
- if (!_v14LoggedFirstWarp) {
1065
- _v14LoggedFirstWarp = true;
1066
- NSLog(@"[V15-warp] hybrid projection=%@ corner=(%d,%d) outSize=%dx%d focal=%.1f",
1067
- _config.hybridProjection == RLISHybridProjectionPlanar
1068
- ? @"Planar" : @"Cylindrical",
1069
- corner.x, corner.y, outImage.cols, outImage.rows, _focalCompose);
1070
- NSLog(@"[V14.0pre-warp] OpenCV CylindricalWarper "
1071
- @"corner=(%d,%d) outSize=%dx%d focal=%.1f",
1072
- corner.x, corner.y, outImage.cols, outImage.rows,
1073
- _focalCompose);
1074
- }
1075
-
1076
- return corner;
1077
- }
1078
-
1079
- /// V9b KLT optical flow refinement. Computes a residual translation
1080
- /// (dx, dy) the new warped frame should be shifted by to align
1081
- /// pixel-perfectly with the existing canvas in their overlap region.
1082
- ///
1083
- /// Algorithm:
1084
- /// 1. Compute the overlap rect between warpedNew (placed at
1085
- /// canvasOrigin) and the existing canvas mask.
1086
- /// 2. Convert both regions to grayscale.
1087
- /// 3. `cv::goodFeaturesToTrack` on the canvas overlap: find ~50
1088
- /// strong corners.
1089
- /// 4. `cv::calcOpticalFlowPyrLK` to track those corners into the
1090
- /// warped overlap.
1091
- /// 5. Median (dx, dy) over inlier tracks = residual shift.
1092
- ///
1093
- /// Returns (0, 0) if not enough tracks, or if the shift exceeds a
1094
- /// sanity threshold (likely a bad frame, don't bias the placement).
1095
- - (cv::Point2f)refineWithOpticalFlow:(const cv::Mat &)warpedNew
1096
- newMask:(const cv::Mat &)warpedNewMask
1097
- canvasOrigin:(cv::Point)canvasOrigin
1098
- {
1099
- if (_accepted == 0) return cv::Point2f(0, 0);
1100
-
1101
- // Compute overlap rect (canvas coords).
1102
- cv::Rect newRect(canvasOrigin.x, canvasOrigin.y,
1103
- warpedNew.cols, warpedNew.rows);
1104
- cv::Rect canvasBounds(0, 0, _canvas.cols, _canvas.rows);
1105
- cv::Rect newOnCanvas = newRect & canvasBounds;
1106
- if (newOnCanvas.width < 16 || newOnCanvas.height < 16) {
1107
- return cv::Point2f(0, 0);
1108
- }
1109
-
1110
- cv::Mat canvasOverlap = _canvas(newOnCanvas);
1111
- cv::Mat canvasMaskOverlap = _canvasMask(newOnCanvas);
1112
- cv::Mat warpedOverlap = warpedNew(cv::Rect(
1113
- newOnCanvas.x - newRect.x, newOnCanvas.y - newRect.y,
1114
- newOnCanvas.width, newOnCanvas.height));
1115
- cv::Mat warpedMaskOverlap = warpedNewMask(cv::Rect(
1116
- newOnCanvas.x - newRect.x, newOnCanvas.y - newRect.y,
1117
- newOnCanvas.width, newOnCanvas.height));
1118
-
1119
- // Both regions need pixels — bail if either side is mostly empty.
1120
- int canvasFilled = cv::countNonZero(canvasMaskOverlap);
1121
- int warpedFilled = cv::countNonZero(warpedMaskOverlap);
1122
- int totalPx = newOnCanvas.width * newOnCanvas.height;
1123
- if (canvasFilled < totalPx / 4 || warpedFilled < totalPx / 4) {
1124
- return cv::Point2f(0, 0);
1125
- }
1126
-
1127
- cv::Mat canvasGray, warpedGray;
1128
- cv::cvtColor(canvasOverlap, canvasGray, cv::COLOR_BGR2GRAY);
1129
- cv::cvtColor(warpedOverlap, warpedGray, cv::COLOR_BGR2GRAY);
1130
-
1131
- // Find strong corners in canvas, restricted to the overlap mask.
1132
- std::vector<cv::Point2f> canvasPts;
1133
- cv::Mat featureMask;
1134
- cv::bitwise_and(canvasMaskOverlap, warpedMaskOverlap, featureMask);
1135
- cv::goodFeaturesToTrack(canvasGray, canvasPts, /*maxCorners=*/64,
1136
- /*qualityLevel=*/0.01, /*minDistance=*/10,
1137
- featureMask, /*blockSize=*/3, false, 0.04);
1138
- if (canvasPts.size() < 8) return cv::Point2f(0, 0);
1139
-
1140
- // Forward track: canvas → warped.
1141
- std::vector<cv::Point2f> warpedPts;
1142
- std::vector<uchar> statusFwd;
1143
- std::vector<float> errFwd;
1144
- cv::calcOpticalFlowPyrLK(canvasGray, warpedGray,
1145
- canvasPts, warpedPts, statusFwd, errFwd,
1146
- cv::Size(21, 21), 3,
1147
- cv::TermCriteria(
1148
- cv::TermCriteria::COUNT
1149
- | cv::TermCriteria::EPS,
1150
- 20, 0.03));
1151
-
1152
- // V11 Gap #9: forward-backward bidirectional check. Track the
1153
- // forward-tracked points BACK into the canvas frame; reject any
1154
- // point whose round-trip error exceeds 1 px. Status flag and
1155
- // patch-error threshold catch obvious failures, but trackers
1156
- // can drift inside the search window without flipping status —
1157
- // FB error is the standard reliability gate (every production
1158
- // OF aligner uses it).
1159
- std::vector<cv::Point2f> canvasPtsRev;
1160
- std::vector<uchar> statusBwd;
1161
- std::vector<float> errBwd;
1162
- cv::calcOpticalFlowPyrLK(warpedGray, canvasGray,
1163
- warpedPts, canvasPtsRev, statusBwd, errBwd,
1164
- cv::Size(21, 21), 3,
1165
- cv::TermCriteria(
1166
- cv::TermCriteria::COUNT
1167
- | cv::TermCriteria::EPS,
1168
- 20, 0.03));
1169
-
1170
- // Build the inlier set: forward + backward both succeeded, FB
1171
- // error < 1 px, displacement in plausible range.
1172
- std::vector<cv::Point2f> goodCanvas, goodWarped;
1173
- goodCanvas.reserve(canvasPts.size());
1174
- goodWarped.reserve(canvasPts.size());
1175
- for (size_t i = 0; i < canvasPts.size(); i++) {
1176
- if (!statusFwd[i] || !statusBwd[i]) continue;
1177
- if (errFwd[i] > 30.0f || errBwd[i] > 30.0f) continue;
1178
- float fbDx = canvasPtsRev[i].x - canvasPts[i].x;
1179
- float fbDy = canvasPtsRev[i].y - canvasPts[i].y;
1180
- if (fbDx*fbDx + fbDy*fbDy > 1.0f) continue; // > 1 px FB error
1181
- float dx = warpedPts[i].x - canvasPts[i].x;
1182
- float dy = warpedPts[i].y - canvasPts[i].y;
1183
- if (std::fabs(dx) > 30.0f || std::fabs(dy) > 30.0f) continue;
1184
- goodCanvas.push_back(canvasPts[i]);
1185
- goodWarped.push_back(warpedPts[i]);
1186
- }
1187
- if (goodCanvas.size() < 6) return cv::Point2f(0, 0);
1188
-
1189
- // V11 Gap #10: 2-D RANSAC translation fit (instead of per-axis
1190
- // independent median, which can pick a (dx, dy) that no single
1191
- // point voted for — an issue in multi-modal flow scenes like a
1192
- // shelf with a moving customer).
1193
- //
1194
- // `estimateAffinePartial2D` fits a 2.5-DoF (rotation + uniform
1195
- // scale + translation) similarity transform with RANSAC. We
1196
- // use only the translation component; the rotation/scale fall-
1197
- // out shouldn't be applied (cylindrical warp already handled
1198
- // those — OF is only correcting residual translation).
1199
- std::vector<uchar> ransacInliers;
1200
- cv::Mat affine = cv::estimateAffinePartial2D(
1201
- goodCanvas, goodWarped, ransacInliers, cv::RANSAC,
1202
- /*ransacReprojThreshold=*/3.0,
1203
- /*maxIters=*/2000,
1204
- /*confidence=*/0.99,
1205
- /*refineIters=*/10);
1206
- if (affine.empty()) return cv::Point2f(0, 0);
1207
- // Inlier ratio sanity check.
1208
- int inlierCount = cv::countNonZero(ransacInliers);
1209
- if (inlierCount < 6 || inlierCount * 2 < (int)goodCanvas.size()) {
1210
- return cv::Point2f(0, 0);
1211
- }
1212
- float medDx = (float)affine.at<double>(0, 2);
1213
- float medDy = (float)affine.at<double>(1, 2);
1214
-
1215
- // The track tells us "to align canvas pixels with warped pixels,
1216
- // shift WARPED by (-medDx, -medDy)". We shift the placement
1217
- // of warped on canvas by the opposite to compensate.
1218
- return cv::Point2f(-medDx, -medDy);
1219
- }
1220
-
1221
- /// V11 Gap #11: NARROW-band feather blend. Earlier versions used
1222
- /// `alpha = distNew / (distNew + distCanvas)` over the FULL overlap,
1223
- /// which smears every pixel of disagreement across the entire
1224
- /// overlap region — the textbook ghosting source called out by
1225
- /// Brown-Lowe 2007. At the typical ARKit ~1-2° pose error + KLT-
1226
- /// refined ~5 px residual, full-overlap feather creates visible
1227
- /// double-image.
1228
- ///
1229
- /// Narrow-band approach: define the SEAM as `distNew == distCanvas`
1230
- /// (the locus of equal-distance-from-each-frame's-edge points).
1231
- /// Within `kSeamBandPx` of the seam, smoothly transition alpha 0→1.
1232
- /// Outside the band, alpha is binary (0 or 1). Each pixel comes
1233
- /// from EXACTLY ONE frame, except in the small seam band — so any
1234
- /// per-pixel misalignment can't produce ghosts.
1235
- - (void)featherBlendWarped:(cv::Mat)warped
1236
- mask:(cv::Mat)warpedMask
1237
- intoCanvas:(cv::Mat)canvasRoi
1238
- canvasMask:(cv::Mat)canvasMaskRoi
1239
- {
1240
- // V11 Gap #13: per-pair gain compensation BEFORE blending.
1241
- // Frames captured 200ms apart often differ in luminance by
1242
- // 5-15% due to auto-exposure drift; without compensation the
1243
- // panorama shows visible vertical/horizontal banding at every
1244
- // seam. Apply a per-channel mean ratio (canvas / warped)
1245
- // computed on the overlap region. Conservative bounds to
1246
- // avoid amplifying noise.
1247
- cv::Mat warpedAdj;
1248
- cv::Mat overlapMask;
1249
- cv::bitwise_and(canvasMaskRoi, warpedMask, overlapMask);
1250
- int overlapPx = cv::countNonZero(overlapMask);
1251
- if (overlapPx > 100) {
1252
- cv::Scalar canvasMean = cv::mean(canvasRoi, overlapMask);
1253
- cv::Scalar warpedMean = cv::mean(warped, overlapMask);
1254
- double gainB = (warpedMean[0] > 1.0) ? (canvasMean[0] / warpedMean[0]) : 1.0;
1255
- double gainG = (warpedMean[1] > 1.0) ? (canvasMean[1] / warpedMean[1]) : 1.0;
1256
- double gainR = (warpedMean[2] > 1.0) ? (canvasMean[2] / warpedMean[2]) : 1.0;
1257
- // Clamp gains to ±25% to avoid blowing out highlights or
1258
- // crushing shadows on a single noisy mean estimate.
1259
- gainB = std::clamp(gainB, 0.75, 1.25);
1260
- gainG = std::clamp(gainG, 0.75, 1.25);
1261
- gainR = std::clamp(gainR, 0.75, 1.25);
1262
- cv::Mat warpedF;
1263
- warped.convertTo(warpedF, CV_32FC3);
1264
- std::vector<cv::Mat> ch(3);
1265
- cv::split(warpedF, ch);
1266
- ch[0] *= gainB;
1267
- ch[1] *= gainG;
1268
- ch[2] *= gainR;
1269
- cv::merge(ch, warpedF);
1270
- warpedF.convertTo(warpedAdj, CV_8UC3);
1271
- } else {
1272
- warpedAdj = warped;
1273
- }
1274
-
1275
- cv::Mat distNew, distCanvas;
1276
- cv::distanceTransform(warpedMask, distNew, cv::DIST_L2, 3);
1277
- cv::distanceTransform(canvasMaskRoi, distCanvas, cv::DIST_L2, 3);
1278
-
1279
- // Signed distance from seam: positive = "new wins side", negative
1280
- // = "canvas wins side". Pixels >= +bandHalf use new (alpha=1),
1281
- // pixels <= -bandHalf use canvas (alpha=0), in-between band gets
1282
- // a smooth ramp.
1283
- constexpr float kSeamBandPx = 5.0f;
1284
- cv::Mat signedDist = distNew - distCanvas;
1285
- cv::Mat alpha;
1286
- // alpha = clamp((signedDist + bandHalf) / band, 0, 1)
1287
- signedDist.convertTo(alpha, CV_32F,
1288
- 1.0 / (2.0 * kSeamBandPx),
1289
- 0.5);
1290
- cv::min(alpha, 1.0f, alpha);
1291
- cv::max(alpha, 0.0f, alpha);
1292
-
1293
- // First-touch regions: alpha=1 unconditionally (canvas was empty
1294
- // here, new frame is the only source).
1295
- cv::Mat noPrior;
1296
- cv::compare(canvasMaskRoi, 0, noPrior, cv::CMP_EQ);
1297
- alpha.setTo(1.0f, noPrior);
1298
-
1299
- // Outside-of-new regions: keep canvas (alpha=0).
1300
- cv::Mat noNew;
1301
- cv::compare(warpedMask, 0, noNew, cv::CMP_EQ);
1302
- alpha.setTo(0.0f, noNew);
1303
-
1304
- // Per-channel blend.
1305
- cv::Mat alpha3, invAlpha3;
1306
- cv::Mat ch[] = {alpha, alpha, alpha};
1307
- cv::merge(ch, 3, alpha3);
1308
- invAlpha3 = cv::Scalar(1, 1, 1) - alpha3;
1309
-
1310
- cv::Mat warpedF, canvasF;
1311
- warpedAdj.convertTo(warpedF, CV_32FC3); // V11 Gap #13: use gain-corrected
1312
- canvasRoi.convertTo(canvasF, CV_32FC3);
1313
- cv::Mat blendedF = warpedF.mul(alpha3) + canvasF.mul(invAlpha3);
1314
- cv::Mat blended8;
1315
- blendedF.convertTo(blended8, CV_8UC3);
1316
-
1317
- // Write back: only where the union mask has content.
1318
- cv::Mat unionMask;
1319
- cv::bitwise_or(warpedMask, canvasMaskRoi, unionMask);
1320
- blended8.copyTo(canvasRoi, unionMask);
1321
- cv::bitwise_or(canvasMaskRoi, warpedMask, canvasMaskRoi);
1322
- }
1323
-
1324
- // V11 Gap #21: deleted ~85 lines of dead `warpAndBlend` (legacy v7
1325
- // planar warp + Gaussian-blurred binary alpha-blend). Was never
1326
- // called after v9 switched to cylindricalWarp + featherBlendWarped.
1327
-
1328
- @end