react-native-image-stitcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/LICENSE +201 -0
- package/NOTICE +21 -0
- package/README.md +189 -0
- package/RNImageStitcher.podspec +76 -0
- package/android/build.gradle +224 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/cpp/CMakeLists.txt +124 -0
- package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
- package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
- package/cpp/ar_frame_pose.h +63 -0
- package/cpp/keyframe_gate.cpp +927 -0
- package/cpp/keyframe_gate.hpp +240 -0
- package/cpp/stitcher.cpp +2207 -0
- package/cpp/stitcher.hpp +275 -0
- package/dist/ar/useARSession.d.ts +102 -0
- package/dist/ar/useARSession.js +133 -0
- package/dist/camera/ARCameraView.d.ts +93 -0
- package/dist/camera/ARCameraView.js +170 -0
- package/dist/camera/Camera.d.ts +134 -0
- package/dist/camera/Camera.js +688 -0
- package/dist/camera/CameraShutter.d.ts +80 -0
- package/dist/camera/CameraShutter.js +237 -0
- package/dist/camera/CameraView.d.ts +65 -0
- package/dist/camera/CameraView.js +117 -0
- package/dist/camera/CaptureControlsBar.d.ts +87 -0
- package/dist/camera/CaptureControlsBar.js +82 -0
- package/dist/camera/CaptureHeader.d.ts +62 -0
- package/dist/camera/CaptureHeader.js +81 -0
- package/dist/camera/CapturePreview.d.ts +70 -0
- package/dist/camera/CapturePreview.js +188 -0
- package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
- package/dist/camera/CaptureStatusOverlay.js +326 -0
- package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
- package/dist/camera/CaptureThumbnailStrip.js +177 -0
- package/dist/camera/IncrementalPanGuide.d.ts +83 -0
- package/dist/camera/IncrementalPanGuide.js +267 -0
- package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
- package/dist/camera/PanoramaBandOverlay.js +399 -0
- package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
- package/dist/camera/PanoramaConfirmModal.js +128 -0
- package/dist/camera/PanoramaGuidance.d.ts +79 -0
- package/dist/camera/PanoramaGuidance.js +246 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
- package/dist/camera/PanoramaSettingsModal.js +611 -0
- package/dist/camera/ViewportCropOverlay.d.ts +46 -0
- package/dist/camera/ViewportCropOverlay.js +67 -0
- package/dist/camera/useCapture.d.ts +111 -0
- package/dist/camera/useCapture.js +160 -0
- package/dist/camera/useDeviceOrientation.d.ts +48 -0
- package/dist/camera/useDeviceOrientation.js +131 -0
- package/dist/camera/useVideoCapture.d.ts +79 -0
- package/dist/camera/useVideoCapture.js +151 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.js +39 -0
- package/dist/quality/normaliseOrientation.d.ts +36 -0
- package/dist/quality/normaliseOrientation.js +62 -0
- package/dist/quality/runQualityCheck.d.ts +41 -0
- package/dist/quality/runQualityCheck.js +98 -0
- package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
- package/dist/sensors/useIMUTranslationGate.js +235 -0
- package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
- package/dist/stitching/IncrementalStitcherView.js +157 -0
- package/dist/stitching/incremental.d.ts +930 -0
- package/dist/stitching/incremental.js +133 -0
- package/dist/stitching/stitchFrames.d.ts +55 -0
- package/dist/stitching/stitchFrames.js +56 -0
- package/dist/stitching/stitchVideo.d.ts +119 -0
- package/dist/stitching/stitchVideo.js +57 -0
- package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
- package/dist/stitching/useIncrementalJSDriver.js +199 -0
- package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
- package/dist/stitching/useIncrementalStitcher.js +172 -0
- package/dist/types.d.ts +58 -0
- package/dist/types.js +15 -0
- package/ios/Package.swift +72 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
- package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
- package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
- package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
- package/package.json +73 -0
- package/react-native.config.js +34 -0
- package/scripts/opencv-version.txt +1 -0
- package/scripts/postinstall-fetch-binaries.js +286 -0
- package/src/ar/useARSession.ts +210 -0
- package/src/camera/.gitkeep +0 -0
- package/src/camera/ARCameraView.tsx +256 -0
- package/src/camera/Camera.tsx +1053 -0
- package/src/camera/CameraShutter.tsx +292 -0
- package/src/camera/CameraView.tsx +157 -0
- package/src/camera/CaptureControlsBar.tsx +204 -0
- package/src/camera/CaptureHeader.tsx +184 -0
- package/src/camera/CapturePreview.tsx +318 -0
- package/src/camera/CaptureStatusOverlay.tsx +391 -0
- package/src/camera/CaptureThumbnailStrip.tsx +277 -0
- package/src/camera/IncrementalPanGuide.tsx +328 -0
- package/src/camera/PanoramaBandOverlay.tsx +498 -0
- package/src/camera/PanoramaConfirmModal.tsx +206 -0
- package/src/camera/PanoramaGuidance.tsx +327 -0
- package/src/camera/PanoramaSettingsModal.tsx +1357 -0
- package/src/camera/ViewportCropOverlay.tsx +81 -0
- package/src/camera/useCapture.ts +279 -0
- package/src/camera/useDeviceOrientation.ts +140 -0
- package/src/camera/useVideoCapture.ts +236 -0
- package/src/index.ts +53 -0
- package/src/quality/.gitkeep +0 -0
- package/src/quality/normaliseOrientation.ts +79 -0
- package/src/quality/runQualityCheck.ts +131 -0
- package/src/sensors/useIMUTranslationGate.ts +347 -0
- package/src/stitching/.gitkeep +0 -0
- package/src/stitching/IncrementalStitcherView.tsx +198 -0
- package/src/stitching/incremental.ts +1021 -0
- package/src/stitching/stitchFrames.ts +88 -0
- package/src/stitching/stitchVideo.ts +153 -0
- package/src/stitching/useIncrementalJSDriver.ts +273 -0
- package/src/stitching/useIncrementalStitcher.ts +252 -0
- package/src/types.ts +78 -0
|
@@ -0,0 +1,3285 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// OpenCVFirstWinsCylindricalStitcher.mm
|
|
4
|
+
//
|
|
5
|
+
// Apple-style slit-scan engine (Option B from the panorama north-star
|
|
6
|
+
// doc). Reuses the panorama-frame coord setup from v9 but paints
|
|
7
|
+
// narrow vertical strips instead of warping whole frames.
|
|
8
|
+
//
|
|
9
|
+
|
|
10
|
+
#ifdef NO
|
|
11
|
+
#undef NO
|
|
12
|
+
#endif
|
|
13
|
+
#ifdef YES
|
|
14
|
+
#undef YES
|
|
15
|
+
#endif
|
|
16
|
+
|
|
17
|
+
#import <opencv2/opencv.hpp>
|
|
18
|
+
#import <opencv2/core.hpp>
|
|
19
|
+
#import <opencv2/imgproc.hpp>
|
|
20
|
+
#import <opencv2/imgcodecs.hpp>
|
|
21
|
+
#import <opencv2/features2d.hpp> // ORB, BFMatcher (V12.11 Step 4)
|
|
22
|
+
#import <opencv2/calib3d.hpp> // findHomography (V12.11 Step 4)
|
|
23
|
+
|
|
24
|
+
#import <vector>
|
|
25
|
+
#import <chrono>
|
|
26
|
+
#import <os/log.h>
|
|
27
|
+
|
|
28
|
+
// V13.0a.4 — diagnostic os_log subsystem. os_log_with_type(FAULT)
|
|
29
|
+
// survives Console.app's NSLog burst rate-limit (~10/sec) — same
|
|
30
|
+
// pattern we used in V12.14.x to get crash-trail breadcrumbs through.
|
|
31
|
+
static os_log_t SlitDiagLog(void) {
|
|
32
|
+
static os_log_t log = NULL;
|
|
33
|
+
static dispatch_once_t once;
|
|
34
|
+
dispatch_once(&once, ^{
|
|
35
|
+
log = os_log_create("com.tiger.retailens.sdk", "slitscan");
|
|
36
|
+
});
|
|
37
|
+
return log;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
#define NO ((BOOL)0)
|
|
41
|
+
#define YES ((BOOL)1)
|
|
42
|
+
|
|
43
|
+
#import "OpenCVSlitScanStitcher.h" // file name kept; class renamed to OpenCVFirstWinsCylindricalStitcher (V11 Gap #23)
|
|
44
|
+
|
|
45
|
+
// V15.0e — class extension declaring private helper methods. The
|
|
46
|
+
// public interface stays in OpenCVSlitScanStitcher.h; these are
|
|
47
|
+
// implementation-only. Without a declaration the compiler still
|
|
48
|
+
// finds them at link time but warns about each call site. Listing
|
|
49
|
+
// them here keeps the build warning-free.
|
|
50
|
+
@interface OpenCVFirstWinsCylindricalStitcher ()
|
|
51
|
+
- (BOOL)tryPaintPlaneProjected:(const cv::Mat &)frameBGR
|
|
52
|
+
R_new:(const cv::Mat &)R_new
|
|
53
|
+
tx:(double)tx
|
|
54
|
+
ty:(double)ty
|
|
55
|
+
tz:(double)tz
|
|
56
|
+
tele:(RLISFrameTelemetry *)tele
|
|
57
|
+
t0:(std::chrono::steady_clock::time_point)t0;
|
|
58
|
+
@end
|
|
59
|
+
|
|
60
|
+
@implementation OpenCVFirstWinsCylindricalStitcher {
|
|
61
|
+
NSInteger _composeWidth;
|
|
62
|
+
NSInteger _composeHeight;
|
|
63
|
+
NSInteger _canvasWidth;
|
|
64
|
+
NSInteger _canvasHeight;
|
|
65
|
+
/// V12.12 — pan-axis canvas extent (defaults to max of
|
|
66
|
+
/// constructor's canvasWidth/Height, e.g. 5000). Used at first-
|
|
67
|
+
/// frame allocation to size the pan-axis dimension of the canvas.
|
|
68
|
+
/// The perpendicular axis is taken from the actual frame size at
|
|
69
|
+
/// first-frame ingest, so the canvas is "just wide enough" along
|
|
70
|
+
/// perpendicular and "5000-deep" along the pan axis. This pairs
|
|
71
|
+
/// with the new engine-internal canvas allocation (deferred to
|
|
72
|
+
/// first frame so we can use the pose-detected orientation).
|
|
73
|
+
NSInteger _canvasPanExtent;
|
|
74
|
+
NSInteger _frameRotationDegrees;
|
|
75
|
+
/// V12.3 orientation-aware cylinder axis — see v9 engine.
|
|
76
|
+
BOOL _isLandscape;
|
|
77
|
+
/// V12.7 Variant B: rectilinear mode. When YES, skip cylindrical
|
|
78
|
+
/// warp entirely. First frame is pasted raw onto canvas.
|
|
79
|
+
/// Subsequent frames contribute a narrow central strip placed by
|
|
80
|
+
/// ARKit pose-delta, with first-painted-wins masking. The
|
|
81
|
+
/// canvas content stays in the camera's native rectilinear
|
|
82
|
+
/// projection — zero cylindrical curvature.
|
|
83
|
+
BOOL _useRectilinear;
|
|
84
|
+
/// V12.7 first-frame anchor for rectilinear placement.
|
|
85
|
+
int _firstFrameDstX;
|
|
86
|
+
int _firstFrameDstY;
|
|
87
|
+
/// V12.11 Step D — running max position along the pan axis.
|
|
88
|
+
/// Tracks the FURTHEST extent reached during the current
|
|
89
|
+
/// capture. If a subsequent frame's homography-corrected
|
|
90
|
+
/// dst position drops below `max - kReverseStopPx` the
|
|
91
|
+
/// engine treats it as a reverse-direction event and emits
|
|
92
|
+
/// RLISFrameOutcomeRejectedReverseDirection without pasting.
|
|
93
|
+
/// Reset by `[reset]` to firstFrameDstX/Y at first frame.
|
|
94
|
+
int _maxDstX;
|
|
95
|
+
int _maxDstY;
|
|
96
|
+
|
|
97
|
+
cv::Mat _canvas;
|
|
98
|
+
cv::Mat _canvasMask;
|
|
99
|
+
|
|
100
|
+
// Panorama-frame state — same shape as v9 hybrid engine.
|
|
101
|
+
cv::Mat _firstRotationArkit;
|
|
102
|
+
cv::Mat _M_arkitToCv;
|
|
103
|
+
cv::Mat _R_panToWorld;
|
|
104
|
+
cv::Mat _K_compose;
|
|
105
|
+
double _focalCompose;
|
|
106
|
+
int _canvasOriginCylX;
|
|
107
|
+
int _canvasOriginCylY;
|
|
108
|
+
|
|
109
|
+
bool _hasFirstFrame;
|
|
110
|
+
NSInteger _accepted;
|
|
111
|
+
NSInteger _snapshotSeq;
|
|
112
|
+
|
|
113
|
+
// Last-strip state — strip placement is incremental from here.
|
|
114
|
+
// V11 Gap #22: deleted dead state (_lastAcceptedTheta,
|
|
115
|
+
// _lastAcceptedH, _slitScanMode) — these were strip-extraction
|
|
116
|
+
// state from the original slit-scan design, never used by the
|
|
117
|
+
// first-painted-wins implementation.
|
|
118
|
+
|
|
119
|
+
// ── V13.0e: ORB-triangulation translation correction state ────────
|
|
120
|
+
// Per-frame ORB feature detection on each accepted slit; matched
|
|
121
|
+
// against the previous accept to triangulate depth and compensate
|
|
122
|
+
// canvas-paste position for camera translation parallax.
|
|
123
|
+
//
|
|
124
|
+
// Why this exists: pose-only paste assumes pure rotation. Field
|
|
125
|
+
// captures show ~30–40 cm of camera translation per pan, which at
|
|
126
|
+
// typical 1–3 m scene depth produces ~30–100 px of perpendicular
|
|
127
|
+
// jitter and missing scene content between adjacent slits (the
|
|
128
|
+
// "translation gaps" Ram observed in V13.0d). Triangulating
|
|
129
|
+
// matched features between adjacent accepts gives a per-frame
|
|
130
|
+
// depth estimate; the parallax correction Δpixel = focal × Δt_cam / Z
|
|
131
|
+
// closes that gap.
|
|
132
|
+
//
|
|
133
|
+
// V12.11 Step 4's per-frame ORB+RANSAC homography approach was
|
|
134
|
+
// brittle under low overlap / low texture / fast pan. V13.0e is
|
|
135
|
+
// different in TWO ways:
|
|
136
|
+
// 1. Triangulation only — we use ORB to compute a depth scalar
|
|
137
|
+
// (median Z), not to drive a homography. Geometric pose still
|
|
138
|
+
// drives placement; ORB just supplies the parallax denominator.
|
|
139
|
+
// 2. Forward-only Y clamp on the correction — never pulls back
|
|
140
|
+
// below the running max along the pan axis. Same lesson as
|
|
141
|
+
// V12.14 (frame-stacking under pull-back).
|
|
142
|
+
cv::Ptr<cv::ORB> _orbDetector;
|
|
143
|
+
std::vector<cv::KeyPoint> _prevKeypoints;
|
|
144
|
+
cv::Mat _prevDescriptors;
|
|
145
|
+
cv::Mat _prevRotationArkit; // 3x3 R at previous accept
|
|
146
|
+
cv::Mat _prevTranslationArkit; // 3x1 t at previous accept (m)
|
|
147
|
+
cv::Mat _firstTranslationArkit; // 3x1 t at first frame (m) — V13.0g unused, kept for diag.
|
|
148
|
+
bool _hasPrevAccept;
|
|
149
|
+
|
|
150
|
+
// V13.0g — per-accept incremental tri-correction accumulator.
|
|
151
|
+
// V13.0e/f computed correction = focal × R^T × (t_now − t_first) / Z
|
|
152
|
+
// (cumulative-from-first), capped at ±50/±100. At realistic pan
|
|
153
|
+
// motion (Δt cumulative ~40 cm) the true correction can reach 200–
|
|
154
|
+
// 400 px; the cap clipped most of it, leaving tens to hundreds of
|
|
155
|
+
// pixels of misalignment between adjacent slits. V13.0g switches
|
|
156
|
+
// to per-accept INCREMENTAL Δt (t_now − t_prev_accept ≈ 4 cm),
|
|
157
|
+
// computes a small per-accept correction (~30–100 px), caps that
|
|
158
|
+
// increment at ±80, and ACCUMULATES it. Total correction over a
|
|
159
|
+
// pan grows naturally to whatever cumulative parallax demands;
|
|
160
|
+
// per-accept artifact stays bounded; bad-Z bursts on individual
|
|
161
|
+
// accepts contribute ±80 (not the global rescaling that V13.0f's
|
|
162
|
+
// cumulative-Z formula caused).
|
|
163
|
+
double _accumTriCorrectionX;
|
|
164
|
+
double _accumTriCorrectionY;
|
|
165
|
+
|
|
166
|
+
// V14.0a — last accept's final canvas paste position. Used to
|
|
167
|
+
// build per-match canvas-coord pairs that feed RANSAC homography
|
|
168
|
+
// in subsequent accepts. Captured AFTER V13.0g's incremental
|
|
169
|
+
// tri+accumulator and the forward-only Y clamp, because that's
|
|
170
|
+
// what was actually painted onto canvas. Initialized at first
|
|
171
|
+
// accept; updated at the end of each subsequent accept.
|
|
172
|
+
int _prevAcceptDstX;
|
|
173
|
+
int _prevAcceptDstY;
|
|
174
|
+
|
|
175
|
+
// V15 — runtime config controlling which correction stages run.
|
|
176
|
+
// See RLISStitcherConfig in OpenCVIncrementalStitcher.h. Set via
|
|
177
|
+
// -setConfig: after init; defaults to slitscan-both factory config
|
|
178
|
+
// if never set.
|
|
179
|
+
RLISStitcherConfig *_config;
|
|
180
|
+
|
|
181
|
+
// V15.0b — latched plane transform for plane-projected stitch
|
|
182
|
+
// mode. 4×4 column-major in CV_64F, world coords. Empty until
|
|
183
|
+
// -setPlaneTransformFlat: is called by the bridge with the
|
|
184
|
+
// ARSession's first detected vertical plane.
|
|
185
|
+
cv::Mat _planeTransform;
|
|
186
|
+
|
|
187
|
+
// V15.0c.2 — off-plane detector. Tracks the max corner ray
|
|
188
|
+
// distance (t_int) for the FIRST successful plane-projected
|
|
189
|
+
// frame. Subsequent frames whose max corner t_int exceeds
|
|
190
|
+
// kOffPlaneMultiplier × _firstPlaneTInt are flagged as off-plane
|
|
191
|
+
// (camera tilted past the wall edge → rays grazing the plane
|
|
192
|
+
// hit the floor / open air rather than the wall) and the engine
|
|
193
|
+
// falls back to the slit-scan path for that frame. Reset to 0
|
|
194
|
+
// on -reset; set on the first frame with a valid raycast.
|
|
195
|
+
double _firstPlaneTInt;
|
|
196
|
+
|
|
197
|
+
// V15.0d — per-capture frame counter for diagnostic-log gating.
|
|
198
|
+
// The function-scope `static _engineCallCounter` persists for the
|
|
199
|
+
// lifetime of the engine instance, so its `<= 5` window only
|
|
200
|
+
// catches the very first capture after app launch. This per-
|
|
201
|
+
// capture counter resets on every -reset, so the
|
|
202
|
+
// `[V15.0c.4-pre]` and other "first 5 frames" diagnostic logs
|
|
203
|
+
// fire on every capture. Incremented at the top of
|
|
204
|
+
// ingestPixelBuffer alongside _engineCallCounter.
|
|
205
|
+
NSInteger _captureFrameCounter;
|
|
206
|
+
|
|
207
|
+
// V15.0d — EMA state for 1B 2D-NCC smoothing. Holds the LAST
|
|
208
|
+
// applied (after-EMA) Δx/Δy correction so the next frame can
|
|
209
|
+
// blend with it. Reset on -reset. Used only when
|
|
210
|
+
// _config.enableNcc2dEmaSmoothing is YES.
|
|
211
|
+
int _lastNcc2dDxApplied;
|
|
212
|
+
int _lastNcc2dDyApplied;
|
|
213
|
+
BOOL _haveNcc2dEmaHistory;
|
|
214
|
+
|
|
215
|
+
// V15.0g.1 — adaptive pixels-per-meter for the plane-projection
|
|
216
|
+
// canvas mapping. Set on the FIRST successful plane-projected
|
|
217
|
+
// frame to `focal / t_int_center` so the rendered rectangle
|
|
218
|
+
// matches sensor dimensions 1:1 (scale = 1.0). Without this,
|
|
219
|
+
// a hardcoded ppm=1000 produced narrow output at typical retail
|
|
220
|
+
// scan distances (~1m → rectangle 63% of sensor width — Ram
|
|
221
|
+
// reported 2026-05-08 that the panorama looked much narrower
|
|
222
|
+
// than the live camera preview). Persists for the rest of the
|
|
223
|
+
// capture; reset on -reset. Zero = not yet set (use 1000.0
|
|
224
|
+
// fallback so off-plane detection works on the very first frame
|
|
225
|
+
// before this is initialised).
|
|
226
|
+
double _dynamicKPixelsPerMeter;
|
|
227
|
+
|
|
228
|
+
// V15.0g.4 — first-frame plane-local coordinates of the camera
|
|
229
|
+
// CENTER ray's intersection with the plane. Used to OFFSET the
|
|
230
|
+
// canvas mapping so the FIRST plane-projected frame lands at
|
|
231
|
+
// canvas center (cCenterX, cCenterY) regardless of where ARKit
|
|
232
|
+
// chose to place the plane anchor's local origin in 3D space.
|
|
233
|
+
// Without this offset, the rectangle's canvas position depended
|
|
234
|
+
// on ARKit's arbitrary anchor placement and could land far from
|
|
235
|
+
// canvas center → frames clipped at canvas bounds → narrow
|
|
236
|
+
// output (Ram observed 2026-05-08 with cooler scan). Subsequent
|
|
237
|
+
// frames' canvas position = cCenter + (current_UV − first_UV)
|
|
238
|
+
// × ppm, giving a panorama anchored on the user's first-frame
|
|
239
|
+
// aim. `_haveFirstPlaneAnchor` flag tracks whether they're set.
|
|
240
|
+
double _firstPlaneAnchorUp;
|
|
241
|
+
double _firstPlaneAnchorVp;
|
|
242
|
+
BOOL _haveFirstPlaneAnchor;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
- (instancetype)initWithComposeWidth:(NSInteger)composeWidth
|
|
246
|
+
composeHeight:(NSInteger)composeHeight
|
|
247
|
+
canvasWidth:(NSInteger)canvasWidth
|
|
248
|
+
canvasHeight:(NSInteger)canvasHeight
|
|
249
|
+
featherPx:(NSInteger)featherPx
|
|
250
|
+
frameRotationDegrees:(NSInteger)frameRotationDegrees
|
|
251
|
+
useRectilinear:(BOOL)useRectilinear
|
|
252
|
+
{
|
|
253
|
+
if (self = [super init]) {
|
|
254
|
+
_composeWidth = composeWidth > 0 ? composeWidth : 960;
|
|
255
|
+
_composeHeight = composeHeight > 0 ? composeHeight : 720;
|
|
256
|
+
// V12.12 — canvasWidth/Height args become HINTS, not exact
|
|
257
|
+
// dims: the engine allocates the actual canvas at first frame
|
|
258
|
+
// based on detected orientation. The pan-axis dimension is
|
|
259
|
+
// max(canvasWidth, canvasHeight) — both are typically 5000 so
|
|
260
|
+
// the value is unambiguous; the perpendicular dim is
|
|
261
|
+
// determined by the actual frame size at first-frame ingest.
|
|
262
|
+
_canvasWidth = canvasWidth > 0 ? canvasWidth : 5000;
|
|
263
|
+
_canvasHeight = canvasHeight > 0 ? canvasHeight : 5000;
|
|
264
|
+
_canvasPanExtent = std::max(_canvasWidth, _canvasHeight);
|
|
265
|
+
_frameRotationDegrees = frameRotationDegrees;
|
|
266
|
+
_useRectilinear = useRectilinear;
|
|
267
|
+
_firstFrameDstX = 0;
|
|
268
|
+
_firstFrameDstY = 0;
|
|
269
|
+
_maxDstX = 0;
|
|
270
|
+
_maxDstY = 0;
|
|
271
|
+
// V12.6 Step C: detected at first-frame init from R_panToCam,
|
|
272
|
+
// not from frameRotationDegrees. Default false here is just
|
|
273
|
+
// a safe initialiser.
|
|
274
|
+
_isLandscape = NO;
|
|
275
|
+
|
|
276
|
+
_M_arkitToCv = (cv::Mat_<double>(3, 3) <<
|
|
277
|
+
1, 0, 0,
|
|
278
|
+
0, -1, 0,
|
|
279
|
+
0, 0, -1);
|
|
280
|
+
|
|
281
|
+
// V13.0e — ORB detector lives for the engine's lifetime;
|
|
282
|
+
// detectAndCompute is called per accepted slit. 500 features
|
|
283
|
+
// is plenty for matching across 50–60 px sliver advances; ORB
|
|
284
|
+
// on 1920×1080 with this budget runs in ~5 ms on iPhone 14+.
|
|
285
|
+
_orbDetector = cv::ORB::create(500);
|
|
286
|
+
|
|
287
|
+
// V15 — default config (slitscan-both). Caller should override
|
|
288
|
+
// via -setConfig: after init. Default chosen so engines work
|
|
289
|
+
// correctly with the legacy `useRectilinear=YES` path even if
|
|
290
|
+
// setConfig is never called.
|
|
291
|
+
_config = [RLISStitcherConfig configForMode:@"slitscan-both"];
|
|
292
|
+
|
|
293
|
+
[self reset];
|
|
294
|
+
}
|
|
295
|
+
return self;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
- (void)setConfig:(RLISStitcherConfig *)config {
|
|
299
|
+
if (config == nil) return;
|
|
300
|
+
// V15.0d backward-compat: legacy callers set the boolean
|
|
301
|
+
// useDetectedPlane=YES. New planeSource enum subsumes that flag.
|
|
302
|
+
// If planeSource is at its default (Disabled) but the legacy
|
|
303
|
+
// boolean is YES, upgrade to ARKitDetected — preserving V15.0b
|
|
304
|
+
// semantics for callers that haven't migrated to planeSource yet.
|
|
305
|
+
if (config.planeSource == RLISPlaneSourceDisabled && config.useDetectedPlane) {
|
|
306
|
+
config.planeSource = RLISPlaneSourceARKitDetected;
|
|
307
|
+
}
|
|
308
|
+
_config = config;
|
|
309
|
+
// V15.0c.4 — converted to fault log so it isn't rate-limited.
|
|
310
|
+
// V15.0d — added planeSource + new NCC knobs to the snapshot.
|
|
311
|
+
const char *planeSrc =
|
|
312
|
+
_config.planeSource == RLISPlaneSourceVirtual ? "Virtual"
|
|
313
|
+
: (_config.planeSource == RLISPlaneSourceARKitDetected ? "ARKitDetected"
|
|
314
|
+
: "Disabled");
|
|
315
|
+
const char *planeStyle =
|
|
316
|
+
_config.planeProjectionStyle == RLISPlaneProjectionStyleRectified
|
|
317
|
+
? "Rectified" : "Trapezoidal";
|
|
318
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
319
|
+
"[V15-config] slit-scan config applied: panAxisFrac=%.2f "
|
|
320
|
+
"acceptGate=%ld tri=%d triAccum=%d 1dNcc=%d 2dNcc=%d "
|
|
321
|
+
"ransac=%d paint=%s planeSource=%s planeStyle=%s "
|
|
322
|
+
"(virtualDepth=%.2fm arkitAlignThr=%.2f) "
|
|
323
|
+
"ncc2d(margin=%ld confThr=%.2f ema=%d emaA=%.2f "
|
|
324
|
+
"panLock=%d crossLock=%ld)",
|
|
325
|
+
_config.kPanAxisFractionRect, (long)_config.kMinAcceptDeltaPx,
|
|
326
|
+
(int)_config.enableTriangulation, (int)_config.enableTriAccumulator,
|
|
327
|
+
(int)_config.enable1dNcc, (int)_config.enable2dNcc,
|
|
328
|
+
(int)_config.enableRansacHomography,
|
|
329
|
+
_config.paintMode == RLISPaintModeFeatherBlend
|
|
330
|
+
? "FeatherBlend" : "FirstPaintedWins",
|
|
331
|
+
planeSrc, planeStyle,
|
|
332
|
+
_config.virtualPlaneDepthMeters,
|
|
333
|
+
_config.arkitPlaneAlignmentThreshold,
|
|
334
|
+
(long)_config.nccSearchMargin2d,
|
|
335
|
+
_config.nccConfidenceThreshold2d,
|
|
336
|
+
(int)_config.enableNcc2dEmaSmoothing,
|
|
337
|
+
_config.ncc2dEmaAlpha,
|
|
338
|
+
(int)_config.enableNcc2dPanAxisLock,
|
|
339
|
+
(long)_config.ncc2dCrossAxisLockPx);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
- (void)setPlaneTransformFlat:(NSArray<NSNumber *> *)transform16 {
|
|
343
|
+
// V15.0d — Virtual plane mode owns _planeTransform itself: it
|
|
344
|
+
// synthesizes a plane on the first plane-projected frame from
|
|
345
|
+
// the camera pose. Ignore bridge propagations in Virtual mode
|
|
346
|
+
// so the bridge's ARKit-detected plane (if any) doesn't clobber
|
|
347
|
+
// the synthesized plane.
|
|
348
|
+
if (_config.planeSource == RLISPlaneSourceVirtual) {
|
|
349
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
350
|
+
"[V15.0d-plane] setPlaneTransformFlat ignored "
|
|
351
|
+
"(planeSource=Virtual; engine synthesizes its own plane)");
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
if (transform16.count != 16) {
|
|
355
|
+
_planeTransform = cv::Mat();
|
|
356
|
+
// V15.0c.2 — clear the off-plane baseline when the plane
|
|
357
|
+
// transform is cleared. Next plane latch will start fresh.
|
|
358
|
+
_firstPlaneTInt = 0.0;
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
// Column-major from Swift's simd_float4x4. Build a 4×4 CV_64F
|
|
362
|
+
// matrix in row-major (OpenCV convention).
|
|
363
|
+
cv::Mat T(4, 4, CV_64F);
|
|
364
|
+
for (int col = 0; col < 4; col++) {
|
|
365
|
+
for (int row = 0; row < 4; row++) {
|
|
366
|
+
const int idx = col * 4 + row;
|
|
367
|
+
T.at<double>(row, col) = transform16[idx].doubleValue;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
_planeTransform = T;
|
|
371
|
+
// V15.0c.2 — clear the off-plane baseline; first successful
|
|
372
|
+
// plane-projected frame after this will set the baseline.
|
|
373
|
+
_firstPlaneTInt = 0.0;
|
|
374
|
+
// V15.0c.4 — fault log (was NSLog/default). See note on setConfig.
|
|
375
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
376
|
+
"[V15.0b-plane] engine received plane transform: "
|
|
377
|
+
"origin=(%.3f, %.3f, %.3f) scale=(%.3f, %.3f, %.3f)",
|
|
378
|
+
T.at<double>(0,3), T.at<double>(1,3), T.at<double>(2,3),
|
|
379
|
+
cv::norm(T.col(0).rowRange(0,3)),
|
|
380
|
+
cv::norm(T.col(1).rowRange(0,3)),
|
|
381
|
+
cv::norm(T.col(2).rowRange(0,3)));
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// V15.0e — plane-projection helper. Called from BOTH the first-frame
|
|
385
|
+
// branch AND the steady-state branch. Paints frameBGR onto _canvas
|
|
386
|
+
// via 4-corner raycast onto the latched plane (or a synthesized
|
|
387
|
+
// virtual plane in Virtual mode). Returns YES if painted (caller
|
|
388
|
+
// should return tele), NO if plane is unavailable / degenerate
|
|
389
|
+
// (caller falls back to rectilinear paste / slit-scan path).
|
|
390
|
+
//
|
|
391
|
+
// Why this is now a helper: V15.0b previously lived inline in the
|
|
392
|
+
// steady-state branch only. Field testing showed the first frame
|
|
393
|
+
// painted at canvas (0, 0) in pixel coords while subsequent slivers
|
|
394
|
+
// painted at plane-derived canvas positions — TWO disjoint coord
|
|
395
|
+
// systems → two disjoint patches on the canvas (Ram screenshot
|
|
396
|
+
// 2026-05-08). The fix is to paint the FIRST frame via plane
|
|
397
|
+
// projection too, so first-frame and slivers share a coord system.
|
|
398
|
+
// Sharing the helper avoids duplicating ~150 lines of warp logic.
|
|
399
|
+
- (BOOL)tryPaintPlaneProjected:(const cv::Mat &)frameBGR
|
|
400
|
+
R_new:(const cv::Mat &)R_new
|
|
401
|
+
tx:(double)tx
|
|
402
|
+
ty:(double)ty
|
|
403
|
+
tz:(double)tz
|
|
404
|
+
tele:(RLISFrameTelemetry *)tele
|
|
405
|
+
t0:(std::chrono::steady_clock::time_point)t0 {
|
|
406
|
+
if (_config.planeSource == RLISPlaneSourceDisabled) return NO;
|
|
407
|
+
|
|
408
|
+
// V15.0d Virtual mode — synthesize plane from current pose if
|
|
409
|
+
// one isn't already set.
|
|
410
|
+
if (_config.planeSource == RLISPlaneSourceVirtual && _planeTransform.empty()) {
|
|
411
|
+
cv::Mat camForwardWorld = R_new *
|
|
412
|
+
(cv::Mat_<double>(3, 1) << 0.0, 0.0, -1.0);
|
|
413
|
+
cv::Mat normalWorld = -camForwardWorld;
|
|
414
|
+
cv::Mat originWorld =
|
|
415
|
+
(cv::Mat_<double>(3, 1) << tx, ty, tz) +
|
|
416
|
+
_config.virtualPlaneDepthMeters * camForwardWorld;
|
|
417
|
+
cv::Mat T = cv::Mat::eye(4, 4, CV_64F);
|
|
418
|
+
T.at<double>(0, 1) = normalWorld.at<double>(0);
|
|
419
|
+
T.at<double>(1, 1) = normalWorld.at<double>(1);
|
|
420
|
+
T.at<double>(2, 1) = normalWorld.at<double>(2);
|
|
421
|
+
T.at<double>(0, 3) = originWorld.at<double>(0);
|
|
422
|
+
T.at<double>(1, 3) = originWorld.at<double>(1);
|
|
423
|
+
T.at<double>(2, 3) = originWorld.at<double>(2);
|
|
424
|
+
_planeTransform = T;
|
|
425
|
+
_firstPlaneTInt = 0.0;
|
|
426
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
427
|
+
"[V15.0d-virtual-plane] synthesized at depth=%.2fm "
|
|
428
|
+
"origin=(%.3f, %.3f, %.3f) normal=(%.3f, %.3f, %.3f)",
|
|
429
|
+
_config.virtualPlaneDepthMeters,
|
|
430
|
+
originWorld.at<double>(0),
|
|
431
|
+
originWorld.at<double>(1),
|
|
432
|
+
originWorld.at<double>(2),
|
|
433
|
+
normalWorld.at<double>(0),
|
|
434
|
+
normalWorld.at<double>(1),
|
|
435
|
+
normalWorld.at<double>(2));
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// ARKitDetected mode — plane may not be ready yet; bail.
|
|
439
|
+
if (_planeTransform.empty()) return NO;
|
|
440
|
+
|
|
441
|
+
cv::Mat t_arkit = (cv::Mat_<double>(3, 1) << tx, ty, tz);
|
|
442
|
+
|
|
443
|
+
cv::Mat planeNormal = (cv::Mat_<double>(3, 1) <<
|
|
444
|
+
_planeTransform.at<double>(0, 1),
|
|
445
|
+
_planeTransform.at<double>(1, 1),
|
|
446
|
+
_planeTransform.at<double>(2, 1));
|
|
447
|
+
const cv::Mat planeOrigin = (cv::Mat_<double>(3, 1) <<
|
|
448
|
+
_planeTransform.at<double>(0, 3),
|
|
449
|
+
_planeTransform.at<double>(1, 3),
|
|
450
|
+
_planeTransform.at<double>(2, 3));
|
|
451
|
+
|
|
452
|
+
// Flip plane normal if it points away from the camera (V15.0c.2).
|
|
453
|
+
cv::Mat camToPlane = planeOrigin - t_arkit;
|
|
454
|
+
if (camToPlane.dot(planeNormal) > 0) {
|
|
455
|
+
planeNormal = -planeNormal;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// Gravity-aligned U/V axes (V15.0c.2).
|
|
459
|
+
const cv::Mat negWorldUp = (cv::Mat_<double>(3, 1) << 0.0, -1.0, 0.0);
|
|
460
|
+
const double upDotN = negWorldUp.dot(planeNormal);
|
|
461
|
+
cv::Mat V_axis = negWorldUp - upDotN * planeNormal;
|
|
462
|
+
double V_axis_norm = cv::norm(V_axis);
|
|
463
|
+
if (V_axis_norm < 1e-6) {
|
|
464
|
+
if (_captureFrameCounter % 30 == 0 || _captureFrameCounter <= 3) {
|
|
465
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
466
|
+
"[V15.0c.2-plane-horizontal] plane normal parallel "
|
|
467
|
+
"to gravity (|V_axis|=%.3e); helper returns NO",
|
|
468
|
+
V_axis_norm);
|
|
469
|
+
}
|
|
470
|
+
return NO;
|
|
471
|
+
}
|
|
472
|
+
V_axis /= V_axis_norm;
|
|
473
|
+
cv::Mat U_axis = planeNormal.cross(V_axis);
|
|
474
|
+
double U_axis_norm = cv::norm(U_axis);
|
|
475
|
+
if (U_axis_norm < 1e-6) return NO;
|
|
476
|
+
U_axis /= U_axis_norm;
|
|
477
|
+
|
|
478
|
+
const double cCenterX = _canvas.cols / 2.0;
|
|
479
|
+
const double cCenterY = _canvas.rows / 2.0;
|
|
480
|
+
cv::Mat K_inv = _K_compose.inv();
|
|
481
|
+
|
|
482
|
+
// V15.0g.1 — adaptive ppm setup BEFORE the corner loop so both
|
|
483
|
+
// Trapezoidal and Rectified modes use the same ppm consistently
|
|
484
|
+
// on the first frame (and every frame after). We need a single
|
|
485
|
+
// raycast for the camera CENTER to compute t_int_center, then
|
|
486
|
+
// ppm = focal / t_int_center → scale = 1.0 → rectangle matches
|
|
487
|
+
// sensor dimensions on canvas. Without this hoist, the corner
|
|
488
|
+
// loop ran at the default ppm=1000 on the first frame (only),
|
|
489
|
+
// producing a narrow output (Ram observed 2026-05-08).
|
|
490
|
+
const double focalForPPM = std::sqrt(
|
|
491
|
+
_K_compose.at<double>(0, 0) * _K_compose.at<double>(1, 1));
|
|
492
|
+
cv::Mat centerPixHomo =
|
|
493
|
+
(cv::Mat_<double>(3, 1) << frameBGR.cols / 2.0, frameBGR.rows / 2.0, 1.0);
|
|
494
|
+
cv::Mat centerRayCv = K_inv * centerPixHomo;
|
|
495
|
+
cv::Mat centerRayArkit = _M_arkitToCv * centerRayCv;
|
|
496
|
+
cv::Mat centerRayWorld = R_new * centerRayArkit;
|
|
497
|
+
cv::Mat diffCenter = planeOrigin - t_arkit;
|
|
498
|
+
double numCenter = diffCenter.dot(planeNormal);
|
|
499
|
+
double denomCenter = centerRayWorld.dot(planeNormal);
|
|
500
|
+
double t_int_center = -1.0;
|
|
501
|
+
if (std::fabs(denomCenter) >= 1e-6) {
|
|
502
|
+
const double t = numCenter / denomCenter;
|
|
503
|
+
if (t >= 0.05 && t <= 50.0) {
|
|
504
|
+
t_int_center = t;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
// V15.0g.2 — adaptive ppm uses PERPENDICULAR cam-to-plane
|
|
508
|
+
// distance, not t_int_center. perp dist is invariant to
|
|
509
|
+
// camera tilt (only changes if the camera translates), so
|
|
510
|
+
// the locked ppm is correct even if the capture starts at a
|
|
511
|
+
// slight tilt. Math: perpDistance = |numCenter| since
|
|
512
|
+
// numCenter = diffCenter.dot(planeNormal) (planeNormal is
|
|
513
|
+
// unit-length post-flip; diffCenter = planeOrigin − cameraPos
|
|
514
|
+
// points in -normal direction so numCenter is signed
|
|
515
|
+
// negative — take fabs).
|
|
516
|
+
const double perpDistanceForPPM = std::fabs(numCenter);
|
|
517
|
+
if (_dynamicKPixelsPerMeter <= 0.0 && perpDistanceForPPM >= 0.05) {
|
|
518
|
+
_dynamicKPixelsPerMeter = focalForPPM / perpDistanceForPPM;
|
|
519
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
520
|
+
"[V15.0g.1-adaptive-ppm] capture ppm locked to %.1f "
|
|
521
|
+
"(focal=%.1f / perpDist=%.3fm) — sensor 1:1 canvas "
|
|
522
|
+
"mapping; first frame's rectangle = sensor dims; "
|
|
523
|
+
"scale stays constant across tilt",
|
|
524
|
+
_dynamicKPixelsPerMeter, focalForPPM, perpDistanceForPPM);
|
|
525
|
+
}
|
|
526
|
+
const double kPixelsPerMeter =
|
|
527
|
+
(_dynamicKPixelsPerMeter > 0.0) ? _dynamicKPixelsPerMeter : 1000.0;
|
|
528
|
+
|
|
529
|
+
// V15.0g.4 — compute first-frame anchor offset. If we have a
|
|
530
|
+
// valid t_int_center, set the first-frame plane-local UV ONCE
|
|
531
|
+
// per capture so the FIRST plane-projected frame lands at canvas
|
|
532
|
+
// (cCenterX, cCenterY). Subsequent frames' canvas positions are
|
|
533
|
+
// RELATIVE to this anchor. Without this, ARKit's arbitrary
|
|
534
|
+
// plane-anchor origin caused frames to land off-canvas (Ram
|
|
535
|
+
// observed 2026-05-08 — cooler clipped at canvas left edge
|
|
536
|
+
// because ARKit anchored the cooler plane 0.3m to the left of
|
|
537
|
+
// where the camera was aimed).
|
|
538
|
+
if (!_haveFirstPlaneAnchor && t_int_center > 0.0) {
|
|
539
|
+
cv::Mat P_world_first = t_arkit + t_int_center * centerRayWorld;
|
|
540
|
+
cv::Mat diffWorld_first = P_world_first - planeOrigin;
|
|
541
|
+
_firstPlaneAnchorUp = diffWorld_first.dot(U_axis);
|
|
542
|
+
_firstPlaneAnchorVp = diffWorld_first.dot(V_axis);
|
|
543
|
+
_haveFirstPlaneAnchor = YES;
|
|
544
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
545
|
+
"[V15.0g.4-anchor] first-frame plane anchor offset locked: "
|
|
546
|
+
"(Up=%.3fm, Vp=%.3fm) — subsequent frames offset relative "
|
|
547
|
+
"to this so first frame lands at canvas centre regardless "
|
|
548
|
+
"of where ARKit placed its plane origin",
|
|
549
|
+
_firstPlaneAnchorUp, _firstPlaneAnchorVp);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
std::vector<cv::Point2f> camCorners = {
|
|
553
|
+
cv::Point2f(0, 0),
|
|
554
|
+
cv::Point2f((float)frameBGR.cols, 0),
|
|
555
|
+
cv::Point2f((float)frameBGR.cols, (float)frameBGR.rows),
|
|
556
|
+
cv::Point2f(0, (float)frameBGR.rows)
|
|
557
|
+
};
|
|
558
|
+
std::vector<cv::Point2f> canvasCorners;
|
|
559
|
+
canvasCorners.reserve(4);
|
|
560
|
+
|
|
561
|
+
bool degenerate = false;
|
|
562
|
+
double currentMaxTInt = 0.0;
|
|
563
|
+
for (const auto &cp : camCorners) {
|
|
564
|
+
cv::Mat pixHomo = (cv::Mat_<double>(3, 1) << cp.x, cp.y, 1.0);
|
|
565
|
+
cv::Mat rayCv = K_inv * pixHomo;
|
|
566
|
+
cv::Mat rayArkit = _M_arkitToCv * rayCv;
|
|
567
|
+
cv::Mat rayWorld = R_new * rayArkit;
|
|
568
|
+
cv::Mat diff = planeOrigin - t_arkit;
|
|
569
|
+
const double num = diff.dot(planeNormal);
|
|
570
|
+
const double denom = rayWorld.dot(planeNormal);
|
|
571
|
+
if (std::fabs(denom) < 1e-6) { degenerate = true; break; }
|
|
572
|
+
const double t_int = num / denom;
|
|
573
|
+
if (t_int < 0.05 || t_int > 50.0) { degenerate = true; break; }
|
|
574
|
+
if (t_int > currentMaxTInt) currentMaxTInt = t_int;
|
|
575
|
+
cv::Mat P_world = t_arkit + t_int * rayWorld;
|
|
576
|
+
cv::Mat diffWorld = P_world - planeOrigin;
|
|
577
|
+
const double Up = diffWorld.dot(U_axis);
|
|
578
|
+
const double Vp = diffWorld.dot(V_axis);
|
|
579
|
+
// V15.0g.4 — apply first-frame anchor offset.
|
|
580
|
+
// V15.0g.5 — lock CROSS-AXIS to first frame's value to
|
|
581
|
+
// suppress unintentional drift from handheld yaw/translate
|
|
582
|
+
// during single-axis pans. _isLandscape == TRUE means the
|
|
583
|
+
// user's intentional pan axis is V (vertical, tilt-down on
|
|
584
|
+
// a landscape phone) → lock U. FALSE means intentional pan
|
|
585
|
+
// is U (horizontal translate on portrait phone) → lock V.
|
|
586
|
+
// Without this, ~20° of unintentional yaw over a 4-second
|
|
587
|
+
// tilt-down pan caused 400 px of horizontal staircase drift
|
|
588
|
+
// in the panorama (Ram observed 2026-05-08).
|
|
589
|
+
double dU = Up - _firstPlaneAnchorUp;
|
|
590
|
+
double dV = Vp - _firstPlaneAnchorVp;
|
|
591
|
+
if (_isLandscape) {
|
|
592
|
+
dU = 0.0; // pan axis = V; lock U to first-frame value
|
|
593
|
+
} else {
|
|
594
|
+
dV = 0.0; // pan axis = U; lock V to first-frame value
|
|
595
|
+
}
|
|
596
|
+
const double cU = cCenterX + dU * kPixelsPerMeter;
|
|
597
|
+
const double cV = cCenterY + dV * kPixelsPerMeter;
|
|
598
|
+
canvasCorners.emplace_back((float)cU, (float)cV);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
// V15.0i — off-plane detection now uses CENTER ray t_int instead of
|
|
602
|
+
// max corner t_int. The max corner heuristic was tripping at
|
|
603
|
+
// ~30° camera tilt because the BOTTOM corners' rays go very
|
|
604
|
+
// oblique to the plane and t_int explodes — even though the
|
|
605
|
+
// camera is still legitimately aimed AT the plane (just oblique).
|
|
606
|
+
// Center-ray t_int tracks "where the camera is aimed" which is
|
|
607
|
+
// what we actually want. Off-plane should fire only when the
|
|
608
|
+
// camera is genuinely no longer looking at the plane (~78°+ tilt
|
|
609
|
+
// past the wall edge), not when corner rays graze the plane.
|
|
610
|
+
constexpr double kOffPlaneMultiplier = 3.0;
|
|
611
|
+
if (!degenerate
|
|
612
|
+
&& _firstPlaneTInt > 0.0
|
|
613
|
+
&& t_int_center > 0.0
|
|
614
|
+
&& t_int_center > kOffPlaneMultiplier * _firstPlaneTInt) {
|
|
615
|
+
if (_captureFrameCounter % 5 == 0 || _captureFrameCounter <= 3) {
|
|
616
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
617
|
+
"[V15.0c.2-offplane] capFr=%ld camera off detected plane "
|
|
618
|
+
"(t_int_center=%.2fm baseline=%.2fm ratio=%.2f "
|
|
619
|
+
"maxCornerTInt=%.2fm); helper returns NO",
|
|
620
|
+
(long)_captureFrameCounter,
|
|
621
|
+
t_int_center, _firstPlaneTInt,
|
|
622
|
+
t_int_center / std::max(0.001, _firstPlaneTInt),
|
|
623
|
+
currentMaxTInt);
|
|
624
|
+
}
|
|
625
|
+
degenerate = true;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
if (degenerate) {
|
|
629
|
+
if (_captureFrameCounter % 30 == 0 || _captureFrameCounter <= 3) {
|
|
630
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
631
|
+
"[V15.0b-plane-degenerate] capFr=%ld ray ∥ plane or out of "
|
|
632
|
+
"range; helper returns NO",
|
|
633
|
+
(long)_captureFrameCounter);
|
|
634
|
+
}
|
|
635
|
+
return NO;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// First successful plane-projected frame in this capture sets the
|
|
639
|
+
// off-plane baseline. V15.0i — baseline is now the FIRST FRAME's
|
|
640
|
+
// CENTER ray t_int, not the max corner. Subsequent frames'
|
|
641
|
+
// center t_int is compared against this.
|
|
642
|
+
if (_firstPlaneTInt <= 0.0 && t_int_center > 0.0) {
|
|
643
|
+
_firstPlaneTInt = t_int_center;
|
|
644
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
645
|
+
"[V15.0c.2-baseline] off-plane baseline _firstPlaneTInt "
|
|
646
|
+
"set to %.3fm (V15.0i: center-ray t_int on first plane-"
|
|
647
|
+
"projected frame; was max-corner pre-V15.0i)",
|
|
648
|
+
_firstPlaneTInt);
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
// V15.0g — Rectified rendering: REPLACE the trapezoidal canvas
|
|
652
|
+
// corners with a clean rectangle centred on the camera-center's
|
|
653
|
+
// raycast anchor. Reuses t_int_center / centerRayWorld /
|
|
654
|
+
// diffCenter computed above (V15.0g.1 hoist) so we don't redo
|
|
655
|
+
// the raycast.
|
|
656
|
+
//
|
|
657
|
+
// V15.0g.2 — scale now uses the PERPENDICULAR camera-to-plane
|
|
658
|
+
// distance, not the center ray's t_int. Why: t_int_center is
|
|
659
|
+
// |perp_dist / cos(tilt)| which GROWS as the user tilts off-
|
|
660
|
+
// perpendicular (Ram observed scale going 1.0→1.55 over a 50°
|
|
661
|
+
// top-to-bottom pan, distorting the panorama by 55%). The
|
|
662
|
+
// perpendicular distance is INVARIANT to tilt — only changes if
|
|
663
|
+
// the camera physically translates closer/farther from the wall.
|
|
664
|
+
// For the typical retail-audit pan (operator stays put, tilts
|
|
665
|
+
// camera), perp distance is constant → scale stays constant →
|
|
666
|
+
// rectangle stays sensor-size → no smear / trumpet effect.
|
|
667
|
+
//
|
|
668
|
+
// ANCHOR position still uses t_int_center (the center ray's
|
|
669
|
+
// intersection on the plane), which is correct: anchor tracks
|
|
670
|
+
// where the camera is AIMED, scale tracks how BIG the camera's
|
|
671
|
+
// view is. These are independent — V15.0g had them tangled.
|
|
672
|
+
if (_config.planeProjectionStyle == RLISPlaneProjectionStyleRectified) {
|
|
673
|
+
if (t_int_center <= 0.0) return NO;
|
|
674
|
+
cv::Mat P_center_world = t_arkit + t_int_center * centerRayWorld;
|
|
675
|
+
cv::Mat diffCenterWorld = P_center_world - planeOrigin;
|
|
676
|
+
double Up_center = diffCenterWorld.dot(U_axis);
|
|
677
|
+
double Vp_center = diffCenterWorld.dot(V_axis);
|
|
678
|
+
// V15.0g.4 — apply first-frame anchor offset so first frame
|
|
679
|
+
// lands at canvas centre, subsequent frames offset relative.
|
|
680
|
+
// V15.0g.5 — also lock the CROSS-AXIS to first-frame value
|
|
681
|
+
// to suppress handheld yaw drift (see corner loop above for
|
|
682
|
+
// the same logic).
|
|
683
|
+
double dU_center = Up_center - _firstPlaneAnchorUp;
|
|
684
|
+
double dV_center = Vp_center - _firstPlaneAnchorVp;
|
|
685
|
+
if (_isLandscape) {
|
|
686
|
+
dU_center = 0.0;
|
|
687
|
+
} else {
|
|
688
|
+
dV_center = 0.0;
|
|
689
|
+
}
|
|
690
|
+
double cU_anchor = cCenterX + dU_center * kPixelsPerMeter;
|
|
691
|
+
double cV_anchor = cCenterY + dV_center * kPixelsPerMeter;
|
|
692
|
+
// Perpendicular cam-to-plane distance. diffCenter =
|
|
693
|
+
// planeOrigin − t_arkit was computed at the top of the
|
|
694
|
+
// helper; planeNormal was flipped to point TOWARD the camera
|
|
695
|
+
// (V15.0c.2), so this dot product is negative — take fabs().
|
|
696
|
+
const double camToPlaneDistance = std::fabs(diffCenter.dot(planeNormal));
|
|
697
|
+
double scale = camToPlaneDistance * kPixelsPerMeter / focalForPPM;
|
|
698
|
+
|
|
699
|
+
double halfW = frameBGR.cols / 2.0;
|
|
700
|
+
double halfH = frameBGR.rows / 2.0;
|
|
701
|
+
canvasCorners.clear();
|
|
702
|
+
canvasCorners.reserve(4);
|
|
703
|
+
canvasCorners.emplace_back(
|
|
704
|
+
(float)(cU_anchor - halfW * scale),
|
|
705
|
+
(float)(cV_anchor - halfH * scale)); // TL
|
|
706
|
+
canvasCorners.emplace_back(
|
|
707
|
+
(float)(cU_anchor + halfW * scale),
|
|
708
|
+
(float)(cV_anchor - halfH * scale)); // TR
|
|
709
|
+
canvasCorners.emplace_back(
|
|
710
|
+
(float)(cU_anchor + halfW * scale),
|
|
711
|
+
(float)(cV_anchor + halfH * scale)); // BR
|
|
712
|
+
canvasCorners.emplace_back(
|
|
713
|
+
(float)(cU_anchor - halfW * scale),
|
|
714
|
+
(float)(cV_anchor + halfH * scale)); // BL
|
|
715
|
+
|
|
716
|
+
if (_captureFrameCounter % 5 == 0 || _captureFrameCounter <= 5) {
|
|
717
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
718
|
+
"[V15.0g-rectified] capFr=%ld anchor=(%.0f,%.0f) "
|
|
719
|
+
"scale=%.3f perpDist=%.3fm t_int=%.3fm",
|
|
720
|
+
(long)_captureFrameCounter,
|
|
721
|
+
cU_anchor, cV_anchor, scale,
|
|
722
|
+
camToPlaneDistance, t_int_center);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
// Warp + paint.
|
|
727
|
+
cv::Mat H = cv::getPerspectiveTransform(camCorners, canvasCorners);
|
|
728
|
+
cv::Mat warpedCanvas = cv::Mat::zeros(_canvas.size(), CV_8UC3);
|
|
729
|
+
cv::warpPerspective(frameBGR, warpedCanvas, H, _canvas.size(),
|
|
730
|
+
cv::INTER_LINEAR,
|
|
731
|
+
cv::BORDER_CONSTANT,
|
|
732
|
+
cv::Scalar(0, 0, 0));
|
|
733
|
+
cv::Mat whiteFrame(frameBGR.size(), CV_8UC1, cv::Scalar(255));
|
|
734
|
+
cv::Mat warpedMask = cv::Mat::zeros(_canvas.size(), CV_8UC1);
|
|
735
|
+
cv::warpPerspective(whiteFrame, warpedMask, H, _canvas.size(),
|
|
736
|
+
cv::INTER_NEAREST,
|
|
737
|
+
cv::BORDER_CONSTANT,
|
|
738
|
+
cv::Scalar(0));
|
|
739
|
+
|
|
740
|
+
// ── V15.0h — 2D NCC alignment refinement for plane projection ──
|
|
741
|
+
// Ram observation 2026-05-08: 2D NCC was bypassed entirely in
|
|
742
|
+
// plane mode (V15.0b assumed 3D-correct alignment made it
|
|
743
|
+
// unnecessary). In practice ARKit pose noise + plane fit error +
|
|
744
|
+
// handheld jitter cause sub-pixel-to-few-pixel alignment errors
|
|
745
|
+
// between adjacent slivers. This block extracts a TOP STRIP of
|
|
746
|
+
// the freshly-warped frame, finds the best (Δx, Δy) translation
|
|
747
|
+
// matching the existing canvas content via cv::matchTemplate,
|
|
748
|
+
// and shifts the warpedCanvas + warpedMask by that delta before
|
|
749
|
+
// painting. Reuses the existing _config NCC settings (search
|
|
750
|
+
// margin, confidence threshold, EMA smoothing, pan-axis lock).
|
|
751
|
+
if (_config.enable2dNcc && _accepted >= 1
|
|
752
|
+
&& cv::countNonZero(_canvasMask) > 0) {
|
|
753
|
+
constexpr int kNccSourceHeight = 100;
|
|
754
|
+
const int kNccSearchMargin =
|
|
755
|
+
std::clamp((int)_config.nccSearchMargin2d, 4, 60);
|
|
756
|
+
constexpr int kNccSourceXInset = 30;
|
|
757
|
+
const double kNccConfidenceThreshold =
|
|
758
|
+
std::clamp(_config.nccConfidenceThreshold2d, 0.30, 0.99);
|
|
759
|
+
|
|
760
|
+
cv::Mat nz;
|
|
761
|
+
cv::findNonZero(warpedMask, nz);
|
|
762
|
+
if (!nz.empty()) {
|
|
763
|
+
cv::Rect bb = cv::boundingRect(nz);
|
|
764
|
+
// Source: top strip of bb (X-inset both sides for slide
|
|
765
|
+
// room in matchTemplate).
|
|
766
|
+
const int sourceX = bb.x + kNccSourceXInset;
|
|
767
|
+
const int sourceY = bb.y;
|
|
768
|
+
const int sourceW = bb.width - 2 * kNccSourceXInset;
|
|
769
|
+
const int sourceH = std::min(kNccSourceHeight, bb.height);
|
|
770
|
+
cv::Rect sourceRect(sourceX, sourceY, sourceW, sourceH);
|
|
771
|
+
sourceRect &= cv::Rect(0, 0, _canvas.cols, _canvas.rows);
|
|
772
|
+
|
|
773
|
+
if (sourceRect.width > 0 && sourceRect.height > 0
|
|
774
|
+
&& sourceRect.width >= sourceW
|
|
775
|
+
&& sourceRect.height >= sourceH) {
|
|
776
|
+
cv::Mat sourceRegion = warpedCanvas(sourceRect);
|
|
777
|
+
|
|
778
|
+
// Search canvas region around sourceRect ± margin.
|
|
779
|
+
const int searchLeft = std::max(
|
|
780
|
+
0, sourceRect.x - kNccSearchMargin);
|
|
781
|
+
const int searchTop = std::max(
|
|
782
|
+
0, sourceRect.y - kNccSearchMargin);
|
|
783
|
+
const int searchRight = std::min(
|
|
784
|
+
(int)_canvas.cols,
|
|
785
|
+
sourceRect.x + sourceRect.width + kNccSearchMargin);
|
|
786
|
+
const int searchBottom = std::min(
|
|
787
|
+
(int)_canvas.rows,
|
|
788
|
+
sourceRect.y + sourceRect.height + kNccSearchMargin);
|
|
789
|
+
const int searchW = searchRight - searchLeft;
|
|
790
|
+
const int searchH = searchBottom - searchTop;
|
|
791
|
+
|
|
792
|
+
int alignDx = 0, alignDy = 0;
|
|
793
|
+
double alignConfidence = 0.0;
|
|
794
|
+
bool alignApplied = false;
|
|
795
|
+
|
|
796
|
+
if (searchW >= sourceRect.width
|
|
797
|
+
&& searchH >= sourceRect.height) {
|
|
798
|
+
cv::Rect searchRect(searchLeft, searchTop, searchW, searchH);
|
|
799
|
+
cv::Mat searchMaskRoi = _canvasMask(searchRect);
|
|
800
|
+
if (cv::countNonZero(searchMaskRoi) > 0) {
|
|
801
|
+
cv::Mat searchRegion = _canvas(searchRect);
|
|
802
|
+
cv::Mat sg, rg;
|
|
803
|
+
cv::cvtColor(sourceRegion, sg, cv::COLOR_BGR2GRAY);
|
|
804
|
+
cv::cvtColor(searchRegion, rg, cv::COLOR_BGR2GRAY);
|
|
805
|
+
|
|
806
|
+
cv::Mat result;
|
|
807
|
+
cv::matchTemplate(rg, sg, result, cv::TM_CCOEFF_NORMED);
|
|
808
|
+
double rmin, rmax;
|
|
809
|
+
cv::Point lmin, lmax;
|
|
810
|
+
cv::minMaxLoc(result, &rmin, &rmax, &lmin, &lmax);
|
|
811
|
+
alignConfidence = rmax;
|
|
812
|
+
|
|
813
|
+
if (alignConfidence >= kNccConfidenceThreshold) {
|
|
814
|
+
const int matchX = searchLeft + lmax.x;
|
|
815
|
+
const int matchY = searchTop + lmax.y;
|
|
816
|
+
int rawDx = matchX - sourceRect.x;
|
|
817
|
+
int rawDy = matchY - sourceRect.y;
|
|
818
|
+
// Clamp to search margin (defensive).
|
|
819
|
+
rawDx = std::clamp(rawDx, -kNccSearchMargin, kNccSearchMargin);
|
|
820
|
+
rawDy = std::clamp(rawDy, -kNccSearchMargin, kNccSearchMargin);
|
|
821
|
+
|
|
822
|
+
// Pan-axis lock (1C): clamp cross-axis tighter.
|
|
823
|
+
if (_config.enableNcc2dPanAxisLock) {
|
|
824
|
+
const int crossLock = std::clamp(
|
|
825
|
+
(int)_config.ncc2dCrossAxisLockPx, 0, 30);
|
|
826
|
+
if (_isLandscape) {
|
|
827
|
+
rawDx = std::clamp(rawDx, -crossLock, crossLock);
|
|
828
|
+
} else {
|
|
829
|
+
rawDy = std::clamp(rawDy, -crossLock, crossLock);
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
// EMA smoothing (1B).
|
|
834
|
+
if (_config.enableNcc2dEmaSmoothing) {
|
|
835
|
+
if (_haveNcc2dEmaHistory) {
|
|
836
|
+
const double a = std::clamp(
|
|
837
|
+
_config.ncc2dEmaAlpha, 0.05, 0.95);
|
|
838
|
+
rawDx = (int)std::round(
|
|
839
|
+
a * rawDx + (1.0 - a) * _lastNcc2dDxApplied);
|
|
840
|
+
rawDy = (int)std::round(
|
|
841
|
+
a * rawDy + (1.0 - a) * _lastNcc2dDyApplied);
|
|
842
|
+
}
|
|
843
|
+
_lastNcc2dDxApplied = rawDx;
|
|
844
|
+
_lastNcc2dDyApplied = rawDy;
|
|
845
|
+
_haveNcc2dEmaHistory = YES;
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
alignDx = rawDx;
|
|
849
|
+
alignDy = rawDy;
|
|
850
|
+
alignApplied = (alignDx != 0 || alignDy != 0);
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
// Apply offset by translating the warpedCanvas + warpedMask.
|
|
856
|
+
if (alignApplied) {
|
|
857
|
+
cv::Mat translation = (cv::Mat_<double>(2, 3) <<
|
|
858
|
+
1, 0, (double)alignDx,
|
|
859
|
+
0, 1, (double)alignDy);
|
|
860
|
+
cv::Mat warpedCanvasShifted, warpedMaskShifted;
|
|
861
|
+
cv::warpAffine(warpedCanvas, warpedCanvasShifted, translation,
|
|
862
|
+
_canvas.size(), cv::INTER_LINEAR,
|
|
863
|
+
cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0));
|
|
864
|
+
cv::warpAffine(warpedMask, warpedMaskShifted, translation,
|
|
865
|
+
_canvas.size(), cv::INTER_NEAREST,
|
|
866
|
+
cv::BORDER_CONSTANT, cv::Scalar(0));
|
|
867
|
+
warpedCanvas = warpedCanvasShifted;
|
|
868
|
+
warpedMask = warpedMaskShifted;
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
if (_captureFrameCounter % 5 == 0 || _captureFrameCounter <= 5) {
|
|
872
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
873
|
+
"[V15.0h-plane-ncc] capFr=%ld dx=%+d dy=%+d "
|
|
874
|
+
"conf=%.3f applied=%d",
|
|
875
|
+
(long)_captureFrameCounter,
|
|
876
|
+
alignDx, alignDy, alignConfidence, (int)alignApplied);
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// Paint mode (composes with plane mode).
|
|
883
|
+
cv::Mat canvasMaskZero;
|
|
884
|
+
cv::compare(_canvasMask, 0, canvasMaskZero, cv::CMP_EQ);
|
|
885
|
+
cv::Mat paintMaskFresh;
|
|
886
|
+
cv::bitwise_and(canvasMaskZero, warpedMask, paintMaskFresh);
|
|
887
|
+
warpedCanvas.copyTo(_canvas, paintMaskFresh);
|
|
888
|
+
cv::bitwise_or(_canvasMask, paintMaskFresh, _canvasMask);
|
|
889
|
+
|
|
890
|
+
if (_config.paintMode == RLISPaintModeFeatherBlend) {
|
|
891
|
+
cv::Mat canvasMaskNonZero;
|
|
892
|
+
cv::compare(_canvasMask, 0, canvasMaskNonZero, cv::CMP_NE);
|
|
893
|
+
cv::Mat overlapMask;
|
|
894
|
+
cv::bitwise_and(canvasMaskNonZero, warpedMask, overlapMask);
|
|
895
|
+
if (cv::countNonZero(overlapMask) > 0) {
|
|
896
|
+
cv::Mat blended;
|
|
897
|
+
cv::addWeighted(warpedCanvas, 0.3, _canvas, 0.7, 0.0, blended);
|
|
898
|
+
blended.copyTo(_canvas, overlapMask);
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
// Update painted-extent telemetry from warpedMask bbox.
|
|
903
|
+
cv::Mat nz;
|
|
904
|
+
cv::findNonZero(warpedMask, nz);
|
|
905
|
+
if (!nz.empty()) {
|
|
906
|
+
cv::Rect bb = cv::boundingRect(nz);
|
|
907
|
+
_maxDstY = std::max((int)_maxDstY, bb.y + bb.height);
|
|
908
|
+
}
|
|
909
|
+
[tele setValue:@(_maxDstY) forKey:@"paintedExtent"];
|
|
910
|
+
_accepted += 1;
|
|
911
|
+
if (_captureFrameCounter % 5 == 0 || _captureFrameCounter <= 5) {
|
|
912
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
913
|
+
"[V15.0b-paint] capFr=%ld plane-projected corners=("
|
|
914
|
+
"%.0f,%.0f)→(%.0f,%.0f)→(%.0f,%.0f)→(%.0f,%.0f) "
|
|
915
|
+
"_accepted=%ld",
|
|
916
|
+
(long)_captureFrameCounter,
|
|
917
|
+
canvasCorners[0].x, canvasCorners[0].y,
|
|
918
|
+
canvasCorners[1].x, canvasCorners[1].y,
|
|
919
|
+
canvasCorners[2].x, canvasCorners[2].y,
|
|
920
|
+
canvasCorners[3].x, canvasCorners[3].y,
|
|
921
|
+
(long)_accepted);
|
|
922
|
+
}
|
|
923
|
+
[tele setValue:@(RLISFrameOutcomeAcceptedHigh) forKey:@"outcome"];
|
|
924
|
+
auto t1 = std::chrono::steady_clock::now();
|
|
925
|
+
double ms = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
926
|
+
t1 - t0).count() / 1000.0;
|
|
927
|
+
[tele setValue:@(ms) forKey:@"processingMs"];
|
|
928
|
+
return YES;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
- (void)reset {
|
|
932
|
+
// V12.12 — canvas alloc DEFERRED to first-frame branch (see
|
|
933
|
+
// ingestPixelBuffer below) so we can size it based on the
|
|
934
|
+
// pose-detected orientation. Empty Mats here signal "no canvas
|
|
935
|
+
// yet" — first-frame branch checks `_canvas.empty()` and
|
|
936
|
+
// allocates accordingly.
|
|
937
|
+
_canvas = cv::Mat();
|
|
938
|
+
_canvasMask = cv::Mat();
|
|
939
|
+
_firstRotationArkit = cv::Mat();
|
|
940
|
+
_R_panToWorld = cv::Mat();
|
|
941
|
+
_K_compose = cv::Mat();
|
|
942
|
+
_focalCompose = 0;
|
|
943
|
+
_canvasOriginCylX = 0;
|
|
944
|
+
_canvasOriginCylY = 0;
|
|
945
|
+
_hasFirstFrame = false;
|
|
946
|
+
_accepted = 0;
|
|
947
|
+
_snapshotSeq = 0;
|
|
948
|
+
// V12.11 Step D — clear running-max trackers; will be
|
|
949
|
+
// re-initialised to first-frame position on next accept.
|
|
950
|
+
_firstFrameDstX = 0;
|
|
951
|
+
_firstFrameDstY = 0;
|
|
952
|
+
_maxDstX = 0;
|
|
953
|
+
_maxDstY = 0;
|
|
954
|
+
|
|
955
|
+
// V13.0e — clear triangulation state so a fresh capture starts
|
|
956
|
+
// with no carried-over keypoints/descriptors/poses. ORB detector
|
|
957
|
+
// itself stays alive (re-created in init only) — it has no
|
|
958
|
+
// per-capture state.
|
|
959
|
+
_prevKeypoints.clear();
|
|
960
|
+
_prevDescriptors = cv::Mat();
|
|
961
|
+
_prevRotationArkit = cv::Mat();
|
|
962
|
+
_prevTranslationArkit = cv::Mat();
|
|
963
|
+
_firstTranslationArkit = cv::Mat();
|
|
964
|
+
_hasPrevAccept = false;
|
|
965
|
+
|
|
966
|
+
// V13.0g — zero the per-accept incremental tri accumulator.
|
|
967
|
+
_accumTriCorrectionX = 0.0;
|
|
968
|
+
_accumTriCorrectionY = 0.0;
|
|
969
|
+
|
|
970
|
+
// V14.0a — clear prev accept canvas position; set on first accept.
|
|
971
|
+
_prevAcceptDstX = 0;
|
|
972
|
+
_prevAcceptDstY = 0;
|
|
973
|
+
|
|
974
|
+
// V15.0c.2 — reset the off-plane detector. First successful
|
|
975
|
+
// plane-projected frame in the new capture will set the baseline.
|
|
976
|
+
_firstPlaneTInt = 0.0;
|
|
977
|
+
|
|
978
|
+
// V15.0d — reset per-capture diagnostics + NCC EMA history +
|
|
979
|
+
// any latched plane (so Virtual mode synthesizes fresh from the
|
|
980
|
+
// new first-frame pose, and ARKit mode awaits a fresh propagation
|
|
981
|
+
// from the bridge).
|
|
982
|
+
_captureFrameCounter = 0;
|
|
983
|
+
_lastNcc2dDxApplied = 0;
|
|
984
|
+
_lastNcc2dDyApplied = 0;
|
|
985
|
+
_haveNcc2dEmaHistory = NO;
|
|
986
|
+
_planeTransform = cv::Mat();
|
|
987
|
+
// V15.0g.1 — clear the adaptive ppm so the next capture's first
|
|
988
|
+
// frame computes a fresh value from its distance.
|
|
989
|
+
_dynamicKPixelsPerMeter = 0.0;
|
|
990
|
+
// V15.0g.4 — clear first-frame plane anchor so the next capture's
|
|
991
|
+
// first plane-projected frame becomes the new canvas-center
|
|
992
|
+
// reference.
|
|
993
|
+
_firstPlaneAnchorUp = 0.0;
|
|
994
|
+
_firstPlaneAnchorVp = 0.0;
|
|
995
|
+
_haveFirstPlaneAnchor = NO;
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
- (NSInteger)acceptedCount { return _accepted; }
|
|
999
|
+
|
|
1000
|
+
// Quaternion → rotation matrix (CV_64F).
|
|
1001
|
+
static cv::Mat quatToR(double qx, double qy, double qz, double qw) {
|
|
1002
|
+
double n = std::sqrt(qx*qx + qy*qy + qz*qz + qw*qw);
|
|
1003
|
+
if (n > 1e-9) { qx /= n; qy /= n; qz /= n; qw /= n; }
|
|
1004
|
+
return (cv::Mat_<double>(3, 3) <<
|
|
1005
|
+
1 - 2*(qy*qy + qz*qz), 2*(qx*qy - qw*qz), 2*(qx*qz + qw*qy),
|
|
1006
|
+
2*(qx*qy + qw*qz), 1 - 2*(qx*qx + qz*qz), 2*(qy*qz - qw*qx),
|
|
1007
|
+
2*(qx*qz - qw*qy), 2*(qy*qz + qw*qx), 1 - 2*(qx*qx + qy*qy));
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
// V11 Gap #22: deleted dead kMinStripWidthPx/kMaxStripWidthPx.
|
|
1011
|
+
// These were strip-width bounds for the original slit-scan design,
|
|
1012
|
+
// never referenced in the first-painted-wins implementation.
|
|
1013
|
+
|
|
1014
|
+
// V12.12 — fraction of the PAN-AXIS the rectilinear engine retains
|
|
1015
|
+
// per frame. The remaining (1 - fraction) is cropped equally from
|
|
1016
|
+
// both edges of the pan axis, keeping the perpendicular axis full.
|
|
1017
|
+
//
|
|
1018
|
+
// Apple-pano-style slit scan: each frame contributes a NARROWER-
|
|
1019
|
+
// than-frame slit centred on the screen, perpendicular to motion.
|
|
1020
|
+
// The clipped-out content (top/bottom in landscape, left/right in
|
|
1021
|
+
// portrait — the user-perceived edges along the pan direction) sits
|
|
1022
|
+
// behind translucent dim bars in the live preview. Earlier versions
|
|
1023
|
+
// (V12.11 Step 3) clipped the LONG sensor axis (perpendicular to
|
|
1024
|
+
// pan in landscape, along pan in portrait) which produced bars on
|
|
1025
|
+
// the wrong screen edge in landscape and forced the JS layer to
|
|
1026
|
+
// guess orientation. V12.12 makes the engine itself
|
|
1027
|
+
// orientation-aware: clip rows for landscape (canvas Y is pan
|
|
1028
|
+
// axis), clip cols for portrait (canvas X is pan axis).
|
|
1029
|
+
//
|
|
1030
|
+
// First-frame and subsequent-frame branches both reference this
|
|
1031
|
+
// constant — DRY-critical because if they ever drift the engine
|
|
1032
|
+
// misbehaves (frame 1 placed bigger than frame 2's source ROI).
|
|
1033
|
+
// V14.0pre.1 — bumped from 0.10 back to 0.30 after V14.0pre field test
|
|
1034
|
+
// surfaced gaps from fast-pan per-accept advance exceeding clipH. At
|
|
1035
|
+
// clipH=108 (V14.0pre), per-accept advance during burst pan (~3000 px/s
|
|
1036
|
+
// pan rate × ~40 ms accept time = 120 px/accept) exceeds the slit, so
|
|
1037
|
+
// adjacent slits don't overlap → unpainted canvas Y bands = gaps.
|
|
1038
|
+
//
|
|
1039
|
+
// At 0.30, clipH = 324 px. Still 2× narrower than V13.0g's 0.70 (756
|
|
1040
|
+
// px) — meaningfully reduces within-slit multi-depth disagreement (the
|
|
1041
|
+
// door-shear in V13.0g) — but with 3× safety margin over typical burst
|
|
1042
|
+
// per-accept advance, no gaps.
|
|
1043
|
+
static const double kPanAxisFractionRect = 0.30;
|
|
1044
|
+
|
|
1045
|
+
// V13.0a — homographyOffset() and the kHomogTier* constants were
|
|
1046
|
+
// removed in the revert from V12.11.1 + V12.14 (ORB+RANSAC homography
|
|
1047
|
+
// correction with 3-tier confidence ladder) back to pose-only paste.
|
|
1048
|
+
//
|
|
1049
|
+
// Per-frame homography refinement was chronically fragile under low-
|
|
1050
|
+
// overlap / low-texture / fast-pan conditions. V12.14's tier ladder
|
|
1051
|
+
// filtered the worst homographies but couldn't fully tame them — MED
|
|
1052
|
+
// tier (translation-only) stripped rotation/scale and produced
|
|
1053
|
+
// visible chevrons + frame-stacking pull-back artifacts.
|
|
1054
|
+
//
|
|
1055
|
+
// V13.0b will reintroduce a LIGHTWEIGHT 1D column-edge NCC
|
|
1056
|
+
// correlation (~1 ms, ±100 px search) for sub-pixel perpendicular
|
|
1057
|
+
// drift correction on top of pose — the algorithm production camera
|
|
1058
|
+
// apps (iOS Camera Pano, Samsung native pano) ship. Until that
|
|
1059
|
+
// lands, perpendicular drift is bounded only by ARKit pose accuracy.
|
|
1060
|
+
|
|
1061
|
+
- (RLISFrameTelemetry *)ingestPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
|
1062
|
+
qx:(double)qx
|
|
1063
|
+
qy:(double)qy
|
|
1064
|
+
qz:(double)qz
|
|
1065
|
+
qw:(double)qw
|
|
1066
|
+
tx:(double)tx
|
|
1067
|
+
ty:(double)ty
|
|
1068
|
+
tz:(double)tz
|
|
1069
|
+
fx:(double)fx
|
|
1070
|
+
fy:(double)fy
|
|
1071
|
+
cx:(double)cx
|
|
1072
|
+
cy:(double)cy
|
|
1073
|
+
imageWidth:(NSInteger)imageWidth
|
|
1074
|
+
imageHeight:(NSInteger)imageHeight
|
|
1075
|
+
yaw:(double)yaw
|
|
1076
|
+
pitch:(double)pitch
|
|
1077
|
+
fovHorizDegrees:(double)fovHorizDegrees
|
|
1078
|
+
fovVertDegrees:(double)fovVertDegrees
|
|
1079
|
+
trackingPoor:(BOOL)trackingPoor
|
|
1080
|
+
{
|
|
1081
|
+
auto t0 = std::chrono::steady_clock::now();
|
|
1082
|
+
RLISFrameTelemetry *tele = [[RLISFrameTelemetry alloc] init];
|
|
1083
|
+
// V12.12 — set isLandscape on every telemetry up-front so every
|
|
1084
|
+
// return path (early-out trackingPoor, alignment-lost, accept,
|
|
1085
|
+
// skip, reverse, etc.) carries the orientation. Stays at the
|
|
1086
|
+
// FIRST-FRAME determination after that point. Pre-first-frame
|
|
1087
|
+
// it's just the default (NO/portrait), which is a safe initial
|
|
1088
|
+
// state for the JS layer to render against.
|
|
1089
|
+
[tele setValue:@(_isLandscape ? YES : NO) forKey:@"isLandscape"];
|
|
1090
|
+
// V12.14.9 — paintedExtent + panExtent on EVERY return path so
|
|
1091
|
+
// the JS band overlay can size the thumb proportionally on every
|
|
1092
|
+
// state event (not just snapshot frames).
|
|
1093
|
+
// V12.14.10: unified — both supported modes use _maxDstY as the
|
|
1094
|
+
// pan-axis leading edge. Pre-first-frame _maxDstY=0 so
|
|
1095
|
+
// paintedExtent=0 → fillRatio=0 → minimum-size thumb.
|
|
1096
|
+
[tele setValue:@(_maxDstY) forKey:@"paintedExtent"];
|
|
1097
|
+
[tele setValue:@(_canvasPanExtent) forKey:@"panExtent"];
|
|
1098
|
+
|
|
1099
|
+
// V13.0a.2 — call counter + per-call state log. Ram's V13.0a.1
|
|
1100
|
+
// trace showed only [V13.0a-focal] firing (1 per capture), no
|
|
1101
|
+
// [V13.0a-pose] / [V13.0a-paint] — meaning subsequent frames are
|
|
1102
|
+
// returning early before reaching the pose projection. Most
|
|
1103
|
+
// likely culprit: trackingPoor=YES on ARKit → silent return at
|
|
1104
|
+
// line ~247. This log fires on EVERY ingestPixelBuffer call
|
|
1105
|
+
// BEFORE any early returns, so we can see how many frames per
|
|
1106
|
+
// capture are coming in and what their tracking state is.
|
|
1107
|
+
static NSInteger _engineCallCounter = 0;
|
|
1108
|
+
_engineCallCounter += 1;
|
|
1109
|
+
// V15.0d — per-capture frame counter resets on -reset (vs the
|
|
1110
|
+
// function-scope static which persists across captures). Used
|
|
1111
|
+
// for diagnostic-log gating so "first 5 frames" diagnostics
|
|
1112
|
+
// fire on every capture, not just the first capture after app
|
|
1113
|
+
// launch.
|
|
1114
|
+
_captureFrameCounter += 1;
|
|
1115
|
+
// Log every 5th call to keep volume manageable but still see
|
|
1116
|
+
// the per-frame pattern. (At 60 fps × 2 sec capture ≈ 120
|
|
1117
|
+
// frames → 24 log lines, fits in Console.app's burst budget.)
|
|
1118
|
+
if (_engineCallCounter % 5 == 0 || _engineCallCounter <= 3) {
|
|
1119
|
+
// V13.0a.4 — FAULT-level os_log to bypass NSLog burst rate-limit.
|
|
1120
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1121
|
+
"[V13.0a-call] #%ld hasFirstFrame=%d trackingPoor=%d "
|
|
1122
|
+
"useRectilinear=%d _accepted=%ld",
|
|
1123
|
+
(long)_engineCallCounter, (int)_hasFirstFrame,
|
|
1124
|
+
(int)trackingPoor, (int)_useRectilinear,
|
|
1125
|
+
(long)_accepted);
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
if (trackingPoor) {
|
|
1129
|
+
[tele setValue:@(RLISFrameOutcomeSkippedTrackingPoor) forKey:@"outcome"];
|
|
1130
|
+
return tele;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
cv::Mat frameBGR;
|
|
1134
|
+
if (![self convertPixelBuffer:pixelBuffer to:frameBGR]) {
|
|
1135
|
+
[tele setValue:@(RLISFrameOutcomeRejectedAlignmentLost) forKey:@"outcome"];
|
|
1136
|
+
return tele;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
cv::Mat R_new = quatToR(qx, qy, qz, qw);
|
|
1140
|
+
|
|
1141
|
+
// ── First frame: build panorama coords + paint the FULL first
|
|
1142
|
+
// frame via cylindrical warp. Earlier slit-scan versions
|
|
1143
|
+
// painted strips only; the user's expectation is "first full
|
|
1144
|
+
// frame visible, slits append at its edges" — which is more
|
|
1145
|
+
// natural than Apple's pure no-first-frame slit-scan.
|
|
1146
|
+
if (!_hasFirstFrame) {
|
|
1147
|
+
_firstRotationArkit = R_new.clone();
|
|
1148
|
+
// V11 Gaps #1, #2: per-axis K scaling + geometric-mean cylinder
|
|
1149
|
+
// radius. See OpenCVIncrementalStitcher.mm for full annotation.
|
|
1150
|
+
double sx = (double)frameBGR.cols / std::max((NSInteger)1, imageWidth);
|
|
1151
|
+
double sy = (double)frameBGR.rows / std::max((NSInteger)1, imageHeight);
|
|
1152
|
+
_K_compose = (cv::Mat_<double>(3, 3) <<
|
|
1153
|
+
fx * sx, 0, cx * sx,
|
|
1154
|
+
0, fy * sy, cy * sy,
|
|
1155
|
+
0, 0, 1);
|
|
1156
|
+
_focalCompose = std::sqrt((fx * sx) * (fy * sy));
|
|
1157
|
+
|
|
1158
|
+
// V13.0a.1 — diagnostic log #1: focal scaling inputs +
|
|
1159
|
+
// computed `focalCompose`. Lets us check whether
|
|
1160
|
+
// imageWidth/imageHeight (from host) match what fx/fy are
|
|
1161
|
+
// normalized to. If `sx` or `sy` ≠ 1.0 unexpectedly, the
|
|
1162
|
+
// `alpha × focalCompose` pixel mapping undercounts (or
|
|
1163
|
+
// overcounts) → "everything shorter / taller than reality"
|
|
1164
|
+
// (the height-shrink artifact in V13.0a captures).
|
|
1165
|
+
NSLog(@"[V13.0a-focal] fx=%.1f fy=%.1f imageW=%ld imageH=%ld "
|
|
1166
|
+
@"frame=%dx%d sx=%.3f sy=%.3f focalCompose=%.1f",
|
|
1167
|
+
fx, fy, (long)imageWidth, (long)imageHeight,
|
|
1168
|
+
frameBGR.cols, frameBGR.rows, sx, sy, _focalCompose);
|
|
1169
|
+
|
|
1170
|
+
cv::Mat fwdArkitCam = (cv::Mat_<double>(3, 1) << 0, 0, -1);
|
|
1171
|
+
cv::Mat fwdWorld = _firstRotationArkit * fwdArkitCam;
|
|
1172
|
+
double fwx = fwdWorld.at<double>(0);
|
|
1173
|
+
double fwz = fwdWorld.at<double>(2);
|
|
1174
|
+
double horiz = std::sqrt(fwx * fwx + fwz * fwz);
|
|
1175
|
+
// V11 Gap #3: reject if camera looking nearly vertical (no
|
|
1176
|
+
// horizontal forward to anchor the panorama frame). See
|
|
1177
|
+
// OpenCVIncrementalStitcher.mm for full annotation.
|
|
1178
|
+
if (horiz < 0.1) {
|
|
1179
|
+
[tele setValue:@(RLISFrameOutcomeRejectedAlignmentLost) forKey:@"outcome"];
|
|
1180
|
+
return tele;
|
|
1181
|
+
}
|
|
1182
|
+
double pzx = fwx / horiz, pzz = fwz / horiz;
|
|
1183
|
+
_R_panToWorld = (cv::Mat_<double>(3, 3) <<
|
|
1184
|
+
pzz, 0, pzx,
|
|
1185
|
+
0, 1, 0,
|
|
1186
|
+
-pzx, 0, pzz);
|
|
1187
|
+
|
|
1188
|
+
// V12.6 Step C: detect orientation from R_panToCam at first
|
|
1189
|
+
// frame — see v9 engine for the rationale. JS's
|
|
1190
|
+
// frameRotationDegrees is unreliable when iOS orientation-
|
|
1191
|
+
// lock is on; ARKit's pose is ground truth.
|
|
1192
|
+
cv::Mat R_panToCam_first = _M_arkitToCv * _firstRotationArkit.t() * _R_panToWorld;
|
|
1193
|
+
const double absR01 = std::fabs(R_panToCam_first.at<double>(0, 1));
|
|
1194
|
+
const double absR11 = std::fabs(R_panToCam_first.at<double>(1, 1));
|
|
1195
|
+
_isLandscape = (absR11 > absR01);
|
|
1196
|
+
// V12.12 — re-stamp the tele with the freshly-detected
|
|
1197
|
+
// isLandscape so the FIRST frame's state event carries the
|
|
1198
|
+
// right value (the up-front set at the top of this method
|
|
1199
|
+
// ran before _isLandscape was computed).
|
|
1200
|
+
[tele setValue:@(_isLandscape ? YES : NO) forKey:@"isLandscape"];
|
|
1201
|
+
NSLog(@"[V12.6-orient] engine=firstwins detected isLandscape=%d "
|
|
1202
|
+
@"|R[0,1]|=%.4f |R[1,1]|=%.4f (frameRotationDegrees from JS = %ld)",
|
|
1203
|
+
(int)_isLandscape, absR01, absR11, (long)_frameRotationDegrees);
|
|
1204
|
+
|
|
1205
|
+
if (_useRectilinear) {
|
|
1206
|
+
// V12.14.10 — UNIFIED clip for both supported modes.
|
|
1207
|
+
// Per the two-mode spec (project memory `ar-stitching-two-modes`),
|
|
1208
|
+
// the supported modes are landscape+vertical-pan and
|
|
1209
|
+
// portrait+horizontal-pan. Both have pan rotation around
|
|
1210
|
+
// CAM +X (the phone's long edge / sensor X direction):
|
|
1211
|
+
//
|
|
1212
|
+
// - landscape vertical pan: phone long edge = user
|
|
1213
|
+
// horizontal; rotation around it = tilt up/down.
|
|
1214
|
+
// - portrait horizontal pan: phone long edge = user
|
|
1215
|
+
// vertical; rotation around it = pan sideways.
|
|
1216
|
+
//
|
|
1217
|
+
// Both rotations move sensor content along sensor Y.
|
|
1218
|
+
// So clip ALONG sensor Y (clip rows to 70% = ~756 px),
|
|
1219
|
+
// perp = full sensor X (1920) — IDENTICAL in both modes.
|
|
1220
|
+
//
|
|
1221
|
+
// Pre-V12.14.10 the portrait branch wrongly clipped sensor X
|
|
1222
|
+
// (assumed cam +Y rotation = portrait+vertical-pan, an
|
|
1223
|
+
// unsupported mode). That mis-wiring was the root cause of
|
|
1224
|
+
// Issue 2 ("sideways portrait → first frame only output").
|
|
1225
|
+
int clipW, clipH, srcClipX, srcClipY;
|
|
1226
|
+
clipW = frameBGR.cols; // perpendicular: full sensor X (1920)
|
|
1227
|
+
// V15.0c — clipH driven by _config.kPanAxisFractionRect.
|
|
1228
|
+
// For the FIRST frame, if firstFrameFullFrame is enabled,
|
|
1229
|
+
// override to use the FULL camera sensor frame so the
|
|
1230
|
+
// canvas is anchored with as much content as possible
|
|
1231
|
+
// before subsequent slivers extend it.
|
|
1232
|
+
if (_config.firstFrameFullFrame) {
|
|
1233
|
+
clipH = frameBGR.rows;
|
|
1234
|
+
srcClipY = 0;
|
|
1235
|
+
} else {
|
|
1236
|
+
clipH = std::max(1, (int)(frameBGR.rows * _config.kPanAxisFractionRect));
|
|
1237
|
+
// V15.0c — srcClipY position from sliverPosition.
|
|
1238
|
+
switch (_config.sliverPosition) {
|
|
1239
|
+
case RLISSliverPositionTop:
|
|
1240
|
+
srcClipY = 0;
|
|
1241
|
+
break;
|
|
1242
|
+
case RLISSliverPositionBottom:
|
|
1243
|
+
srcClipY = frameBGR.rows - clipH;
|
|
1244
|
+
break;
|
|
1245
|
+
case RLISSliverPositionCenter:
|
|
1246
|
+
default:
|
|
1247
|
+
srcClipY = (frameBGR.rows - clipH) / 2;
|
|
1248
|
+
break;
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
srcClipX = 0;
|
|
1252
|
+
cv::Mat frameClipped = frameBGR(cv::Rect(srcClipX, srcClipY, clipW, clipH));
|
|
1253
|
+
|
|
1254
|
+
// V12.14.10 — UNIFIED canvas allocation. Both supported
|
|
1255
|
+
// modes pan along canvas Y (mirrors sensor Y motion).
|
|
1256
|
+
// Canvas: 1920 cols × 5000 rows = 1920w × 5000h.
|
|
1257
|
+
// Memory: ~28 MB BGR.
|
|
1258
|
+
//
|
|
1259
|
+
// For portrait+horizontal-pan, the saved JPEG must be in
|
|
1260
|
+
// user-perspective WIDE horizontal strip orientation
|
|
1261
|
+
// (~5000w × 1920h). Rotation 90° applied at snapshot /
|
|
1262
|
+
// finalize time (see writeOutToPath) — keeps the runtime
|
|
1263
|
+
// engine simple, single rotation per output rather than
|
|
1264
|
+
// per-frame paste rotations.
|
|
1265
|
+
//
|
|
1266
|
+
// Pre-V12.14.10 the portrait canvas was 5000×1080 (perp =
|
|
1267
|
+
// sensor Y, wrong) — caused frames to never advance dstX
|
|
1268
|
+
// because the engine's pose projection was on a different
|
|
1269
|
+
// axis than the actual pan direction.
|
|
1270
|
+
if (_canvas.empty()) {
|
|
1271
|
+
int canvasCols = frameBGR.cols; // perp = sensor X = 1920
|
|
1272
|
+
int canvasRows = (int)_canvasPanExtent; // pan = 5000
|
|
1273
|
+
_canvas = cv::Mat::zeros(canvasRows, canvasCols, CV_8UC3);
|
|
1274
|
+
_canvasMask = cv::Mat::zeros(canvasRows, canvasCols, CV_8UC1);
|
|
1275
|
+
NSLog(@"[V12.14.10-canvas] allocated %dx%d (cols x rows) for "
|
|
1276
|
+
@"isLandscape=%d (pan extent %ld, frame=%dx%d)",
|
|
1277
|
+
canvasCols, canvasRows, (int)_isLandscape,
|
|
1278
|
+
(long)_canvasPanExtent, frameBGR.cols, frameBGR.rows);
|
|
1279
|
+
// V15 — log config snapshot at first frame.
|
|
1280
|
+
// V15.0c.4 — fault log so it always shows.
|
|
1281
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1282
|
+
"[V15-slit] panAxisFrac=%.2f clipH=%d clipW=%d "
|
|
1283
|
+
"acceptGate=%ld tri=%d triAccum=%d 1dNcc=%d "
|
|
1284
|
+
"2dNcc=%d ransac=%d paint=%s useDetectedPlane=%d "
|
|
1285
|
+
"planeTransform.empty=%d",
|
|
1286
|
+
_config.kPanAxisFractionRect, clipH, clipW,
|
|
1287
|
+
(long)_config.kMinAcceptDeltaPx,
|
|
1288
|
+
(int)_config.enableTriangulation,
|
|
1289
|
+
(int)_config.enableTriAccumulator,
|
|
1290
|
+
(int)_config.enable1dNcc,
|
|
1291
|
+
(int)_config.enable2dNcc,
|
|
1292
|
+
(int)_config.enableRansacHomography,
|
|
1293
|
+
_config.paintMode == RLISPaintModeFeatherBlend
|
|
1294
|
+
? "FeatherBlend" : "FirstPaintedWins",
|
|
1295
|
+
(int)_config.useDetectedPlane,
|
|
1296
|
+
(int)_planeTransform.empty());
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
// ── V15.0e — FIRST-FRAME PLANE PROJECTION ───────────────
|
|
1300
|
+
// When planeSource is ARKitDetected or Virtual, the first
|
|
1301
|
+
// frame is painted via plane projection (helper) rather
|
|
1302
|
+
// than via the rectilinear paste at canvas (0, 0). This
|
|
1303
|
+
// keeps the first frame and all subsequent slivers in
|
|
1304
|
+
// the SAME coord system — without it, the first frame
|
|
1305
|
+
// and slivers were two disjoint patches on the canvas
|
|
1306
|
+
// (Ram observed 2026-05-08 with planeSource=ARKitDetected).
|
|
1307
|
+
//
|
|
1308
|
+
// If the helper returns NO (plane unavailable / degenerate),
|
|
1309
|
+
// we REFUSE the first frame (per Ram's UX choice) — the
|
|
1310
|
+
// capture screen shows "waiting for plane" and the user
|
|
1311
|
+
// tries again once the plane locks. Subsequent attempts
|
|
1312
|
+
// re-enter the first-frame branch since _hasFirstFrame
|
|
1313
|
+
// stays NO.
|
|
1314
|
+
if (_config.planeSource != RLISPlaneSourceDisabled) {
|
|
1315
|
+
if ([self tryPaintPlaneProjected:frameBGR
|
|
1316
|
+
R_new:R_new
|
|
1317
|
+
tx:tx
|
|
1318
|
+
ty:ty
|
|
1319
|
+
tz:tz
|
|
1320
|
+
tele:tele
|
|
1321
|
+
t0:t0]) {
|
|
1322
|
+
_hasFirstFrame = true;
|
|
1323
|
+
// Init prev-accept state for slit-scan fallback
|
|
1324
|
+
// safety (a future frame that goes off-plane
|
|
1325
|
+
// falls through to slit-scan, which expects
|
|
1326
|
+
// these set). Mirrors the existing rectilinear
|
|
1327
|
+
// first-frame init below.
|
|
1328
|
+
_firstTranslationArkit = (cv::Mat_<double>(3, 1) << tx, ty, tz);
|
|
1329
|
+
_prevRotationArkit = R_new.clone();
|
|
1330
|
+
_prevTranslationArkit = _firstTranslationArkit.clone();
|
|
1331
|
+
{
|
|
1332
|
+
cv::Mat frameGray;
|
|
1333
|
+
cv::cvtColor(frameBGR, frameGray, cv::COLOR_BGR2GRAY);
|
|
1334
|
+
std::vector<cv::KeyPoint> kps;
|
|
1335
|
+
cv::Mat descs;
|
|
1336
|
+
_orbDetector->detectAndCompute(frameGray, cv::noArray(), kps, descs);
|
|
1337
|
+
_prevKeypoints = std::move(kps);
|
|
1338
|
+
_prevDescriptors = descs;
|
|
1339
|
+
}
|
|
1340
|
+
_hasPrevAccept = true;
|
|
1341
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1342
|
+
"[V15.0e-first-plane] first frame painted via "
|
|
1343
|
+
"plane projection (planeSource=%s); slit-scan "
|
|
1344
|
+
"fallback state initialised",
|
|
1345
|
+
_config.planeSource == RLISPlaneSourceVirtual ? "Virtual" : "ARKitDetected");
|
|
1346
|
+
return tele;
|
|
1347
|
+
} else {
|
|
1348
|
+
// Plane not ready — refuse first frame. User
|
|
1349
|
+
// retries on next. _hasFirstFrame stays NO so
|
|
1350
|
+
// the next ingestPixelBuffer re-enters this branch.
|
|
1351
|
+
[tele setValue:@(RLISFrameOutcomeSkippedTrackingPoor) forKey:@"outcome"];
|
|
1352
|
+
if (_captureFrameCounter <= 5 || _captureFrameCounter % 30 == 0) {
|
|
1353
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1354
|
+
"[V15.0e-first-frame-refused] capFr=%ld plane "
|
|
1355
|
+
"not ready (planeSource=%s, planeTransform.empty=%d); "
|
|
1356
|
+
"first frame skipped — UI should show 'waiting "
|
|
1357
|
+
"for plane' until lock",
|
|
1358
|
+
(long)_captureFrameCounter,
|
|
1359
|
+
_config.planeSource == RLISPlaneSourceVirtual ? "Virtual" : "ARKitDetected",
|
|
1360
|
+
(int)_planeTransform.empty());
|
|
1361
|
+
}
|
|
1362
|
+
return tele;
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
// V12.12 — first-frame placement at canvas ORIGIN (0, 0).
|
|
1367
|
+
// With the new engine-internal canvas allocation the
|
|
1368
|
+
// canvas perpendicular dim EXACTLY matches the clipped
|
|
1369
|
+
// frame's perpendicular dim, so there's no centring
|
|
1370
|
+
// offset — both axes are 0. As the user pans, dstX
|
|
1371
|
+
// (portrait) or dstY (landscape) advances from 0.
|
|
1372
|
+
int dstX = 0;
|
|
1373
|
+
int dstY = 0;
|
|
1374
|
+
cv::Rect roi(dstX, dstY, clipW, clipH);
|
|
1375
|
+
roi &= cv::Rect(0, 0, _canvas.cols, _canvas.rows);
|
|
1376
|
+
cv::Rect srcR(0, 0, roi.width, roi.height);
|
|
1377
|
+
frameClipped(srcR).copyTo(_canvas(roi));
|
|
1378
|
+
cv::rectangle(_canvasMask, roi, cv::Scalar(255), cv::FILLED);
|
|
1379
|
+
_firstFrameDstX = dstX;
|
|
1380
|
+
_firstFrameDstY = dstY;
|
|
1381
|
+
|
|
1382
|
+
// V15.0c FIX — when first frame painted the FULL camera
|
|
1383
|
+
// frame (firstFrameFullFrame=true), reset the canvas-Y
|
|
1384
|
+
// anchor to the sliver's position within the first frame.
|
|
1385
|
+
// Otherwise subsequent slivers (taken from the configured
|
|
1386
|
+
// sliverPosition of each new frame) land at canvas Y
|
|
1387
|
+
// = -alpha × focal, which is small for typical pan tilt
|
|
1388
|
+
// and therefore INSIDE the first frame's full-frame
|
|
1389
|
+
// painted region (0, 0)–(rows, cols) — first-painted-wins
|
|
1390
|
+
// masks them all out → "first frame paints, nothing else
|
|
1391
|
+
// gets added" (Ram observed this in V15.0b).
|
|
1392
|
+
//
|
|
1393
|
+
// The anchor must be set so that subsequent slivers'
|
|
1394
|
+
// canvas Y matches their physical position relative to
|
|
1395
|
+
// first frame's full content. For sliverPosition=Bottom,
|
|
1396
|
+
// the sliver source at sensor Y = rows-subsequentClipH;
|
|
1397
|
+
// first frame's full paste already painted that sensor-Y
|
|
1398
|
+
// region at canvas Y = rows-subsequentClipH, so the
|
|
1399
|
+
// sliver's canvas anchor should be there too.
|
|
1400
|
+
if (_config.firstFrameFullFrame) {
|
|
1401
|
+
const int subsequentClipH = std::max(1,
|
|
1402
|
+
(int)(frameBGR.rows * _config.kPanAxisFractionRect));
|
|
1403
|
+
int anchorY = 0;
|
|
1404
|
+
switch (_config.sliverPosition) {
|
|
1405
|
+
case RLISSliverPositionTop:
|
|
1406
|
+
anchorY = 0;
|
|
1407
|
+
break;
|
|
1408
|
+
case RLISSliverPositionBottom:
|
|
1409
|
+
anchorY = frameBGR.rows - subsequentClipH;
|
|
1410
|
+
break;
|
|
1411
|
+
case RLISSliverPositionCenter:
|
|
1412
|
+
default:
|
|
1413
|
+
anchorY = (frameBGR.rows - subsequentClipH) / 2;
|
|
1414
|
+
break;
|
|
1415
|
+
}
|
|
1416
|
+
_firstFrameDstY = anchorY;
|
|
1417
|
+
// V15.0c.4 — fault log so it always shows. C-string
|
|
1418
|
+
// %s instead of NSString %@ since os_log doesn't accept
|
|
1419
|
+
// %@ formatters.
|
|
1420
|
+
const char *posStr =
|
|
1421
|
+
_config.sliverPosition == RLISSliverPositionTop ? "Top"
|
|
1422
|
+
: (_config.sliverPosition == RLISSliverPositionBottom ? "Bottom" : "Center");
|
|
1423
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1424
|
+
"[V15.0c-anchor] firstFrameFullFrame=on, "
|
|
1425
|
+
"sliverPosition=%s, frameRows=%d, "
|
|
1426
|
+
"subsequentClipH=%d, _firstFrameDstY=%d",
|
|
1427
|
+
posStr, frameBGR.rows, subsequentClipH, anchorY);
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
// V14.0a — first accept's canvas position becomes prev for
|
|
1431
|
+
// the second accept's homography target-pair construction.
|
|
1432
|
+
_prevAcceptDstX = dstX;
|
|
1433
|
+
_prevAcceptDstY = _firstFrameDstY;
|
|
1434
|
+
// V12.11 Step D — initialise the running-max tracker
|
|
1435
|
+
// to first-frame position. Subsequent frames must
|
|
1436
|
+
// monotonically advance from here along the pan axis.
|
|
1437
|
+
_maxDstX = dstX;
|
|
1438
|
+
_maxDstY = _firstFrameDstY;
|
|
1439
|
+
_hasFirstFrame = true;
|
|
1440
|
+
_accepted = 1;
|
|
1441
|
+
[tele setValue:@(RLISFrameOutcomeAcceptedHigh) forKey:@"outcome"];
|
|
1442
|
+
[tele setValue:@(1.0) forKey:@"confidence"];
|
|
1443
|
+
// V12.14.9 — first-frame paintedExtent on the RECTILINEAR
|
|
1444
|
+
// path (this is the actively-used engine; the cylindrical
|
|
1445
|
+
// branch below is V12.2 legacy). Use the slit's pan-axis
|
|
1446
|
+
// size so the band thumb shows non-zero progress on the
|
|
1447
|
+
// very first frame.
|
|
1448
|
+
// V12.14.10: unified — both supported modes use clipH as
|
|
1449
|
+
// the slit's pan-axis extent.
|
|
1450
|
+
[tele setValue:@(clipH) forKey:@"paintedExtent"];
|
|
1451
|
+
NSLog(@"[V12.12-rect] first frame placed at (%d,%d) clipped=%dx%d "
|
|
1452
|
+
@"(srcClip=%d,%d) along-pan-axis isLandscape=%d focal=%.2f canvas=%dx%d",
|
|
1453
|
+
dstX, dstY, clipW, clipH, srcClipX, srcClipY,
|
|
1454
|
+
(int)_isLandscape, _focalCompose,
|
|
1455
|
+
_canvas.cols, _canvas.rows);
|
|
1456
|
+
|
|
1457
|
+
// V13.0e — initialise translation correction state at the
|
|
1458
|
+
// FIRST accepted frame. All subsequent slits' canvas
|
|
1459
|
+
// positions reference this anchor (firstFrameDstX/Y, set
|
|
1460
|
+
// above) AND this translation origin (_firstTranslationArkit).
|
|
1461
|
+
//
|
|
1462
|
+
// Detect ORB on the FULL sensor frame (not just the
|
|
1463
|
+
// 70%-clipped slit) so features near the leading edge of
|
|
1464
|
+
// the next pan are already in the prev set when the second
|
|
1465
|
+
// accept arrives — knnMatch quality depends on dense feature
|
|
1466
|
+
// coverage in the OVERLAP between consecutive slits.
|
|
1467
|
+
_firstTranslationArkit = (cv::Mat_<double>(3, 1) << tx, ty, tz);
|
|
1468
|
+
_prevRotationArkit = R_new.clone();
|
|
1469
|
+
_prevTranslationArkit = _firstTranslationArkit.clone();
|
|
1470
|
+
{
|
|
1471
|
+
cv::Mat frameGray;
|
|
1472
|
+
cv::cvtColor(frameBGR, frameGray, cv::COLOR_BGR2GRAY);
|
|
1473
|
+
std::vector<cv::KeyPoint> kps;
|
|
1474
|
+
cv::Mat descs;
|
|
1475
|
+
_orbDetector->detectAndCompute(frameGray, cv::noArray(), kps, descs);
|
|
1476
|
+
_prevKeypoints = std::move(kps);
|
|
1477
|
+
_prevDescriptors = descs;
|
|
1478
|
+
}
|
|
1479
|
+
_hasPrevAccept = true;
|
|
1480
|
+
return tele;
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
// V12.2 cylindrical-warp the first frame and place at canvas centre.
|
|
1484
|
+
cv::Mat warpedFirst, warpedFirstMask;
|
|
1485
|
+
cv::Point firstCornerCyl =
|
|
1486
|
+
[self cylindricalWarp:frameBGR rArkit:R_new
|
|
1487
|
+
outImage:warpedFirst outMask:warpedFirstMask];
|
|
1488
|
+
if (warpedFirst.empty()) {
|
|
1489
|
+
[tele setValue:@(RLISFrameOutcomeRejectedAlignmentLost) forKey:@"outcome"];
|
|
1490
|
+
return tele;
|
|
1491
|
+
}
|
|
1492
|
+
int dstX = (int)(_canvas.cols - warpedFirst.cols) / 2;
|
|
1493
|
+
int dstY = (int)(_canvas.rows - warpedFirst.rows) / 2;
|
|
1494
|
+
cv::Rect roi(dstX, dstY, warpedFirst.cols, warpedFirst.rows);
|
|
1495
|
+
roi &= cv::Rect(0, 0, _canvas.cols, _canvas.rows);
|
|
1496
|
+
cv::Rect srcR(0, 0, roi.width, roi.height);
|
|
1497
|
+
warpedFirst(srcR).copyTo(_canvas(roi), warpedFirstMask(srcR));
|
|
1498
|
+
warpedFirstMask(srcR).copyTo(_canvasMask(roi),
|
|
1499
|
+
warpedFirstMask(srcR));
|
|
1500
|
+
_canvasOriginCylX = firstCornerCyl.x - dstX;
|
|
1501
|
+
_canvasOriginCylY = firstCornerCyl.y - dstY;
|
|
1502
|
+
|
|
1503
|
+
_hasFirstFrame = true;
|
|
1504
|
+
_accepted = 1;
|
|
1505
|
+
[tele setValue:@(RLISFrameOutcomeAcceptedHigh) forKey:@"outcome"];
|
|
1506
|
+
[tele setValue:@(1.0) forKey:@"confidence"];
|
|
1507
|
+
// V12.14.9 — paintedExtent was set at line ~445 to _maxDstY (= 0
|
|
1508
|
+
// before first frame). In the cylindrical first-frame branch
|
|
1509
|
+
// we don't update _maxDstY (cylindrical uses its own canvas-
|
|
1510
|
+
// centre placement model rather than the slit-leading-edge
|
|
1511
|
+
// tracker). Leaving paintedExtent at 0 here means the band
|
|
1512
|
+
// thumb shows zero progress on the cylindrical first frame —
|
|
1513
|
+
// acceptable; the next frame's running-max update will set
|
|
1514
|
+
// a real value. Cylindrical is V12.2 legacy and not the
|
|
1515
|
+
// primary engine; rectilinear (above) sets a real value.
|
|
1516
|
+
return tele;
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
if (_useRectilinear) {
|
|
1520
|
+
// V12.8 Variant B subsequent frame: paste the SAME long-side-
|
|
1521
|
+
// clipped portion as the first frame, at canvas position
|
|
1522
|
+
// offset by pan_angle × focal along the pan axis. First-
|
|
1523
|
+
// painted-wins masking ensures only the LEADING-EDGE sliver
|
|
1524
|
+
// (the part outside the previously-painted region) actually
|
|
1525
|
+
// gets painted. Even tiny pans produce immediate
|
|
1526
|
+
// incremental growth — no V12.7 dead-zone where strips
|
|
1527
|
+
// entirely overlapped the first frame.
|
|
1528
|
+
// ── V15.0e — plane-projected stitch path (helper-driven) ──
|
|
1529
|
+
// The actual plane-projection logic lives in
|
|
1530
|
+
// -tryPaintPlaneProjected: (extracted in V15.0e so the
|
|
1531
|
+
// first-frame branch can reuse it — paint first frame and
|
|
1532
|
+
// subsequent slivers in the SAME coord system). This
|
|
1533
|
+
// dispatch site just logs the pre-check state and calls
|
|
1534
|
+
// the helper. Helper returns YES if it painted (caller
|
|
1535
|
+
// returns tele); NO if plane is unavailable / degenerate /
|
|
1536
|
+
// off-plane (caller falls through to slit-scan path).
|
|
1537
|
+
const char *planeSrcStr =
|
|
1538
|
+
_config.planeSource == RLISPlaneSourceVirtual ? "Virtual"
|
|
1539
|
+
: (_config.planeSource == RLISPlaneSourceARKitDetected ? "ARKitDetected"
|
|
1540
|
+
: "Disabled");
|
|
1541
|
+
if (_captureFrameCounter <= 5 || _captureFrameCounter % 60 == 0) {
|
|
1542
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1543
|
+
"[V15.0c.4-pre] capFr=%ld engFr=%ld pre-plane-check: "
|
|
1544
|
+
"planeSource=%s planeTransform.empty=%d "
|
|
1545
|
+
"(Disabled→slit-scan; ARKitDetected→plane only if "
|
|
1546
|
+
"empty=0; Virtual→plane after synthesis)",
|
|
1547
|
+
(long)_captureFrameCounter, (long)_engineCallCounter,
|
|
1548
|
+
planeSrcStr,
|
|
1549
|
+
(int)_planeTransform.empty());
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
if ([self tryPaintPlaneProjected:frameBGR
|
|
1553
|
+
R_new:R_new
|
|
1554
|
+
tx:tx
|
|
1555
|
+
ty:ty
|
|
1556
|
+
tz:tz
|
|
1557
|
+
tele:tele
|
|
1558
|
+
t0:t0]) {
|
|
1559
|
+
return tele;
|
|
1560
|
+
}
|
|
1561
|
+
// Helper returned NO → plane unavailable / degenerate / off-
|
|
1562
|
+
// plane. Fall through to the slit-scan path. When the user
|
|
1563
|
+
// selected planeSource != Disabled but we got here, the
|
|
1564
|
+
// existing slit-scan code paints this frame as a fallback —
|
|
1565
|
+
// less ambitious but always works.
|
|
1566
|
+
if (_config.planeSource != RLISPlaneSourceDisabled
|
|
1567
|
+
&& (_captureFrameCounter <= 3 || _captureFrameCounter % 60 == 0)) {
|
|
1568
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1569
|
+
"[V15.0c.3-noplane] capFr=%ld helper returned NO "
|
|
1570
|
+
"(planeSource=%s planeTransform.empty=%d); slit-scan fallback",
|
|
1571
|
+
(long)_captureFrameCounter,
|
|
1572
|
+
planeSrcStr,
|
|
1573
|
+
(int)_planeTransform.empty());
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
// V15.0e legacy V15.0d inline dispatch block removed below — the
|
|
1577
|
+
// helper -tryPaintPlaneProjected: now owns this logic. The
|
|
1578
|
+
// dead code that follows (until skipPlaneProjection: label)
|
|
1579
|
+
// is preserved as comments for one release; a follow-up commit
|
|
1580
|
+
// will physically delete it once V15.0e is field-validated.
|
|
1581
|
+
#if 0
|
|
1582
|
+
// V15.0d — dispatch on planeSource enum. When the legacy
|
|
1583
|
+
// boolean useDetectedPlane was YES, setConfig already
|
|
1584
|
+
// upgraded planeSource → ARKitDetected (see -setConfig:).
|
|
1585
|
+
bool runPlaneProjection = false;
|
|
1586
|
+
if (_config.planeSource == RLISPlaneSourceARKitDetected) {
|
|
1587
|
+
runPlaneProjection = !_planeTransform.empty();
|
|
1588
|
+
} else if (_config.planeSource == RLISPlaneSourceVirtual) {
|
|
1589
|
+
// Synthesize on first frame in Virtual mode. Plane:
|
|
1590
|
+
// origin = camera_pos + depth × camera_forward_world
|
|
1591
|
+
// normal = -camera_forward_world (pointing AT camera)
|
|
1592
|
+
//
|
|
1593
|
+
// R_new is ARKit camera-to-world rotation; ARKit camera
|
|
1594
|
+
// looks down -Z in its local frame, so multiplying
|
|
1595
|
+
// (0, 0, -1) by R_new yields the world-space direction
|
|
1596
|
+
// the camera is facing (i.e. camera-forward in world).
|
|
1597
|
+
if (_planeTransform.empty()) {
|
|
1598
|
+
cv::Mat camForwardWorld = R_new *
|
|
1599
|
+
(cv::Mat_<double>(3, 1) << 0.0, 0.0, -1.0);
|
|
1600
|
+
cv::Mat normalWorld = -camForwardWorld;
|
|
1601
|
+
cv::Mat originWorld =
|
|
1602
|
+
(cv::Mat_<double>(3, 1) << tx, ty, tz) +
|
|
1603
|
+
_config.virtualPlaneDepthMeters * camForwardWorld;
|
|
1604
|
+
// Build a 4×4 transform. V15.0c.2 onward only reads
|
|
1605
|
+
// column 1 (normal) and column 3 (origin); columns
|
|
1606
|
+
// 0 and 2 are unused (U/V derived from gravity).
|
|
1607
|
+
cv::Mat T = cv::Mat::eye(4, 4, CV_64F);
|
|
1608
|
+
T.at<double>(0, 1) = normalWorld.at<double>(0);
|
|
1609
|
+
T.at<double>(1, 1) = normalWorld.at<double>(1);
|
|
1610
|
+
T.at<double>(2, 1) = normalWorld.at<double>(2);
|
|
1611
|
+
T.at<double>(0, 3) = originWorld.at<double>(0);
|
|
1612
|
+
T.at<double>(1, 3) = originWorld.at<double>(1);
|
|
1613
|
+
T.at<double>(2, 3) = originWorld.at<double>(2);
|
|
1614
|
+
_planeTransform = T;
|
|
1615
|
+
_firstPlaneTInt = 0.0; // baseline reset for new plane
|
|
1616
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1617
|
+
"[V15.0d-virtual-plane] synthesized at depth=%.2fm "
|
|
1618
|
+
"origin=(%.3f, %.3f, %.3f) normal=(%.3f, %.3f, %.3f) "
|
|
1619
|
+
"(camera-forward × depth in front of camera_pos)",
|
|
1620
|
+
_config.virtualPlaneDepthMeters,
|
|
1621
|
+
originWorld.at<double>(0),
|
|
1622
|
+
originWorld.at<double>(1),
|
|
1623
|
+
originWorld.at<double>(2),
|
|
1624
|
+
normalWorld.at<double>(0),
|
|
1625
|
+
normalWorld.at<double>(1),
|
|
1626
|
+
normalWorld.at<double>(2));
|
|
1627
|
+
}
|
|
1628
|
+
runPlaneProjection = true;
|
|
1629
|
+
}
|
|
1630
|
+
// planeSource == Disabled → runPlaneProjection stays false
|
|
1631
|
+
if (runPlaneProjection) {
|
|
1632
|
+
cv::Mat t_arkit = (cv::Mat_<double>(3, 1) << tx, ty, tz);
|
|
1633
|
+
|
|
1634
|
+
// V15.0c.2 — gravity-aligned U/V axes. Replaces V15.0c's
|
|
1635
|
+
// direct read of plane-local X/Z, which was wrong because
|
|
1636
|
+
// ARKit's ARPlaneAnchor.transform DOES NOT guarantee column-2
|
|
1637
|
+
// points DOWN in world. It is one of two possible vectors
|
|
1638
|
+
// parallel to gravity, and the orientation depends on how
|
|
1639
|
+
// the plane was detected — there's no documented contract.
|
|
1640
|
+
//
|
|
1641
|
+
// Symptom of the V15.0c bug (Ram observed on top-to-bottom
|
|
1642
|
+
// pan): the second-pass content rendered as a tilted/
|
|
1643
|
+
// rotated quadrilateral below the first frame because the
|
|
1644
|
+
// sign of the V axis flipped relative to expected.
|
|
1645
|
+
//
|
|
1646
|
+
// V15.0c.2 fix: derive the in-plane U/V axes deterministically
|
|
1647
|
+
// from world gravity, regardless of how ARKit labelled
|
|
1648
|
+
// the plane's local Z.
|
|
1649
|
+
//
|
|
1650
|
+
// • Surface normal (n): column 1 of plane transform — the
|
|
1651
|
+
// ARKit convention IS documented for this one.
|
|
1652
|
+
// • V axis (canvas-Y, +V = down on canvas):
|
|
1653
|
+
// projection of -world_up onto the plane, normalized.
|
|
1654
|
+
// For a vertical wall plane, this resolves to the
|
|
1655
|
+
// in-plane direction parallel to gravity, pointing DOWN.
|
|
1656
|
+
// • U axis (canvas-X, +U = right when looking AT the wall):
|
|
1657
|
+
// n × V (right-handed).
|
|
1658
|
+
// • Canvas mapping: cU = cx + (P_world − O) · U_axis × ppm
|
|
1659
|
+
// cV = cy + (P_world − O) · V_axis × ppm
|
|
1660
|
+
cv::Mat planeNormal = (cv::Mat_<double>(3, 1) <<
|
|
1661
|
+
_planeTransform.at<double>(0, 1),
|
|
1662
|
+
_planeTransform.at<double>(1, 1),
|
|
1663
|
+
_planeTransform.at<double>(2, 1));
|
|
1664
|
+
const cv::Mat planeOrigin = (cv::Mat_<double>(3, 1) <<
|
|
1665
|
+
_planeTransform.at<double>(0, 3),
|
|
1666
|
+
_planeTransform.at<double>(1, 3),
|
|
1667
|
+
_planeTransform.at<double>(2, 3));
|
|
1668
|
+
|
|
1669
|
+
// V15.0c.2 — ensure plane normal points TOWARD the camera.
|
|
1670
|
+
// ARKit's column 1 is the surface normal but its sign isn't
|
|
1671
|
+
// guaranteed (could point INTO the wall). Flipping ensures
|
|
1672
|
+
// U_axis = n × V_axis follows the right-hand rule correctly:
|
|
1673
|
+
// when the camera is looking at the wall, +U → camera's right.
|
|
1674
|
+
cv::Mat camToPlane = planeOrigin - t_arkit;
|
|
1675
|
+
if (camToPlane.dot(planeNormal) > 0) {
|
|
1676
|
+
// Normal points away from camera — flip so it points
|
|
1677
|
+
// toward the camera (away from the wall surface).
|
|
1678
|
+
planeNormal = -planeNormal;
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
// World up in ARKit is (0, 1, 0). For a vertical plane,
|
|
1682
|
+
// negUp's projection onto the plane points DOWN in the
|
|
1683
|
+
// wall's surface — exactly the canvas-Y "down" direction.
|
|
1684
|
+
const cv::Mat negWorldUp = (cv::Mat_<double>(3, 1) << 0.0, -1.0, 0.0);
|
|
1685
|
+
const double upDotN = negWorldUp.dot(planeNormal);
|
|
1686
|
+
cv::Mat V_axis = negWorldUp - upDotN * planeNormal;
|
|
1687
|
+
double V_axis_norm = cv::norm(V_axis);
|
|
1688
|
+
// Degenerate edge case: plane is HORIZONTAL (normal parallel
|
|
1689
|
+
// to gravity). V_axis collapses to zero — there's no
|
|
1690
|
+
// unambiguous "down" in the plane. Skip plane projection
|
|
1691
|
+
// for this frame and fall through to slit-scan. Should
|
|
1692
|
+
// never happen with planeDetection = .vertical, but guarding
|
|
1693
|
+
// anyway in case the ARKit detector ever returns a wonky
|
|
1694
|
+
// plane (e.g., a tilted shelf labelled vertical).
|
|
1695
|
+
if (V_axis_norm < 1e-6) {
|
|
1696
|
+
if (_engineCallCounter % 30 == 0) {
|
|
1697
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1698
|
+
"[V15.0c.2-plane-horizontal] #%ld plane normal "
|
|
1699
|
+
"parallel to gravity (|V_axis|=%.3e); falling "
|
|
1700
|
+
"back to slit-scan",
|
|
1701
|
+
(long)_engineCallCounter, V_axis_norm);
|
|
1702
|
+
}
|
|
1703
|
+
goto skipPlaneProjection;
|
|
1704
|
+
}
|
|
1705
|
+
V_axis /= V_axis_norm;
|
|
1706
|
+
// U_axis = n × V_axis. Right-handed: when the camera is
|
|
1707
|
+
// looking AT the wall (so -n points away from camera), +U
|
|
1708
|
+
// points to the camera's right.
|
|
1709
|
+
cv::Mat U_axis = planeNormal.cross(V_axis);
|
|
1710
|
+
double U_axis_norm = cv::norm(U_axis);
|
|
1711
|
+
if (U_axis_norm < 1e-6) {
|
|
1712
|
+
// Should be impossible given V_axis ⟂ n, but guard
|
|
1713
|
+
// defensively against numerical edge cases.
|
|
1714
|
+
goto skipPlaneProjection;
|
|
1715
|
+
}
|
|
1716
|
+
U_axis /= U_axis_norm;
|
|
1717
|
+
|
|
1718
|
+
constexpr double kPixelsPerMeter = 1000.0;
|
|
1719
|
+
const double cCenterX = _canvas.cols / 2.0;
|
|
1720
|
+
const double cCenterY = _canvas.rows / 2.0;
|
|
1721
|
+
|
|
1722
|
+
cv::Mat K_inv = _K_compose.inv();
|
|
1723
|
+
|
|
1724
|
+
std::vector<cv::Point2f> camCorners = {
|
|
1725
|
+
cv::Point2f(0, 0),
|
|
1726
|
+
cv::Point2f((float)frameBGR.cols, 0),
|
|
1727
|
+
cv::Point2f((float)frameBGR.cols, (float)frameBGR.rows),
|
|
1728
|
+
cv::Point2f(0, (float)frameBGR.rows)
|
|
1729
|
+
};
|
|
1730
|
+
std::vector<cv::Point2f> canvasCorners;
|
|
1731
|
+
canvasCorners.reserve(4);
|
|
1732
|
+
|
|
1733
|
+
bool degenerate = false;
|
|
1734
|
+
// V15.0c.2 — track the max |t_int| across the 4 corners
|
|
1735
|
+
// for off-plane detection (see fallback check after the
|
|
1736
|
+
// loop).
|
|
1737
|
+
double currentMaxTInt = 0.0;
|
|
1738
|
+
for (const auto &cp : camCorners) {
|
|
1739
|
+
cv::Mat pixHomo = (cv::Mat_<double>(3, 1) << cp.x, cp.y, 1.0);
|
|
1740
|
+
cv::Mat rayCv = K_inv * pixHomo;
|
|
1741
|
+
cv::Mat rayArkit = _M_arkitToCv * rayCv;
|
|
1742
|
+
cv::Mat rayWorld = R_new * rayArkit;
|
|
1743
|
+
// Plane intersection: t_int = ((P0 − O) · n) / (d · n)
|
|
1744
|
+
cv::Mat diff = planeOrigin - t_arkit;
|
|
1745
|
+
const double num = diff.dot(planeNormal);
|
|
1746
|
+
const double denom = rayWorld.dot(planeNormal);
|
|
1747
|
+
if (std::fabs(denom) < 1e-6) { degenerate = true; break; }
|
|
1748
|
+
const double t_int = num / denom;
|
|
1749
|
+
if (t_int < 0.05 || t_int > 50.0) { degenerate = true; break; }
|
|
1750
|
+
if (t_int > currentMaxTInt) currentMaxTInt = t_int;
|
|
1751
|
+
cv::Mat P_world = t_arkit + t_int * rayWorld;
|
|
1752
|
+
// V15.0c.2 — project onto in-plane U/V axes (gravity-
|
|
1753
|
+
// aligned), not raw plane-local X/Z. diffWorld is
|
|
1754
|
+
// P_world − O, the in-plane displacement of this corner
|
|
1755
|
+
// from the plane's origin.
|
|
1756
|
+
cv::Mat diffWorld = P_world - planeOrigin;
|
|
1757
|
+
const double Up = diffWorld.dot(U_axis);
|
|
1758
|
+
const double Vp = diffWorld.dot(V_axis);
|
|
1759
|
+
const double cU = cCenterX + Up * kPixelsPerMeter;
|
|
1760
|
+
// V_axis already points DOWN on the wall (gravity-
|
|
1761
|
+
// aligned), so canvas Y just adds Vp directly.
|
|
1762
|
+
const double cV = cCenterY + Vp * kPixelsPerMeter;
|
|
1763
|
+
canvasCorners.emplace_back((float)cU, (float)cV);
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
// V15.0c.2 — off-plane fallback. Once the camera tilts
|
|
1767
|
+
// past the wall edge (e.g., user pans down past where
|
|
1768
|
+
// the pegboard ends, onto the bench / floor in front of
|
|
1769
|
+
// the wall), the rays still mathematically intersect the
|
|
1770
|
+
// INFINITE wall plane equation, but the actual content
|
|
1771
|
+
// they see is NOT on the plane. The result is a tilted
|
|
1772
|
+
// / wrong-scale parallelogram pasted at the WRONG canvas
|
|
1773
|
+
// position — visually, the user sees the floor / bench
|
|
1774
|
+
// wrongly placed below the first frame.
|
|
1775
|
+
//
|
|
1776
|
+
// Heuristic: track the max corner t_int for the FIRST
|
|
1777
|
+
// successful plane-projected frame (set as baseline).
|
|
1778
|
+
// For subsequent frames, if the max corner t_int exceeds
|
|
1779
|
+
// 3× the baseline, the camera is rotated such that at
|
|
1780
|
+
// least one corner ray is grazing the plane far away —
|
|
1781
|
+
// strong evidence the camera has panned off-plane.
|
|
1782
|
+
// Skip the plane warp this frame; fall through to the
|
|
1783
|
+
// slit-scan path so the capture continues with a known
|
|
1784
|
+
// (less ambitious) algorithm rather than producing a
|
|
1785
|
+
// visibly wrong output.
|
|
1786
|
+
//
|
|
1787
|
+
// Threshold of 3× picked from typical retail aisle
|
|
1788
|
+
// geometry: a phone scan of a 1.5 m fixture face at 1.0 m
|
|
1789
|
+
// depth has corner rays in [0.8, 1.4] m. Tilting past
|
|
1790
|
+
// the fixture edge takes the FAR corner past 4 m before
|
|
1791
|
+
// the eye-line clears. 3× catches this comfortably.
|
|
1792
|
+
constexpr double kOffPlaneMultiplier = 3.0;
|
|
1793
|
+
if (!degenerate
|
|
1794
|
+
&& _firstPlaneTInt > 0.0
|
|
1795
|
+
&& currentMaxTInt > kOffPlaneMultiplier * _firstPlaneTInt) {
|
|
1796
|
+
if (_engineCallCounter % 5 == 0) {
|
|
1797
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1798
|
+
"[V15.0c.2-offplane] #%ld camera off detected "
|
|
1799
|
+
"plane (currentMaxTInt=%.2fm baseline=%.2fm "
|
|
1800
|
+
"ratio=%.2f); falling back to slit-scan",
|
|
1801
|
+
(long)_engineCallCounter,
|
|
1802
|
+
currentMaxTInt, _firstPlaneTInt,
|
|
1803
|
+
currentMaxTInt / std::max(0.001, _firstPlaneTInt));
|
|
1804
|
+
}
|
|
1805
|
+
degenerate = true;
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
if (!degenerate) {
|
|
1809
|
+
// V15.0c.2 — set the off-plane baseline on the FIRST
|
|
1810
|
+
// successful plane-projected frame. All subsequent
|
|
1811
|
+
// frames are compared to this baseline (see kOffPlaneMultiplier
|
|
1812
|
+
// check above). We use the max-corner t_int from THIS
|
|
1813
|
+
// frame as the "what's reasonable" reference.
|
|
1814
|
+
if (_firstPlaneTInt <= 0.0 && currentMaxTInt > 0.0) {
|
|
1815
|
+
_firstPlaneTInt = currentMaxTInt;
|
|
1816
|
+
// V15.0c.4 — fault log so it always shows.
|
|
1817
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1818
|
+
"[V15.0c.2-baseline] off-plane baseline "
|
|
1819
|
+
"_firstPlaneTInt set to %.3fm (max corner t_int "
|
|
1820
|
+
"on first plane-projected frame)",
|
|
1821
|
+
_firstPlaneTInt);
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
cv::Mat H = cv::getPerspectiveTransform(camCorners, canvasCorners);
|
|
1825
|
+
|
|
1826
|
+
cv::Mat warpedCanvas = cv::Mat::zeros(_canvas.size(), CV_8UC3);
|
|
1827
|
+
cv::warpPerspective(frameBGR, warpedCanvas, H,
|
|
1828
|
+
_canvas.size(),
|
|
1829
|
+
cv::INTER_LINEAR,
|
|
1830
|
+
cv::BORDER_CONSTANT,
|
|
1831
|
+
cv::Scalar(0, 0, 0));
|
|
1832
|
+
|
|
1833
|
+
cv::Mat whiteFrame(frameBGR.size(), CV_8UC1, cv::Scalar(255));
|
|
1834
|
+
cv::Mat warpedMask = cv::Mat::zeros(_canvas.size(), CV_8UC1);
|
|
1835
|
+
cv::warpPerspective(whiteFrame, warpedMask, H,
|
|
1836
|
+
_canvas.size(),
|
|
1837
|
+
cv::INTER_NEAREST,
|
|
1838
|
+
cv::BORDER_CONSTANT,
|
|
1839
|
+
cv::Scalar(0));
|
|
1840
|
+
|
|
1841
|
+
// Paint mode (composes with plane mode).
|
|
1842
|
+
cv::Mat canvasMaskZero;
|
|
1843
|
+
cv::compare(_canvasMask, 0, canvasMaskZero, cv::CMP_EQ);
|
|
1844
|
+
cv::Mat paintMaskFresh;
|
|
1845
|
+
cv::bitwise_and(canvasMaskZero, warpedMask, paintMaskFresh);
|
|
1846
|
+
warpedCanvas.copyTo(_canvas, paintMaskFresh);
|
|
1847
|
+
cv::bitwise_or(_canvasMask, paintMaskFresh, _canvasMask);
|
|
1848
|
+
|
|
1849
|
+
if (_config.paintMode == RLISPaintModeFeatherBlend) {
|
|
1850
|
+
cv::Mat canvasMaskNonZero;
|
|
1851
|
+
cv::compare(_canvasMask, 0, canvasMaskNonZero, cv::CMP_NE);
|
|
1852
|
+
cv::Mat overlapMask;
|
|
1853
|
+
cv::bitwise_and(canvasMaskNonZero, warpedMask, overlapMask);
|
|
1854
|
+
if (cv::countNonZero(overlapMask) > 0) {
|
|
1855
|
+
cv::Mat blended;
|
|
1856
|
+
cv::addWeighted(warpedCanvas, 0.3, _canvas, 0.7, 0.0, blended);
|
|
1857
|
+
blended.copyTo(_canvas, overlapMask);
|
|
1858
|
+
}
|
|
1859
|
+
}
|
|
1860
|
+
|
|
1861
|
+
// Update painted-extent telemetry from warpedMask bbox.
|
|
1862
|
+
cv::Mat nz;
|
|
1863
|
+
cv::findNonZero(warpedMask, nz);
|
|
1864
|
+
if (!nz.empty()) {
|
|
1865
|
+
cv::Rect bb = cv::boundingRect(nz);
|
|
1866
|
+
_maxDstY = std::max((int)_maxDstY, bb.y + bb.height);
|
|
1867
|
+
}
|
|
1868
|
+
[tele setValue:@(_maxDstY) forKey:@"paintedExtent"];
|
|
1869
|
+
_accepted += 1;
|
|
1870
|
+
if (_engineCallCounter % 5 == 0 || _engineCallCounter <= 5) {
|
|
1871
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1872
|
+
"[V15.0b-paint] #%ld plane-projected corners=("
|
|
1873
|
+
"%.0f,%.0f)→(%.0f,%.0f)→(%.0f,%.0f)→(%.0f,%.0f) "
|
|
1874
|
+
"_accepted=%ld",
|
|
1875
|
+
(long)_engineCallCounter,
|
|
1876
|
+
canvasCorners[0].x, canvasCorners[0].y,
|
|
1877
|
+
canvasCorners[1].x, canvasCorners[1].y,
|
|
1878
|
+
canvasCorners[2].x, canvasCorners[2].y,
|
|
1879
|
+
canvasCorners[3].x, canvasCorners[3].y,
|
|
1880
|
+
(long)_accepted);
|
|
1881
|
+
}
|
|
1882
|
+
[tele setValue:@(RLISFrameOutcomeAcceptedHigh) forKey:@"outcome"];
|
|
1883
|
+
auto t1 = std::chrono::steady_clock::now();
|
|
1884
|
+
double ms = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count() / 1000.0;
|
|
1885
|
+
[tele setValue:@(ms) forKey:@"processingMs"];
|
|
1886
|
+
return tele;
|
|
1887
|
+
}
|
|
1888
|
+
// Degenerate raycast — fall through to slit-scan path
|
|
1889
|
+
// (also logs that we couldn't use plane this frame).
|
|
1890
|
+
if (_engineCallCounter % 30 == 0) {
|
|
1891
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1892
|
+
"[V15.0b-plane-degenerate] #%ld ray ∥ plane or out of "
|
|
1893
|
+
"range; falling back to slit-scan path",
|
|
1894
|
+
(long)_engineCallCounter);
|
|
1895
|
+
}
|
|
1896
|
+
} else {
|
|
1897
|
+
// V15.0c.3 / V15.0d — plane-projection branch was SKIPPED.
|
|
1898
|
+
// Decoded: tells you WHICH of the two skip reasons fired.
|
|
1899
|
+
//
|
|
1900
|
+
// • planeSource=Disabled → user opted out of plane mode
|
|
1901
|
+
// entirely (or backward-compat upgrade didn't fire
|
|
1902
|
+
// because legacy useDetectedPlane was also NO)
|
|
1903
|
+
// • planeSource=ARKitDetected + planeTransform.empty=1
|
|
1904
|
+
// → ARKit hasn't found an aligned plane yet (filter
|
|
1905
|
+
// in didAdd may have rejected unaligned candidates);
|
|
1906
|
+
// check for `[V15.0b-plane] latched vertical plane`
|
|
1907
|
+
// and `[V15.0b-plane] engine received plane transform`
|
|
1908
|
+
// • planeSource=Virtual + planeTransform.empty=1 →
|
|
1909
|
+
// should NEVER happen (Virtual synthesizes on entry);
|
|
1910
|
+
// would indicate a code path bypassing the synthesis
|
|
1911
|
+
if (_captureFrameCounter <= 3 || _captureFrameCounter % 60 == 0) {
|
|
1912
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1913
|
+
"[V15.0c.3-noplane] capFr=%ld plane-proj branch "
|
|
1914
|
+
"SKIPPED: planeSource=%s planeTransform.empty=%d "
|
|
1915
|
+
"(see [V15-config] line for why)",
|
|
1916
|
+
(long)_captureFrameCounter,
|
|
1917
|
+
planeSrcStr,
|
|
1918
|
+
(int)_planeTransform.empty());
|
|
1919
|
+
}
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
#endif // V15.0e — end of #if 0'd legacy V15.0d inline dispatch
|
|
1923
|
+
|
|
1924
|
+
cv::Mat R_rel = _firstRotationArkit.t() * R_new;
|
|
1925
|
+
// V12.14.10 — UNIFIED clip for both supported modes (see
|
|
1926
|
+
// first-frame branch comment). Both pan around cam +X →
|
|
1927
|
+
// sensor content moves along sensor Y → clip sensor Y to
|
|
1928
|
+
// 70%, full sensor X.
|
|
1929
|
+
int clipW, clipH, srcClipX, srcClipY;
|
|
1930
|
+
clipW = frameBGR.cols;
|
|
1931
|
+
// V15.0c — clipH + srcClipY from _config. Subsequent frames
|
|
1932
|
+
// ALWAYS use the configured sliver clip even if firstFrameFullFrame
|
|
1933
|
+
// was on (that flag only changes the FIRST frame's behaviour).
|
|
1934
|
+
clipH = std::max(1, (int)(frameBGR.rows * _config.kPanAxisFractionRect));
|
|
1935
|
+
srcClipX = 0;
|
|
1936
|
+
switch (_config.sliverPosition) {
|
|
1937
|
+
case RLISSliverPositionTop:
|
|
1938
|
+
srcClipY = 0;
|
|
1939
|
+
break;
|
|
1940
|
+
case RLISSliverPositionBottom:
|
|
1941
|
+
srcClipY = frameBGR.rows - clipH;
|
|
1942
|
+
break;
|
|
1943
|
+
case RLISSliverPositionCenter:
|
|
1944
|
+
default:
|
|
1945
|
+
srcClipY = (frameBGR.rows - clipH) / 2;
|
|
1946
|
+
break;
|
|
1947
|
+
}
|
|
1948
|
+
cv::Mat frameClipped = frameBGR(cv::Rect(srcClipX, srcClipY, clipW, clipH));
|
|
1949
|
+
|
|
1950
|
+
// V12.14.10 — UNIFIED pose projection. Both supported modes
|
|
1951
|
+
// have pan rotation around cam +X axis. alpha > 0 means the
|
|
1952
|
+
// camera rotated such that sensor content moved DOWN (cam-Z
|
|
1953
|
+
// toward cam+Y in OpenCV right-down-forward convention) →
|
|
1954
|
+
// dstY in canvas advances NEGATIVE (content shifts UP / to
|
|
1955
|
+
// smaller Y), matching the existing landscape paste convention.
|
|
1956
|
+
//
|
|
1957
|
+
// For portrait+horizontal-pan the user perceives this as
|
|
1958
|
+
// "look right" or "look left" depending on hold direction;
|
|
1959
|
+
// the canvas-Y growth still represents pan progress, and
|
|
1960
|
+
// the canvas-rotated saved JPEG (writeOutToPath) maps that
|
|
1961
|
+
// progress to the user-perspective horizontal direction.
|
|
1962
|
+
double alpha = std::atan2(R_rel.at<double>(2, 1), R_rel.at<double>(1, 1));
|
|
1963
|
+
int dstX = _firstFrameDstX;
|
|
1964
|
+
int dstY = _firstFrameDstY - (int)std::round(alpha * _focalCompose);
|
|
1965
|
+
|
|
1966
|
+
// V13.0a.4 — FAULT-level os_log (bypasses NSLog burst rate-
|
|
1967
|
+
// limit; throttle still applied to keep Console.app readable).
|
|
1968
|
+
if (_engineCallCounter % 5 == 0 || _engineCallCounter <= 5) {
|
|
1969
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
1970
|
+
"[V13.0a-pose] #%ld alpha_deg=%.3f focal=%.1f dstX=%d dstY=%d "
|
|
1971
|
+
"(firstDstY=%d, deltaDstY=%d)",
|
|
1972
|
+
(long)_engineCallCounter, alpha * 180.0 / M_PI, _focalCompose,
|
|
1973
|
+
dstX, dstY, _firstFrameDstY, dstY - _firstFrameDstY);
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
// V13.0a — REVERTED V12.11 Step 4 + V12.11.1 Item E + V12.14
|
|
1977
|
+
// homography refinement. Restored pose-only paste.
|
|
1978
|
+
//
|
|
1979
|
+
// Why reverted: the ORB+RANSAC homography correction (V12.11)
|
|
1980
|
+
// and the full warpPerspective paste (V12.11.1) introduced
|
|
1981
|
+
// chronic chevron / banding / frame-stacking artifacts under
|
|
1982
|
+
// realistic capture conditions (low overlap, low texture,
|
|
1983
|
+
// fast pan). V12.14's 3-tier ladder filtered the worst
|
|
1984
|
+
// homographies but couldn't fully tame them — MED tier
|
|
1985
|
+
// translation-only correction strips rotation/scale and
|
|
1986
|
+
// produces the "frames pull back" pattern in field traces.
|
|
1987
|
+
//
|
|
1988
|
+
// Pose-only paste is what production camera apps (iOS Camera
|
|
1989
|
+
// Pano, Samsung native pano) use as their PRIMARY tracking;
|
|
1990
|
+
// they layer lightweight 1D edge correlation on top for
|
|
1991
|
+
// sub-pixel perpendicular drift correction. V13.0b will
|
|
1992
|
+
// restore that lightweight refinement (1D NCC column
|
|
1993
|
+
// correlation, ±100 px search) on top of this pose-only
|
|
1994
|
+
// baseline.
|
|
1995
|
+
//
|
|
1996
|
+
// For now (V13.0a): rotation is correct from ARKit pose;
|
|
1997
|
+
// perpendicular drift may be visible (~5–10 px on short
|
|
1998
|
+
// pans) until V13.0b lands. Translation along the pan
|
|
1999
|
+
// axis comes from `alpha * _focalCompose` above.
|
|
2000
|
+
|
|
2001
|
+
// V12.11 Step D — reverse-direction detection.
|
|
2002
|
+
//
|
|
2003
|
+
// After homography correction, check whether the new paste
|
|
2004
|
+
// position has REGRESSED from the running max along the
|
|
2005
|
+
// pan axis by more than `kReverseStopPx`. If so, the
|
|
2006
|
+
// operator has reversed direction (intentionally or
|
|
2007
|
+
// accidentally) — skip the paste, emit
|
|
2008
|
+
// `RejectedReverseDirection`, and let the host auto-finalise.
|
|
2009
|
+
// The high-water-mark (max) is what we want to ship as the
|
|
2010
|
+
// pano; back-tracking would only damage it under
|
|
2011
|
+
// first-painted-wins.
|
|
2012
|
+
//
|
|
2013
|
+
// Threshold: 150 px ≈ 4° of pan at the typical iPhone focal —
|
|
2014
|
+
// comfortably above normal alignment-correction wobble.
|
|
2015
|
+
// V12.14.10 — UNIFIED running-max for both supported modes.
|
|
2016
|
+
// Both pan along canvas Y, so _maxDstY is the leading-edge
|
|
2017
|
+
// tracker in both. _maxDstX stays at 0 (perp axis static).
|
|
2018
|
+
// Reverse-direction detection: when dstY regresses
|
|
2019
|
+
// > kReverseStopPx (150 px ≈ 4° at iPhone focal), auto-stop
|
|
2020
|
+
// the engine — operator has clearly reversed pan direction.
|
|
2021
|
+
constexpr int kReverseStopPx = 150;
|
|
2022
|
+
if (dstY < _maxDstY - kReverseStopPx) {
|
|
2023
|
+
NSLog(@"[V12.11-reverse] %s stop: dstY=%d max=%d (regressed %d px)",
|
|
2024
|
+
_isLandscape ? "landscape" : "portrait",
|
|
2025
|
+
dstY, _maxDstY, _maxDstY - dstY);
|
|
2026
|
+
[tele setValue:@(RLISFrameOutcomeRejectedReverseDirection) forKey:@"outcome"];
|
|
2027
|
+
return tele;
|
|
2028
|
+
}
|
|
2029
|
+
|
|
2030
|
+
// V13.0b — minimum-Δ accept gate. Slow handheld pans
|
|
2031
|
+
// currently produce 1–6 px slivers per accept (Ram's
|
|
2032
|
+
// V13.0a.4 trace showed ~2–3 px typical), creating ~270
|
|
2033
|
+
// zig-zag boundaries in 663 px of pan growth — the high-
|
|
2034
|
+
// frequency wobble pattern the eye reads as "compressed
|
|
2035
|
+
// vertical features" / TV-stand-looks-shorter.
|
|
2036
|
+
//
|
|
2037
|
+
// Throttle accepts to require at least kMinAcceptDeltaPx
|
|
2038
|
+
// of pan-axis advance from the last accepted frame. Per-
|
|
2039
|
+
// accept sliver becomes ~50 px tall instead of ~2 px,
|
|
2040
|
+
// dropping zig-zag boundary count ~25× over the same pan
|
|
2041
|
+
// extent. Same wobble magnitude per boundary, but spread
|
|
2042
|
+
// across far fewer transitions = visually much smoother.
|
|
2043
|
+
//
|
|
2044
|
+
// Mirrors how production camera apps gate slit-scan accepts
|
|
2045
|
+
// by motion-distance threshold (iOS Camera Pano, Samsung
|
|
2046
|
+
// native pano). No new outcome enum — reuse SkippedTooClose
|
|
2047
|
+
// since the gate's intent matches: "frame too close to
|
|
2048
|
+
// previous accept to contribute meaningfully".
|
|
2049
|
+
// V15 — accept gate is config-driven. When _config.kMinAcceptDeltaPx
|
|
2050
|
+
// is 0 (slitscan-rotate / slitscan-both defaults), the gate is
|
|
2051
|
+
// effectively disabled — we accept on every frame the engine isn't
|
|
2052
|
+
// already busy with. When 50 (V13.0g/V14.0a default), throttles
|
|
2053
|
+
// accepts to one per 50 px of pan-axis advance to reduce zig-zag
|
|
2054
|
+
// boundary density. Settings UI exposes this for testing.
|
|
2055
|
+
const int kMinAcceptDeltaPx = (int)_config.kMinAcceptDeltaPx;
|
|
2056
|
+
const int panDelta = dstY - _maxDstY;
|
|
2057
|
+
if (kMinAcceptDeltaPx > 0 && panDelta < kMinAcceptDeltaPx) {
|
|
2058
|
+
// V13.0b — diagnostic gate-fire log (throttled).
|
|
2059
|
+
if (_engineCallCounter % 5 == 0) {
|
|
2060
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
2061
|
+
"[V13.0b-gate] #%ld dstY=%d _maxDstY=%d panDelta=%d "
|
|
2062
|
+
"< kMinAcceptDeltaPx=%d -> SkippedTooClose",
|
|
2063
|
+
(long)_engineCallCounter, dstY, _maxDstY, panDelta,
|
|
2064
|
+
kMinAcceptDeltaPx);
|
|
2065
|
+
}
|
|
2066
|
+
[tele setValue:@(RLISFrameOutcomeSkippedTooClose) forKey:@"outcome"];
|
|
2067
|
+
auto t1 = std::chrono::steady_clock::now();
|
|
2068
|
+
double ms = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
2069
|
+
t1 - t0).count() / 1000.0;
|
|
2070
|
+
[tele setValue:@(ms) forKey:@"processingMs"];
|
|
2071
|
+
return tele;
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
// V13.0f — TWO-LAYER alignment: ORB triangulation (depth-aware
|
|
2075
|
+
// big shifts) + 2D NCC (visual fine refinement) on top of
|
|
2076
|
+
// pose-only. Paint is first-painted-wins on the full clipH.
|
|
2077
|
+
//
|
|
2078
|
+
// ─── ARCHITECTURE ────────────────────────────────────────────
|
|
2079
|
+
// Past the V13.0b 50 px gate; this slit will be accepted. Three
|
|
2080
|
+
// independent error sources move scene pixels off their pose-
|
|
2081
|
+
// predicted canvas position:
|
|
2082
|
+
//
|
|
2083
|
+
// (1) Translation parallax — the camera doesn't pivot in
|
|
2084
|
+
// place; ARKit logs ~30–40 cm of camera translation per
|
|
2085
|
+
// pan. At 1–3 m scene depth that's tens to a hundred px
|
|
2086
|
+
// of perpendicular drift. This is depth-dependent and
|
|
2087
|
+
// hence Z-aware.
|
|
2088
|
+
// (2) ARKit pose drift / sensor wobble — sub-degree pose
|
|
2089
|
+
// errors that show up as a few-px shift in image plane.
|
|
2090
|
+
// Depth-INdependent (uniform across the slit).
|
|
2091
|
+
// (3) Multi-depth disagreement — close objects have larger
|
|
2092
|
+
// parallax than far objects; a single canvas-paste shift
|
|
2093
|
+
// can't satisfy both. Architectural limit; out of scope
|
|
2094
|
+
// for V13.0f (would need per-pixel depth = LiDAR).
|
|
2095
|
+
//
|
|
2096
|
+
// V13.0e fixed (1) only — triangulation handles the depth-aware
|
|
2097
|
+
// big shifts. But V13.0e bad-Z cases (close textured features
|
|
2098
|
+
// bias median Z) over-corrected, and the test exposed (2) as
|
|
2099
|
+
// visible hard seams in the rotation case. V13.0f layers NCC
|
|
2100
|
+
// on top of triangulation so:
|
|
2101
|
+
//
|
|
2102
|
+
// tri handles parallax (when Z is plausible)
|
|
2103
|
+
// NCC handles residual visual mismatch (~ ±30 px)
|
|
2104
|
+
//
|
|
2105
|
+
// ─── TIGHTER TRIANGULATION ───────────────────────────────────
|
|
2106
|
+
// • Z range filter [0.5m, 10m] (was [0.1, 20m]). V13.0e test
|
|
2107
|
+
// showed median Z = 0.27 m — real, but biased by
|
|
2108
|
+
// close-textured peg-hook features. Rejecting Z < 0.5 m
|
|
2109
|
+
// biases median onto the dominant mid-scene plane (1–3 m).
|
|
2110
|
+
// • Cap |triDx|, |triDy| at ±50 px (was ±100). Smaller cap
|
|
2111
|
+
// keeps the post-tri position close enough that NCC's ±30
|
|
2112
|
+
// search can find the visually correct match if tri was off.
|
|
2113
|
+
//
|
|
2114
|
+
// ─── 2D NCC FINE-ALIGNMENT ──────────────────────────────────
|
|
2115
|
+
// V13.0d's NCC bug: the X-search width clamped to canvas.cols
|
|
2116
|
+
// left zero slide room (`searchW > clipW` was 1440 > 1440).
|
|
2117
|
+
// V13.0f fixes this by NARROWING the SOURCE template by 30 px
|
|
2118
|
+
// on each side (sourceW = clipW − 60). matchTemplate over a
|
|
2119
|
+
// 1440 × ~160 search region against a 1380 × 100 source has
|
|
2120
|
+
// 60 × 60 of slide room — enough to recover ±30 px in X and Y.
|
|
2121
|
+
//
|
|
2122
|
+
// Source: top kNccSourceHeight = 100 px of clipped slit, X-
|
|
2123
|
+
// inset by kNccSourceXInset = 30. Search: canvas region
|
|
2124
|
+
// around (dstX, dstY) with ±kNccSearchMargin = 30 px slop.
|
|
2125
|
+
// NCC confidence ≥ 0.6 to apply, else skip.
|
|
2126
|
+
//
|
|
2127
|
+
// ─── FORWARD-ONLY Y CLAMP (after both layers) ───────────────
|
|
2128
|
+
// The combined tri + NCC correction can pull dstY below
|
|
2129
|
+
// _maxDstY (V12.14 frame-stacking risk). Clamp dstY ≥
|
|
2130
|
+
// _maxDstY at the very end — applied ONCE on the final value,
|
|
2131
|
+
// not inside each layer.
|
|
2132
|
+
//
|
|
2133
|
+
// ─── FIRST-PAINTED-WINS PAINT ───────────────────────────────
|
|
2134
|
+
// No feather blend. V13.0d showed feather + imperfect
|
|
2135
|
+
// alignment = ghosting in detail-rich regions. With tri + NCC
|
|
2136
|
+
// doing the alignment work, residuals are sub-px to a few px
|
|
2137
|
+
// — hard seams in those regions read as thin lines rather
|
|
2138
|
+
// than blurred edges.
|
|
2139
|
+
const int poseDstY = dstY;
|
|
2140
|
+
const int poseDstX = dstX;
|
|
2141
|
+
|
|
2142
|
+
// ── V15 LAYER 0: 1D NCC perpendicular-axis wobble correction ──
|
|
2143
|
+
// For slitscan-rotate (rotation-only pan), the dominant residual
|
|
2144
|
+
// after pose-only paste is small horizontal jitter in canvas-X
|
|
2145
|
+
// from handheld wobble (rotation around an imperfectly-stable
|
|
2146
|
+
// axis introduces small perpendicular displacement frame-to-
|
|
2147
|
+
// frame). A narrow 1D NCC search in canvas-X (Y fixed at pose
|
|
2148
|
+
// value) recovers the sub-pixel offset.
|
|
2149
|
+
//
|
|
2150
|
+
// Source: a thin strip from the top of the new clipped slit
|
|
2151
|
+
// (the part that overlaps already-painted canvas). Search:
|
|
2152
|
+
// canvas region around (dstX, poseDstY) with ±kRadius in X,
|
|
2153
|
+
// narrow Y window. Templates correlate via TM_CCOEFF_NORMED;
|
|
2154
|
+
// confidence ≥ 0.6 to apply, else skip.
|
|
2155
|
+
//
|
|
2156
|
+
// Independent of triangulation/RANSAC stages — those handle
|
|
2157
|
+
// translation parallax (depth-dependent shift, big magnitudes).
|
|
2158
|
+
// 1D NCC handles depth-INDEPENDENT wobble (small, perpendicular).
|
|
2159
|
+
// Both can run simultaneously if the user enables them.
|
|
2160
|
+
int ncc1dDx = 0;
|
|
2161
|
+
double ncc1dConfidence = 0.0;
|
|
2162
|
+
bool ncc1dApplied = false;
|
|
2163
|
+
|
|
2164
|
+
if (_config.enable1dNcc && _hasPrevAccept && _accepted >= 1) {
|
|
2165
|
+
const int kRadius = std::max(5, std::min(60,
|
|
2166
|
+
(int)_config.nccSearchRadius1d));
|
|
2167
|
+
const int kSourceHeight = 60; // shallow strip
|
|
2168
|
+
const int kSourceXInset = kRadius; // leave slide room
|
|
2169
|
+
const int sourceW = clipW - 2 * kSourceXInset;
|
|
2170
|
+
|
|
2171
|
+
if (sourceW > 0
|
|
2172
|
+
&& srcClipY >= 0
|
|
2173
|
+
&& srcClipY + kSourceHeight <= frameBGR.rows
|
|
2174
|
+
&& srcClipX + kSourceXInset + sourceW <= frameBGR.cols
|
|
2175
|
+
&& dstY >= 0
|
|
2176
|
+
&& dstY + kSourceHeight <= _canvas.rows) {
|
|
2177
|
+
|
|
2178
|
+
cv::Mat sourceRegion = frameBGR(cv::Rect(
|
|
2179
|
+
srcClipX + kSourceXInset, srcClipY,
|
|
2180
|
+
sourceW, kSourceHeight));
|
|
2181
|
+
|
|
2182
|
+
int searchLeft = std::max(0, dstX + kSourceXInset - kRadius);
|
|
2183
|
+
int searchRight = std::min((int)_canvas.cols,
|
|
2184
|
+
dstX + kSourceXInset + sourceW + kRadius);
|
|
2185
|
+
int searchTop = std::max(0, dstY);
|
|
2186
|
+
int searchBottom = std::min((int)_canvas.rows,
|
|
2187
|
+
dstY + kSourceHeight);
|
|
2188
|
+
const int searchW = searchRight - searchLeft;
|
|
2189
|
+
const int searchH = searchBottom - searchTop;
|
|
2190
|
+
|
|
2191
|
+
if (searchW >= sourceW && searchH >= kSourceHeight) {
|
|
2192
|
+
cv::Mat searchMaskRoi = _canvasMask(cv::Rect(
|
|
2193
|
+
searchLeft, searchTop, searchW, searchH));
|
|
2194
|
+
if (cv::countNonZero(searchMaskRoi) > 0) {
|
|
2195
|
+
cv::Mat searchRegion = _canvas(cv::Rect(
|
|
2196
|
+
searchLeft, searchTop, searchW, searchH));
|
|
2197
|
+
|
|
2198
|
+
cv::Mat sg, rg;
|
|
2199
|
+
cv::cvtColor(sourceRegion, sg, cv::COLOR_BGR2GRAY);
|
|
2200
|
+
cv::cvtColor(searchRegion, rg, cv::COLOR_BGR2GRAY);
|
|
2201
|
+
|
|
2202
|
+
cv::Mat result;
|
|
2203
|
+
cv::matchTemplate(rg, sg, result,
|
|
2204
|
+
cv::TM_CCOEFF_NORMED);
|
|
2205
|
+
|
|
2206
|
+
double rmin, rmax;
|
|
2207
|
+
cv::Point lmin, lmax;
|
|
2208
|
+
cv::minMaxLoc(result, &rmin, &rmax, &lmin, &lmax);
|
|
2209
|
+
ncc1dConfidence = rmax;
|
|
2210
|
+
|
|
2211
|
+
if (ncc1dConfidence >= 0.6) {
|
|
2212
|
+
const int matchX = searchLeft + lmax.x;
|
|
2213
|
+
int rawDx = matchX - (dstX + kSourceXInset);
|
|
2214
|
+
if (rawDx > kRadius) rawDx = kRadius;
|
|
2215
|
+
if (rawDx < -kRadius) rawDx = -kRadius;
|
|
2216
|
+
ncc1dDx = rawDx;
|
|
2217
|
+
dstX += ncc1dDx;
|
|
2218
|
+
ncc1dApplied = true;
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
if (_engineCallCounter % 5 == 0 || _engineCallCounter <= 5) {
|
|
2226
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
2227
|
+
"[V15-1dncc] #%ld dx=%+d conf=%.3f applied=%d "
|
|
2228
|
+
"(poseDstX=%d -> dstX=%d)",
|
|
2229
|
+
(long)_engineCallCounter, ncc1dDx, ncc1dConfidence,
|
|
2230
|
+
(int)ncc1dApplied, poseDstX, dstX);
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
// V15 — ORB detect runs only if any feature-using stage is
|
|
2234
|
+
// enabled. Skips ~5 ms per accept when slitscan-rotate /
|
|
2235
|
+
// slitscan-both-default (pose-only + 1D NCC + feather) don't
|
|
2236
|
+
// need it.
|
|
2237
|
+
const bool needFeatures =
|
|
2238
|
+
_config.enableTriangulation ||
|
|
2239
|
+
_config.enable2dNcc ||
|
|
2240
|
+
_config.enableRansacHomography;
|
|
2241
|
+
|
|
2242
|
+
std::vector<cv::KeyPoint> curKeypoints;
|
|
2243
|
+
cv::Mat curDescriptors;
|
|
2244
|
+
if (needFeatures) {
|
|
2245
|
+
cv::Mat curGray;
|
|
2246
|
+
cv::cvtColor(frameBGR, curGray, cv::COLOR_BGR2GRAY);
|
|
2247
|
+
_orbDetector->detectAndCompute(curGray, cv::noArray(),
|
|
2248
|
+
curKeypoints, curDescriptors);
|
|
2249
|
+
}
|
|
2250
|
+
|
|
2251
|
+
// ── LAYER 1: triangulation-based parallax correction ────────
|
|
2252
|
+
// V13.0g: triDx/triDy report the ACCUMULATED total (rounded
|
|
2253
|
+
// _accumTriCorrectionX/Y) actually applied to dstX/Y.
|
|
2254
|
+
// triIncX/triIncY report this accept's incremental contribution
|
|
2255
|
+
// to that accumulator — useful for spotting bad-Z bursts.
|
|
2256
|
+
int triMatches = 0;
|
|
2257
|
+
double medianZ = 0.0;
|
|
2258
|
+
int triDx = 0;
|
|
2259
|
+
int triDy = 0;
|
|
2260
|
+
int triIncX = 0;
|
|
2261
|
+
int triIncY = 0;
|
|
2262
|
+
bool triApplied = false;
|
|
2263
|
+
|
|
2264
|
+
// V14.0a — prevPts/curPts declared at outer scope (was inside
|
|
2265
|
+
// the triangulation if-block in V13.0g) so V14.0a's RANSAC
|
|
2266
|
+
// homography pass below can reuse them. Empty when no matches
|
|
2267
|
+
// were computed (e.g., first accept, or _hasPrevAccept=false).
|
|
2268
|
+
std::vector<cv::Point2d> prevPts, curPts;
|
|
2269
|
+
|
|
2270
|
+
// V15 — feature matching runs whenever ORB is detected. The
|
|
2271
|
+
// resulting prevPts/curPts are reused by triangulation (this
|
|
2272
|
+
// block), 2D NCC (V13.0g code, gated on _config.enable2dNcc),
|
|
2273
|
+
// and RANSAC homography (V14.0a code, gated on
|
|
2274
|
+
// _config.enableRansacHomography). Triangulation logic itself
|
|
2275
|
+
// is gated on _config.enableTriangulation inside.
|
|
2276
|
+
if (needFeatures
|
|
2277
|
+
&& _hasPrevAccept
|
|
2278
|
+
&& !curDescriptors.empty()
|
|
2279
|
+
&& !_prevDescriptors.empty()
|
|
2280
|
+
&& _prevKeypoints.size() >= 8
|
|
2281
|
+
&& curKeypoints.size() >= 8) {
|
|
2282
|
+
|
|
2283
|
+
cv::BFMatcher matcher(cv::NORM_HAMMING);
|
|
2284
|
+
std::vector<std::vector<cv::DMatch>> knnMatches;
|
|
2285
|
+
matcher.knnMatch(_prevDescriptors, curDescriptors, knnMatches, 2);
|
|
2286
|
+
|
|
2287
|
+
// Lowe ratio test: keep matches where the best match's
|
|
2288
|
+
// descriptor distance is < 0.7 × second-best's, weeding
|
|
2289
|
+
// out ambiguous repeated-texture pairs.
|
|
2290
|
+
// V14.0a — prevPts/curPts moved to outer scope (just before
|
|
2291
|
+
// the if-block) so V14.0a's RANSAC homography pass below
|
|
2292
|
+
// can use them; declarations no longer here.
|
|
2293
|
+
prevPts.reserve(knnMatches.size());
|
|
2294
|
+
curPts.reserve(knnMatches.size());
|
|
2295
|
+
constexpr float kLoweRatio = 0.7f;
|
|
2296
|
+
for (const auto &m : knnMatches) {
|
|
2297
|
+
if (m.size() == 2 && m[0].distance < kLoweRatio * m[1].distance) {
|
|
2298
|
+
prevPts.emplace_back(
|
|
2299
|
+
_prevKeypoints[m[0].queryIdx].pt.x,
|
|
2300
|
+
_prevKeypoints[m[0].queryIdx].pt.y);
|
|
2301
|
+
curPts.emplace_back(
|
|
2302
|
+
curKeypoints[m[0].trainIdx].pt.x,
|
|
2303
|
+
curKeypoints[m[0].trainIdx].pt.y);
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
// V15 — triangulation algorithm gated on enableTriangulation.
|
|
2308
|
+
// ORB matches above (prevPts/curPts) are computed regardless
|
|
2309
|
+
// because they're shared with V14.0a RANSAC homography below.
|
|
2310
|
+
if (_config.enableTriangulation && prevPts.size() >= 8) {
|
|
2311
|
+
// Build cv-frame projection matrices. Pose convention:
|
|
2312
|
+
// R_arkit is camera-to-world in arkit coords.
|
|
2313
|
+
// R_cv = M × R_arkit × M^T (M = diag(1,-1,-1)).
|
|
2314
|
+
// t_cv = M × t_arkit (camera origin in world cv).
|
|
2315
|
+
// World-to-camera projection: K × [R_cv^T | -R_cv^T × t_cv].
|
|
2316
|
+
cv::Mat R0_cv = _M_arkitToCv * _prevRotationArkit * _M_arkitToCv;
|
|
2317
|
+
cv::Mat R1_cv = _M_arkitToCv * R_new * _M_arkitToCv;
|
|
2318
|
+
cv::Mat t0_cv = _M_arkitToCv * _prevTranslationArkit;
|
|
2319
|
+
cv::Mat t1_cv = _M_arkitToCv * (cv::Mat_<double>(3, 1) << tx, ty, tz);
|
|
2320
|
+
|
|
2321
|
+
cv::Mat T0(3, 4, CV_64F);
|
|
2322
|
+
cv::Mat T1(3, 4, CV_64F);
|
|
2323
|
+
{
|
|
2324
|
+
cv::Mat R0t = R0_cv.t();
|
|
2325
|
+
cv::Mat origin0 = -R0t * t0_cv;
|
|
2326
|
+
R0t.copyTo(T0(cv::Rect(0, 0, 3, 3)));
|
|
2327
|
+
origin0.copyTo(T0(cv::Rect(3, 0, 1, 3)));
|
|
2328
|
+
|
|
2329
|
+
cv::Mat R1t = R1_cv.t();
|
|
2330
|
+
cv::Mat origin1 = -R1t * t1_cv;
|
|
2331
|
+
R1t.copyTo(T1(cv::Rect(0, 0, 3, 3)));
|
|
2332
|
+
origin1.copyTo(T1(cv::Rect(3, 0, 1, 3)));
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
cv::Mat P0 = _K_compose * T0;
|
|
2336
|
+
cv::Mat P1 = _K_compose * T1;
|
|
2337
|
+
|
|
2338
|
+
// 2xN pixel-coord matrices for each camera.
|
|
2339
|
+
cv::Mat prevMat(2, (int)prevPts.size(), CV_64F);
|
|
2340
|
+
cv::Mat curMat(2, (int)curPts.size(), CV_64F);
|
|
2341
|
+
for (int i = 0; i < (int)prevPts.size(); i++) {
|
|
2342
|
+
prevMat.at<double>(0, i) = prevPts[i].x;
|
|
2343
|
+
prevMat.at<double>(1, i) = prevPts[i].y;
|
|
2344
|
+
curMat.at<double>(0, i) = curPts[i].x;
|
|
2345
|
+
curMat.at<double>(1, i) = curPts[i].y;
|
|
2346
|
+
}
|
|
2347
|
+
|
|
2348
|
+
cv::Mat pts4D;
|
|
2349
|
+
cv::triangulatePoints(P0, P1, prevMat, curMat, pts4D);
|
|
2350
|
+
|
|
2351
|
+
// Compute Z (depth) in CURRENT camera frame for each
|
|
2352
|
+
// triangulated point. V13.0f tightens the filter to
|
|
2353
|
+
// [0.5m, 10m] (was [0.1, 20]) — V13.0e test showed
|
|
2354
|
+
// close textured features (peg hooks at ~30 cm) biased
|
|
2355
|
+
// the median. Rejecting Z < 0.5 m biases the median
|
|
2356
|
+
// onto the dominant mid-scene plane.
|
|
2357
|
+
std::vector<double> zs;
|
|
2358
|
+
zs.reserve(pts4D.cols);
|
|
2359
|
+
cv::Mat R1_cv_T = R1_cv.t();
|
|
2360
|
+
for (int i = 0; i < pts4D.cols; i++) {
|
|
2361
|
+
double w = pts4D.at<double>(3, i);
|
|
2362
|
+
if (std::fabs(w) < 1e-9) continue;
|
|
2363
|
+
cv::Mat Pworld = (cv::Mat_<double>(3, 1) <<
|
|
2364
|
+
pts4D.at<double>(0, i) / w,
|
|
2365
|
+
pts4D.at<double>(1, i) / w,
|
|
2366
|
+
pts4D.at<double>(2, i) / w);
|
|
2367
|
+
cv::Mat Pcam = R1_cv_T * (Pworld - t1_cv);
|
|
2368
|
+
double Z = Pcam.at<double>(2);
|
|
2369
|
+
if (Z > 0.5 && Z < 10.0) {
|
|
2370
|
+
zs.push_back(Z);
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
triMatches = (int)zs.size();
|
|
2374
|
+
|
|
2375
|
+
if (zs.size() >= 5) {
|
|
2376
|
+
std::sort(zs.begin(), zs.end());
|
|
2377
|
+
medianZ = zs[zs.size() / 2];
|
|
2378
|
+
|
|
2379
|
+
// V13.0g — INCREMENTAL Δt since previous accept,
|
|
2380
|
+
// NOT cumulative since first frame. V13.0e/f's
|
|
2381
|
+
// cumulative-from-first formulation produced
|
|
2382
|
+
// corrections of hundreds of px at typical pan
|
|
2383
|
+
// motion (Δt cumulative ≈ 40 cm × focal / Z), and
|
|
2384
|
+
// the cap clipped most of them — leaving severe
|
|
2385
|
+
// misalignment between adjacent slits in the late
|
|
2386
|
+
// half of every pan. Per-accept Δt is small
|
|
2387
|
+
// (~4 cm); per-accept correction is small (~30–100
|
|
2388
|
+
// px); we accumulate the increments in
|
|
2389
|
+
// _accumTriCorrectionX/Y to recover the cumulative
|
|
2390
|
+
// total without a hard cap on that total.
|
|
2391
|
+
cv::Mat dt_world_cv = _M_arkitToCv *
|
|
2392
|
+
((cv::Mat_<double>(3, 1) << tx, ty, tz) - _prevTranslationArkit);
|
|
2393
|
+
cv::Mat dt_cam_cv = R1_cv_T * dt_world_cv;
|
|
2394
|
+
|
|
2395
|
+
double rawTriDxInc = +_focalCompose * dt_cam_cv.at<double>(0) / medianZ;
|
|
2396
|
+
double rawTriDyInc = +_focalCompose * dt_cam_cv.at<double>(1) / medianZ;
|
|
2397
|
+
|
|
2398
|
+
// V13.0g per-accept cap = ±80. Bigger than V13.0f's
|
|
2399
|
+
// ±50 because the INCREMENT lands at typical per-
|
|
2400
|
+
// accept parallax magnitude (Δt ~4 cm × focal / Z ≈
|
|
2401
|
+
// 30–100 px for Z ∈ [0.5, 1.5] m). Bad-Z spikes on
|
|
2402
|
+
// a single accept contribute up to ±80 to the
|
|
2403
|
+
// accumulator; subsequent good-Z accepts continue
|
|
2404
|
+
// with their correct magnitudes (no global
|
|
2405
|
+
// rescaling like V13.0e/f cumulative formulas).
|
|
2406
|
+
constexpr double kMaxTriIncCorrection = 80.0;
|
|
2407
|
+
if (std::fabs(rawTriDxInc) > kMaxTriIncCorrection) {
|
|
2408
|
+
rawTriDxInc = (rawTriDxInc > 0 ? 1.0 : -1.0) * kMaxTriIncCorrection;
|
|
2409
|
+
}
|
|
2410
|
+
if (std::fabs(rawTriDyInc) > kMaxTriIncCorrection) {
|
|
2411
|
+
rawTriDyInc = (rawTriDyInc > 0 ? 1.0 : -1.0) * kMaxTriIncCorrection;
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
// Per-accept increment (ints, for diagnostics).
|
|
2415
|
+
triIncX = (int)std::round(rawTriDxInc);
|
|
2416
|
+
triIncY = (int)std::round(rawTriDyInc);
|
|
2417
|
+
|
|
2418
|
+
// Accumulate. Total correction grows naturally
|
|
2419
|
+
// as accepts process; no hard cap on the running
|
|
2420
|
+
// total. The applied dstX/Y delta is the ROUNDED
|
|
2421
|
+
// accumulator, not just this accept's increment —
|
|
2422
|
+
// pose-only dstX/Y is reset to first-frame anchor
|
|
2423
|
+
// every accept, so we layer the full accumulator.
|
|
2424
|
+
_accumTriCorrectionX += rawTriDxInc;
|
|
2425
|
+
_accumTriCorrectionY += rawTriDyInc;
|
|
2426
|
+
|
|
2427
|
+
triDx = (int)std::round(_accumTriCorrectionX);
|
|
2428
|
+
triDy = (int)std::round(_accumTriCorrectionY);
|
|
2429
|
+
|
|
2430
|
+
// V13.0f: NO inner Y clamp here. The final
|
|
2431
|
+
// forward-only clamp runs once after BOTH layers
|
|
2432
|
+
// (tri + NCC), which is the point where dstY's
|
|
2433
|
+
// value is actually committed.
|
|
2434
|
+
dstX += triDx;
|
|
2435
|
+
dstY += triDy;
|
|
2436
|
+
triApplied = true;
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
|
|
2441
|
+
if (_engineCallCounter % 5 == 0 || _engineCallCounter <= 5) {
|
|
2442
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
2443
|
+
"[V13.0g-tri] #%ld matches=%d medianZ=%.2fm inc=%+d,%+d "
|
|
2444
|
+
"accum=%+d,%+d applied=%d (poseDstX=%d poseDstY=%d -> "
|
|
2445
|
+
"dstX=%d dstY=%d)",
|
|
2446
|
+
(long)_engineCallCounter, triMatches, medianZ,
|
|
2447
|
+
triIncX, triIncY, triDx, triDy, (int)triApplied,
|
|
2448
|
+
poseDstX, poseDstY, dstX, dstY);
|
|
2449
|
+
}
|
|
2450
|
+
|
|
2451
|
+
// ── V15 LAYER 1.5: V13.0g-style 2D NCC fine-alignment ────────
|
|
2452
|
+
// Restored from V13.0g (was deleted in V14.0a in favour of
|
|
2453
|
+
// RANSAC homography). Gated on _config.enable2dNcc. When both
|
|
2454
|
+
// 2D NCC and RANSAC are enabled, 2D NCC's translation refines
|
|
2455
|
+
// dstX/dstY first; RANSAC then runs on top — if RANSAC produces
|
|
2456
|
+
// a valid homography it warps the slit non-rigidly, overriding
|
|
2457
|
+
// the rectangular paste path. When RANSAC is disabled, 2D NCC
|
|
2458
|
+
// is the only refinement after triangulation.
|
|
2459
|
+
//
|
|
2460
|
+
// Source: top 100 px of clipped slit, X-inset 30 px each side
|
|
2461
|
+
// (so cv::matchTemplate has slide room). Search: canvas region
|
|
2462
|
+
// around (dstX, dstY) with ±30 px X+Y slop. Confidence ≥ 0.6
|
|
2463
|
+
// to apply, Δx/Δy each capped at ±30.
|
|
2464
|
+
int ncc2dDx = 0, ncc2dDy = 0;
|
|
2465
|
+
double ncc2dConfidence = 0.0;
|
|
2466
|
+
bool ncc2dApplied = false;
|
|
2467
|
+
|
|
2468
|
+
if (_config.enable2dNcc) {
|
|
2469
|
+
constexpr int kNccSourceHeight = 100;
|
|
2470
|
+
// V15.0d — search margin and confidence threshold are
|
|
2471
|
+
// now config-driven (was hardcoded ±12 / 0.75 in V15.0c.4).
|
|
2472
|
+
// Bounded by the search-region computation below; very
|
|
2473
|
+
// small values (< 4) bypass NCC effectively, very large
|
|
2474
|
+
// values (> 30) reintroduce the snap-to-pattern problem.
|
|
2475
|
+
const int kNccSearchMargin =
|
|
2476
|
+
std::clamp((int)_config.nccSearchMargin2d, 4, 60);
|
|
2477
|
+
constexpr int kNccSourceXInset = 30;
|
|
2478
|
+
|
|
2479
|
+
const int sourceW = clipW - 2 * kNccSourceXInset;
|
|
2480
|
+
|
|
2481
|
+
if (sourceW > 0
|
|
2482
|
+
&& srcClipY + kNccSourceHeight <= frameBGR.rows
|
|
2483
|
+
&& srcClipX + kNccSourceXInset + sourceW <= frameBGR.cols) {
|
|
2484
|
+
|
|
2485
|
+
cv::Mat sourceRegion = frameBGR(cv::Rect(
|
|
2486
|
+
srcClipX + kNccSourceXInset,
|
|
2487
|
+
srcClipY,
|
|
2488
|
+
sourceW, kNccSourceHeight));
|
|
2489
|
+
|
|
2490
|
+
const int expectedMatchX = dstX + kNccSourceXInset;
|
|
2491
|
+
const int expectedMatchY = dstY;
|
|
2492
|
+
|
|
2493
|
+
int searchLeft = std::max(0, expectedMatchX - kNccSearchMargin);
|
|
2494
|
+
int searchTop = std::max(0, expectedMatchY - kNccSearchMargin);
|
|
2495
|
+
int searchRight = std::min((int)_canvas.cols,
|
|
2496
|
+
expectedMatchX + sourceW + kNccSearchMargin);
|
|
2497
|
+
int searchBottom = std::min((int)_canvas.rows,
|
|
2498
|
+
expectedMatchY + kNccSourceHeight + kNccSearchMargin);
|
|
2499
|
+
const int searchW = searchRight - searchLeft;
|
|
2500
|
+
const int searchH = searchBottom - searchTop;
|
|
2501
|
+
|
|
2502
|
+
if (searchW >= sourceW && searchH >= kNccSourceHeight) {
|
|
2503
|
+
cv::Mat searchMaskRoi = _canvasMask(cv::Rect(
|
|
2504
|
+
searchLeft, searchTop, searchW, searchH));
|
|
2505
|
+
if (cv::countNonZero(searchMaskRoi) > 0) {
|
|
2506
|
+
cv::Mat searchRegion = _canvas(cv::Rect(
|
|
2507
|
+
searchLeft, searchTop, searchW, searchH));
|
|
2508
|
+
|
|
2509
|
+
cv::Mat sg, rg;
|
|
2510
|
+
cv::cvtColor(sourceRegion, sg, cv::COLOR_BGR2GRAY);
|
|
2511
|
+
cv::cvtColor(searchRegion, rg, cv::COLOR_BGR2GRAY);
|
|
2512
|
+
|
|
2513
|
+
cv::Mat result;
|
|
2514
|
+
cv::matchTemplate(rg, sg, result, cv::TM_CCOEFF_NORMED);
|
|
2515
|
+
|
|
2516
|
+
double rmin, rmax;
|
|
2517
|
+
cv::Point lmin, lmax;
|
|
2518
|
+
cv::minMaxLoc(result, &rmin, &rmax, &lmin, &lmax);
|
|
2519
|
+
ncc2dConfidence = rmax;
|
|
2520
|
+
|
|
2521
|
+
// V15.0d — confidence threshold is config-driven
|
|
2522
|
+
// (was hardcoded 0.6 in V13.0g and 0.75 in V15.0c.4).
|
|
2523
|
+
// Clamped to a sensible range so misconfigured
|
|
2524
|
+
// values can't silently disable NCC entirely.
|
|
2525
|
+
const double kNccConfidenceThreshold =
|
|
2526
|
+
std::clamp(_config.nccConfidenceThreshold2d, 0.30, 0.99);
|
|
2527
|
+
if (ncc2dConfidence >= kNccConfidenceThreshold) {
|
|
2528
|
+
const int matchX = searchLeft + lmax.x;
|
|
2529
|
+
const int matchY = searchTop + lmax.y;
|
|
2530
|
+
int rawDx = matchX - expectedMatchX;
|
|
2531
|
+
int rawDy = matchY - expectedMatchY;
|
|
2532
|
+
// Clamp to the search window first.
|
|
2533
|
+
if (rawDx > kNccSearchMargin) rawDx = kNccSearchMargin;
|
|
2534
|
+
if (rawDx < -kNccSearchMargin) rawDx = -kNccSearchMargin;
|
|
2535
|
+
if (rawDy > kNccSearchMargin) rawDy = kNccSearchMargin;
|
|
2536
|
+
if (rawDy < -kNccSearchMargin) rawDy = -kNccSearchMargin;
|
|
2537
|
+
|
|
2538
|
+
// V15.0d 1C — pan-axis-aware NCC. The
|
|
2539
|
+
// cross-axis (perpendicular to pan) is
|
|
2540
|
+
// already handled by 1D NCC and pose; 2D
|
|
2541
|
+
// NCC's cross-axis search is mostly noise.
|
|
2542
|
+
// Clamp it tighter than the pan-axis when
|
|
2543
|
+
// the user has opted in.
|
|
2544
|
+
// _isLandscape=YES → pan axis = Y → cross = X
|
|
2545
|
+
// _isLandscape=NO → pan axis = X → cross = Y
|
|
2546
|
+
if (_config.enableNcc2dPanAxisLock) {
|
|
2547
|
+
const int crossLock =
|
|
2548
|
+
std::clamp((int)_config.ncc2dCrossAxisLockPx, 0, 30);
|
|
2549
|
+
if (_isLandscape) {
|
|
2550
|
+
if (rawDx > crossLock) rawDx = crossLock;
|
|
2551
|
+
if (rawDx < -crossLock) rawDx = -crossLock;
|
|
2552
|
+
} else {
|
|
2553
|
+
if (rawDy > crossLock) rawDy = crossLock;
|
|
2554
|
+
if (rawDy < -crossLock) rawDy = -crossLock;
|
|
2555
|
+
}
|
|
2556
|
+
}
|
|
2557
|
+
|
|
2558
|
+
// V15.0d 1B — EMA smoothing. Damps single-
|
|
2559
|
+
// frame snaps when the NCC peak jumps to a
|
|
2560
|
+
// different copy of a repeated pattern.
|
|
2561
|
+
// Applied AFTER pan-axis lock so the lock
|
|
2562
|
+
// bounds the input to the EMA.
|
|
2563
|
+
if (_config.enableNcc2dEmaSmoothing) {
|
|
2564
|
+
if (_haveNcc2dEmaHistory) {
|
|
2565
|
+
const double a =
|
|
2566
|
+
std::clamp(_config.ncc2dEmaAlpha, 0.05, 0.95);
|
|
2567
|
+
rawDx = (int)std::round(
|
|
2568
|
+
a * rawDx + (1.0 - a) * _lastNcc2dDxApplied);
|
|
2569
|
+
rawDy = (int)std::round(
|
|
2570
|
+
a * rawDy + (1.0 - a) * _lastNcc2dDyApplied);
|
|
2571
|
+
}
|
|
2572
|
+
_lastNcc2dDxApplied = rawDx;
|
|
2573
|
+
_lastNcc2dDyApplied = rawDy;
|
|
2574
|
+
_haveNcc2dEmaHistory = YES;
|
|
2575
|
+
}
|
|
2576
|
+
|
|
2577
|
+
ncc2dDx = rawDx;
|
|
2578
|
+
ncc2dDy = rawDy;
|
|
2579
|
+
dstX += ncc2dDx;
|
|
2580
|
+
dstY += ncc2dDy;
|
|
2581
|
+
ncc2dApplied = true;
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
}
|
|
2587
|
+
|
|
2588
|
+
if (_engineCallCounter % 5 == 0 || _engineCallCounter <= 5) {
|
|
2589
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
2590
|
+
"[V15-2dncc] #%ld dx=%+d dy=%+d conf=%.3f applied=%d "
|
|
2591
|
+
"(after-tri dstX=%d dstY=%d)",
|
|
2592
|
+
(long)_engineCallCounter, ncc2dDx, ncc2dDy,
|
|
2593
|
+
ncc2dConfidence, (int)ncc2dApplied, dstX, dstY);
|
|
2594
|
+
}
|
|
2595
|
+
|
|
2596
|
+
// ── LAYER 2: V14.0a RANSAC homography refinement ─────────────
|
|
2597
|
+
// V13.0g used 2D NCC to find a single (Δx, Δy) translation that
|
|
2598
|
+
// best aligned the slit's overlap region with the canvas's
|
|
2599
|
+
// existing painted region. NCC's single-translation model
|
|
2600
|
+
// cannot satisfy multi-depth scenes (a door with frame at 1.4 m,
|
|
2601
|
+
// surface at 1.5 m, wall behind at 1.8 m): each depth wants a
|
|
2602
|
+
// different shift, so a single shift visibly mis-aligns at
|
|
2603
|
+
// least one depth → the door-shear visible in V13.0g and
|
|
2604
|
+
// V14.0pre.1 outputs.
|
|
2605
|
+
//
|
|
2606
|
+
// V14.0a feeds the SAME ORB matches V13.0g already computed
|
|
2607
|
+
// for triangulation into RANSAC homography fitting.
|
|
2608
|
+
// Homography is 8-DOF: each matched feature gets its own
|
|
2609
|
+
// implied position via a 3×3 projective transform. The
|
|
2610
|
+
// dominant scene plane fits exactly; off-plane features (with
|
|
2611
|
+
// residual parallax) become RANSAC outliers and are filtered.
|
|
2612
|
+
// The slit is then warped via cv::warpPerspective into canvas
|
|
2613
|
+
// space — producing a non-rectangular footprint that aligns
|
|
2614
|
+
// visually for all features simultaneously.
|
|
2615
|
+
//
|
|
2616
|
+
// Failure mode: degenerate matches (fewer than 8 inliers, or
|
|
2617
|
+
// matches on a single line) → fall back to V13.0g pose+tri-only
|
|
2618
|
+
// rectangular paste. No multi-tier ladder (V12.14's lesson:
|
|
2619
|
+
// confidence tiers compound errors).
|
|
2620
|
+
bool homographyApplied = false;
|
|
2621
|
+
cv::Mat homographyH;
|
|
2622
|
+
int homographyInlierCount = 0;
|
|
2623
|
+
double homographyAvgReproj = 0.0;
|
|
2624
|
+
|
|
2625
|
+
// V15 — RANSAC homography gated on enableRansacHomography.
|
|
2626
|
+
if (_config.enableRansacHomography
|
|
2627
|
+
&& _hasPrevAccept
|
|
2628
|
+
&& prevPts.size() >= 8
|
|
2629
|
+
&& curPts.size() >= 8) {
|
|
2630
|
+
// Build per-match canvas-coord targets: where each prev
|
|
2631
|
+
// feature was actually painted on the canvas.
|
|
2632
|
+
//
|
|
2633
|
+
// prev_pts is in PREV FRAME pixel coords (full sensor).
|
|
2634
|
+
// prev was painted at canvas position
|
|
2635
|
+
// (_prevAcceptDstX + prev_fx − srcClipX,
|
|
2636
|
+
// _prevAcceptDstY + prev_fy − srcClipY)
|
|
2637
|
+
// assuming a rectangular paste at (_prevAcceptDstX,
|
|
2638
|
+
// _prevAcceptDstY). If the previous accept used homography
|
|
2639
|
+
// warp itself, this is approximate within the prev
|
|
2640
|
+
// homography's deviation from translation; RANSAC absorbs
|
|
2641
|
+
// the residual as match-pair noise.
|
|
2642
|
+
std::vector<cv::Point2f> srcPts; // cur frame pixel coords
|
|
2643
|
+
std::vector<cv::Point2f> dstCanvasPts; // canvas pixel coords
|
|
2644
|
+
srcPts.reserve(prevPts.size());
|
|
2645
|
+
dstCanvasPts.reserve(prevPts.size());
|
|
2646
|
+
for (size_t i = 0; i < prevPts.size(); i++) {
|
|
2647
|
+
const double prev_canvas_x =
|
|
2648
|
+
_prevAcceptDstX + prevPts[i].x - srcClipX;
|
|
2649
|
+
const double prev_canvas_y =
|
|
2650
|
+
_prevAcceptDstY + prevPts[i].y - srcClipY;
|
|
2651
|
+
srcPts.emplace_back((float)curPts[i].x, (float)curPts[i].y);
|
|
2652
|
+
dstCanvasPts.emplace_back((float)prev_canvas_x,
|
|
2653
|
+
(float)prev_canvas_y);
|
|
2654
|
+
}
|
|
2655
|
+
|
|
2656
|
+
std::vector<unsigned char> ransacInliers;
|
|
2657
|
+
cv::Mat H = cv::findHomography(srcPts, dstCanvasPts,
|
|
2658
|
+
cv::RANSAC,
|
|
2659
|
+
3.0, // reproj threshold (px)
|
|
2660
|
+
ransacInliers,
|
|
2661
|
+
2000, // max iters
|
|
2662
|
+
0.995); // confidence
|
|
2663
|
+
|
|
2664
|
+
if (!H.empty()) {
|
|
2665
|
+
homographyInlierCount = cv::countNonZero(ransacInliers);
|
|
2666
|
+
|
|
2667
|
+
// Reject degenerate homographies: too few inliers OR
|
|
2668
|
+
// a near-singular matrix (det close to 0, common when
|
|
2669
|
+
// matches lie on a single line — peg-board scenes).
|
|
2670
|
+
const double H_det = cv::determinant(H);
|
|
2671
|
+
const bool degenerate =
|
|
2672
|
+
(homographyInlierCount < 8) ||
|
|
2673
|
+
(std::fabs(H_det) < 1e-6);
|
|
2674
|
+
|
|
2675
|
+
if (!degenerate) {
|
|
2676
|
+
// Compute mean reprojection residual on inliers
|
|
2677
|
+
// (diagnostic: tight homographies → small residual).
|
|
2678
|
+
double sumResid = 0.0;
|
|
2679
|
+
int nResid = 0;
|
|
2680
|
+
for (size_t i = 0; i < srcPts.size(); i++) {
|
|
2681
|
+
if (!ransacInliers[i]) continue;
|
|
2682
|
+
cv::Mat src_h = (cv::Mat_<double>(3, 1) <<
|
|
2683
|
+
srcPts[i].x, srcPts[i].y, 1.0);
|
|
2684
|
+
cv::Mat dst_h = H * src_h;
|
|
2685
|
+
const double w = dst_h.at<double>(2);
|
|
2686
|
+
if (std::fabs(w) < 1e-9) continue;
|
|
2687
|
+
const double dxR = dst_h.at<double>(0) / w
|
|
2688
|
+
- dstCanvasPts[i].x;
|
|
2689
|
+
const double dyR = dst_h.at<double>(1) / w
|
|
2690
|
+
- dstCanvasPts[i].y;
|
|
2691
|
+
sumResid += std::sqrt(dxR * dxR + dyR * dyR);
|
|
2692
|
+
nResid++;
|
|
2693
|
+
}
|
|
2694
|
+
homographyAvgReproj = (nResid > 0)
|
|
2695
|
+
? (sumResid / nResid) : 0.0;
|
|
2696
|
+
|
|
2697
|
+
homographyH = H;
|
|
2698
|
+
homographyApplied = true;
|
|
2699
|
+
}
|
|
2700
|
+
}
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
if (_engineCallCounter % 5 == 0 || _engineCallCounter <= 5) {
|
|
2704
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
2705
|
+
"[V14.0a-ransac] #%ld matches=%zu inliers=%d "
|
|
2706
|
+
"avgReproj=%.2fpx applied=%d",
|
|
2707
|
+
(long)_engineCallCounter,
|
|
2708
|
+
prevPts.size(), homographyInlierCount,
|
|
2709
|
+
homographyAvgReproj, (int)homographyApplied);
|
|
2710
|
+
}
|
|
2711
|
+
|
|
2712
|
+
// ── Forward-only Y guard on running max ─────────────────────
|
|
2713
|
+
// For pose+tri fallback, dstY (post-clamp) is the slit top.
|
|
2714
|
+
// For homography path, the warped slit's actual top edge is
|
|
2715
|
+
// the warpedMask's bounding-box top. Clamp dstY (used by
|
|
2716
|
+
// pose+tri fallback path AND as the diagnostic "pre-warp dstY"
|
|
2717
|
+
// value in the paint log).
|
|
2718
|
+
if (dstY < (int)_maxDstY) {
|
|
2719
|
+
dstY = (int)_maxDstY;
|
|
2720
|
+
}
|
|
2721
|
+
|
|
2722
|
+
// ── V14.0a paint ────────────────────────────────────────────
|
|
2723
|
+
// Two paths: homography warp (preferred) or pose+tri rectangular
|
|
2724
|
+
// paste (fallback). Both apply first-painted-wins masking so
|
|
2725
|
+
// earlier slits' content is preserved.
|
|
2726
|
+
cv::Mat warpedCanvas;
|
|
2727
|
+
cv::Mat warpedCanvasMask;
|
|
2728
|
+
|
|
2729
|
+
if (homographyApplied) {
|
|
2730
|
+
// Warp the clipped slit into canvas-sized output via the
|
|
2731
|
+
// RANSAC homography. Compose H with a translation matrix
|
|
2732
|
+
// so the warp source is the slit-local clipped ROI rather
|
|
2733
|
+
// than the full frame:
|
|
2734
|
+
// pixel_canvas = H_full * pixel_full
|
|
2735
|
+
// pixel_full = pixel_slit + (srcClipX, srcClipY)
|
|
2736
|
+
// ⇒ H_slit = H_full * T(srcClipX, srcClipY)
|
|
2737
|
+
cv::Rect srcSlitRect(srcClipX, srcClipY, clipW, clipH);
|
|
2738
|
+
cv::Mat srcSlit = frameBGR(srcSlitRect);
|
|
2739
|
+
|
|
2740
|
+
cv::Mat T_slit = (cv::Mat_<double>(3, 3) <<
|
|
2741
|
+
1, 0, (double)srcClipX,
|
|
2742
|
+
0, 1, (double)srcClipY,
|
|
2743
|
+
0, 0, 1);
|
|
2744
|
+
cv::Mat H_slit = homographyH * T_slit;
|
|
2745
|
+
|
|
2746
|
+
warpedCanvas = cv::Mat::zeros(_canvas.size(), CV_8UC3);
|
|
2747
|
+
cv::warpPerspective(srcSlit, warpedCanvas, H_slit,
|
|
2748
|
+
_canvas.size(),
|
|
2749
|
+
cv::INTER_LINEAR,
|
|
2750
|
+
cv::BORDER_CONSTANT,
|
|
2751
|
+
cv::Scalar(0, 0, 0));
|
|
2752
|
+
|
|
2753
|
+
cv::Mat whiteSlit(srcSlit.size(), CV_8UC1, cv::Scalar(255));
|
|
2754
|
+
warpedCanvasMask = cv::Mat::zeros(_canvas.size(), CV_8UC1);
|
|
2755
|
+
cv::warpPerspective(whiteSlit, warpedCanvasMask, H_slit,
|
|
2756
|
+
_canvas.size(),
|
|
2757
|
+
cv::INTER_NEAREST,
|
|
2758
|
+
cv::BORDER_CONSTANT,
|
|
2759
|
+
cv::Scalar(0));
|
|
2760
|
+
} else {
|
|
2761
|
+
// Fallback: V13.0g pose+tri-only rectangular paste at
|
|
2762
|
+
// (dstX, dstY). Build a canvas-sized image with just the
|
|
2763
|
+
// slit pasted at its rectangular position.
|
|
2764
|
+
warpedCanvas = cv::Mat::zeros(_canvas.size(), CV_8UC3);
|
|
2765
|
+
warpedCanvasMask = cv::Mat::zeros(_canvas.size(), CV_8UC1);
|
|
2766
|
+
|
|
2767
|
+
cv::Rect dstRoi(dstX, dstY, clipW, clipH);
|
|
2768
|
+
cv::Rect canvasBoundsRect(0, 0, _canvas.cols, _canvas.rows);
|
|
2769
|
+
cv::Rect dstClipped = dstRoi & canvasBoundsRect;
|
|
2770
|
+
if (dstClipped.width <= 0 || dstClipped.height <= 0) {
|
|
2771
|
+
[tele setValue:@(RLISFrameOutcomeRejectedAlignmentLost)
|
|
2772
|
+
forKey:@"outcome"];
|
|
2773
|
+
return tele;
|
|
2774
|
+
}
|
|
2775
|
+
|
|
2776
|
+
cv::Rect srcRoiInFrame(
|
|
2777
|
+
srcClipX + (dstClipped.x - dstX),
|
|
2778
|
+
srcClipY + (dstClipped.y - dstY),
|
|
2779
|
+
dstClipped.width, dstClipped.height);
|
|
2780
|
+
frameBGR(srcRoiInFrame).copyTo(warpedCanvas(dstClipped));
|
|
2781
|
+
warpedCanvasMask(dstClipped).setTo(255);
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
// ── Update running max along pan axis ───────────────────────
|
|
2785
|
+
// For non-rectangular warped footprints, use the mask's
|
|
2786
|
+
// bounding-box top as the upper edge (forward-only invariant
|
|
2787
|
+
// means we never let _maxDstY decrease).
|
|
2788
|
+
{
|
|
2789
|
+
int newMaxDstY = (int)_maxDstY;
|
|
2790
|
+
if (homographyApplied) {
|
|
2791
|
+
cv::Mat nz;
|
|
2792
|
+
cv::findNonZero(warpedCanvasMask, nz);
|
|
2793
|
+
if (!nz.empty()) {
|
|
2794
|
+
cv::Rect bb = cv::boundingRect(nz);
|
|
2795
|
+
newMaxDstY = std::max(newMaxDstY, bb.y);
|
|
2796
|
+
} else {
|
|
2797
|
+
newMaxDstY = std::max(newMaxDstY, dstY);
|
|
2798
|
+
}
|
|
2799
|
+
} else {
|
|
2800
|
+
newMaxDstY = std::max(newMaxDstY, dstY);
|
|
2801
|
+
}
|
|
2802
|
+
_maxDstY = newMaxDstY;
|
|
2803
|
+
}
|
|
2804
|
+
|
|
2805
|
+
[tele setValue:@(_maxDstY + clipH) forKey:@"paintedExtent"];
|
|
2806
|
+
|
|
2807
|
+
// ── V15: Paint warpedCanvas onto _canvas, mode per config ────
|
|
2808
|
+
// FirstPaintedWins (default for slitscan-rotate, V13.0e+ baseline):
|
|
2809
|
+
// Paint only where canvas is currently UNPAINTED (mask==0).
|
|
2810
|
+
// Already-painted pixels are protected.
|
|
2811
|
+
//
|
|
2812
|
+
// FeatherBlend (default for slitscan-both):
|
|
2813
|
+
// Paint UNPAINTED canvas pixels straight (mask==0 → copy).
|
|
2814
|
+
// Already-painted overlap pixels (mask==255) get an alpha
|
|
2815
|
+
// blend with the new content, preserving the first slit's
|
|
2816
|
+
// structural signal while smoothing slit boundaries.
|
|
2817
|
+
// Hypothesis (Ram): with no accept gate (kMinAcceptDeltaPx
|
|
2818
|
+
// = 0 in slitscan-both default), per-accept advance is small
|
|
2819
|
+
// (~5–10 px) → per-accept misalignment is small → blending
|
|
2820
|
+
// small misalignment over large overlap looks smooth, not
|
|
2821
|
+
// ghosted (the V13.0d ghosting came from blending 50 px
|
|
2822
|
+
// misalignment in a 50 px overlap zone — much larger error/
|
|
2823
|
+
// overlap ratio).
|
|
2824
|
+
cv::Mat canvasMaskZero;
|
|
2825
|
+
cv::compare(_canvasMask, 0, canvasMaskZero, cv::CMP_EQ);
|
|
2826
|
+
|
|
2827
|
+
cv::Mat paintMaskFresh;
|
|
2828
|
+
cv::bitwise_and(canvasMaskZero, warpedCanvasMask, paintMaskFresh);
|
|
2829
|
+
|
|
2830
|
+
// Always paint unpainted canvas pixels straight from new slit.
|
|
2831
|
+
warpedCanvas.copyTo(_canvas, paintMaskFresh);
|
|
2832
|
+
cv::bitwise_or(_canvasMask, paintMaskFresh, _canvasMask);
|
|
2833
|
+
|
|
2834
|
+
if (_config.paintMode == RLISPaintModeFeatherBlend) {
|
|
2835
|
+
// For overlap pixels (already painted AND warpedCanvasMask
|
|
2836
|
+
// has new content): alpha-blend at 0.3 weight on new
|
|
2837
|
+
// content (= 70% prev / 30% new). Choice of 0.3 keeps
|
|
2838
|
+
// first-arrival's signal dominant while letting later
|
|
2839
|
+
// slits soften visible seams. At dense per-accept advance
|
|
2840
|
+
// (gate=0), each canvas pixel sees ~30 successive blends;
|
|
2841
|
+
// first-arrival's effective weight is 0.7^N + decay terms,
|
|
2842
|
+
// which converges so the FIRST slit dominates ~50% of
|
|
2843
|
+
// final value — analogous in spirit to first-painted-wins
|
|
2844
|
+
// but smoother at boundaries.
|
|
2845
|
+
cv::Mat canvasMaskNonZero;
|
|
2846
|
+
cv::compare(_canvasMask, 0, canvasMaskNonZero, cv::CMP_NE);
|
|
2847
|
+
cv::Mat overlapMask;
|
|
2848
|
+
cv::bitwise_and(canvasMaskNonZero, warpedCanvasMask, overlapMask);
|
|
2849
|
+
if (cv::countNonZero(overlapMask) > 0) {
|
|
2850
|
+
cv::Mat blended;
|
|
2851
|
+
cv::addWeighted(warpedCanvas, 0.3, _canvas, 0.7, 0.0, blended);
|
|
2852
|
+
blended.copyTo(_canvas, overlapMask);
|
|
2853
|
+
}
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
_accepted += 1;
|
|
2857
|
+
|
|
2858
|
+
// Update prev state for next-frame triangulation. Move (not
|
|
2859
|
+
// copy) the keypoint vector — std::move on cv::Mat is also
|
|
2860
|
+
// cheap (header copy, refcount bump).
|
|
2861
|
+
_prevKeypoints = std::move(curKeypoints);
|
|
2862
|
+
_prevDescriptors = curDescriptors;
|
|
2863
|
+
_prevRotationArkit = R_new.clone();
|
|
2864
|
+
_prevTranslationArkit = (cv::Mat_<double>(3, 1) << tx, ty, tz);
|
|
2865
|
+
// V14.0a — store this accept's final canvas position for the
|
|
2866
|
+
// NEXT accept's homography target-pair construction. Use the
|
|
2867
|
+
// post-clamp dstX/dstY because that's what got painted (or, for
|
|
2868
|
+
// the homography path, the pose+tri pre-warp position — the
|
|
2869
|
+
// homography itself encodes the warp, so prev_pts in frame
|
|
2870
|
+
// coords + this dst position correctly identifies where each
|
|
2871
|
+
// matched feature landed on canvas).
|
|
2872
|
+
_prevAcceptDstX = dstX;
|
|
2873
|
+
_prevAcceptDstY = dstY;
|
|
2874
|
+
|
|
2875
|
+
if (_engineCallCounter % 5 == 0 || _engineCallCounter <= 5) {
|
|
2876
|
+
// V14.0a — note that when homo=1 the actual painted footprint
|
|
2877
|
+
// is non-rectangular; dstX/dstY here describe the pose+tri
|
|
2878
|
+
// pre-warp position (used by the fallback path), not the
|
|
2879
|
+
// warp result's centroid.
|
|
2880
|
+
os_log_with_type(SlitDiagLog(), OS_LOG_TYPE_FAULT,
|
|
2881
|
+
"[V14.0a-paint] #%ld dstX=%d dstY=%d homo=%d _accepted=%ld",
|
|
2882
|
+
(long)_engineCallCounter, dstX, dstY,
|
|
2883
|
+
(int)homographyApplied, (long)_accepted);
|
|
2884
|
+
}
|
|
2885
|
+
[tele setValue:@(RLISFrameOutcomeAcceptedHigh) forKey:@"outcome"];
|
|
2886
|
+
auto t1 = std::chrono::steady_clock::now();
|
|
2887
|
+
double ms = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count() / 1000.0;
|
|
2888
|
+
[tele setValue:@(ms) forKey:@"processingMs"];
|
|
2889
|
+
return tele;
|
|
2890
|
+
}
|
|
2891
|
+
|
|
2892
|
+
// ── Subsequent frame: cylindrical-warp the FULL new frame, then
|
|
2893
|
+
// paint into the canvas ONLY where the canvas mask is empty.
|
|
2894
|
+
// "First-painted wins" — the original first frame stays
|
|
2895
|
+
// untouched in its footprint, and new frames append content
|
|
2896
|
+
// only into pixels that have never been painted before. This
|
|
2897
|
+
// is what gives the user the "first full frame, slits at the
|
|
2898
|
+
// edges" behaviour. No gaps because the cylindrical-warped
|
|
2899
|
+
// new frame covers a wide angular extent (similar to the
|
|
2900
|
+
// first frame's), so adjacent frames overlap heavily and
|
|
2901
|
+
// every pixel between them gets covered.
|
|
2902
|
+
cv::Mat warpedNew, warpedNewMask;
|
|
2903
|
+
cv::Point newCornerCyl =
|
|
2904
|
+
[self cylindricalWarp:frameBGR rArkit:R_new
|
|
2905
|
+
outImage:warpedNew outMask:warpedNewMask];
|
|
2906
|
+
if (warpedNew.empty()) {
|
|
2907
|
+
[tele setValue:@(RLISFrameOutcomeRejectedAlignmentLost) forKey:@"outcome"];
|
|
2908
|
+
return tele;
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2911
|
+
cv::Point newCornerCanvas(newCornerCyl.x - _canvasOriginCylX,
|
|
2912
|
+
newCornerCyl.y - _canvasOriginCylY);
|
|
2913
|
+
cv::Rect dstRoi(newCornerCanvas.x, newCornerCanvas.y,
|
|
2914
|
+
warpedNew.cols, warpedNew.rows);
|
|
2915
|
+
cv::Rect canvasBounds(0, 0, _canvas.cols, _canvas.rows);
|
|
2916
|
+
cv::Rect dstClipped = dstRoi & canvasBounds;
|
|
2917
|
+
if (dstClipped.width <= 0 || dstClipped.height <= 0) {
|
|
2918
|
+
[tele setValue:@(RLISFrameOutcomeRejectedAlignmentLost) forKey:@"outcome"];
|
|
2919
|
+
return tele;
|
|
2920
|
+
}
|
|
2921
|
+
cv::Rect srcRoi(dstClipped.x - dstRoi.x, dstClipped.y - dstRoi.y,
|
|
2922
|
+
dstClipped.width, dstClipped.height);
|
|
2923
|
+
|
|
2924
|
+
cv::Mat warpedNewClipped = warpedNew(srcRoi);
|
|
2925
|
+
cv::Mat warpedNewMaskClipped = warpedNewMask(srcRoi);
|
|
2926
|
+
cv::Mat canvasRoi = _canvas(dstClipped);
|
|
2927
|
+
cv::Mat canvasMaskRoi = _canvasMask(dstClipped);
|
|
2928
|
+
|
|
2929
|
+
// Paint into NEW pixels only.
|
|
2930
|
+
cv::Mat noPrior;
|
|
2931
|
+
cv::compare(canvasMaskRoi, 0, noPrior, cv::CMP_EQ);
|
|
2932
|
+
cv::Mat paintMask;
|
|
2933
|
+
cv::bitwise_and(noPrior, warpedNewMaskClipped, paintMask);
|
|
2934
|
+
if (cv::countNonZero(paintMask) > 0) {
|
|
2935
|
+
warpedNewClipped.copyTo(canvasRoi, paintMask);
|
|
2936
|
+
cv::bitwise_or(canvasMaskRoi, paintMask, canvasMaskRoi);
|
|
2937
|
+
_accepted += 1;
|
|
2938
|
+
[tele setValue:@(RLISFrameOutcomeAcceptedHigh) forKey:@"outcome"];
|
|
2939
|
+
[tele setValue:@(1.0) forKey:@"confidence"];
|
|
2940
|
+
} else {
|
|
2941
|
+
// No new content to paint — the new frame's coverage is
|
|
2942
|
+
// entirely inside the existing canvas. Skip silently.
|
|
2943
|
+
[tele setValue:@(RLISFrameOutcomeSkippedTooClose) forKey:@"outcome"];
|
|
2944
|
+
}
|
|
2945
|
+
|
|
2946
|
+
auto t1 = std::chrono::steady_clock::now();
|
|
2947
|
+
double ms = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count() / 1000.0;
|
|
2948
|
+
[tele setValue:@(ms) forKey:@"processingMs"];
|
|
2949
|
+
return tele;
|
|
2950
|
+
}
|
|
2951
|
+
|
|
2952
|
+
// Hand-rolled cylindrical projection. Same algorithm as v9's helper
|
|
2953
|
+
// in OpenCVIncrementalStitcher.mm — duplicated here to keep the two
|
|
2954
|
+
// engines cleanly separated as separate files. See that file for
|
|
2955
|
+
// the full annotation; in short:
|
|
2956
|
+
// - panorama frame is gravity-up Y, first-camera-forward Z
|
|
2957
|
+
// - R_panToCam = M · R_arkit⁻¹ · R_panToWorld
|
|
2958
|
+
// - V12.2 cylindrical projection (theta = atan2(-wx, wz),
|
|
2959
|
+
// h = wy/sqrt(wx²+wz²)). The −wx flip is the V12 mirror fix.
|
|
2960
|
+
// V12 had switched to spherical to handle extreme pitch; that
|
|
2961
|
+
// bulged level frames into a fisheye, so V12.2 reverts to
|
|
2962
|
+
// cylindrical and will solve pitch via an orientation-aware
|
|
2963
|
+
// cylinder axis (Step 3).
|
|
2964
|
+
// - inverse-map each canvas pixel back to a source pixel
|
|
2965
|
+
// - cv::remap fills the output bbox
|
|
2966
|
+
// - output mask has 255 only where the inverse map landed inside
|
|
2967
|
+
// the source frame
|
|
2968
|
+
- (cv::Point)cylindricalWarp:(const cv::Mat &)src
|
|
2969
|
+
rArkit:(const cv::Mat &)rArkit
|
|
2970
|
+
outImage:(cv::Mat &)outImage
|
|
2971
|
+
outMask:(cv::Mat &)outMask
|
|
2972
|
+
{
|
|
2973
|
+
if (_R_panToWorld.empty() || _focalCompose <= 0) {
|
|
2974
|
+
outImage = cv::Mat(); outMask = cv::Mat();
|
|
2975
|
+
return cv::Point(0, 0);
|
|
2976
|
+
}
|
|
2977
|
+
cv::Mat R_panToCam = _M_arkitToCv * rArkit.t() * _R_panToWorld;
|
|
2978
|
+
const double fx = _K_compose.at<double>(0, 0);
|
|
2979
|
+
const double fy = _K_compose.at<double>(1, 1);
|
|
2980
|
+
const double cx = _K_compose.at<double>(0, 2);
|
|
2981
|
+
const double cy = _K_compose.at<double>(1, 2);
|
|
2982
|
+
const double f = _focalCompose;
|
|
2983
|
+
|
|
2984
|
+
cv::Mat R_camToPan = R_panToCam.t();
|
|
2985
|
+
// V12.3: orientation-aware cylinder axis — see v9 engine for the
|
|
2986
|
+
// full derivation. Portrait → vertical-axis cylinder. Landscape
|
|
2987
|
+
// → transverse (pan_X-axis) cylinder.
|
|
2988
|
+
auto projectCorner = ^cv::Point2d(double u, double v) {
|
|
2989
|
+
double rx = (u - cx) / fx;
|
|
2990
|
+
double ry = (v - cy) / fy;
|
|
2991
|
+
double rz = 1.0;
|
|
2992
|
+
double wx = R_camToPan.at<double>(0,0)*rx + R_camToPan.at<double>(0,1)*ry + R_camToPan.at<double>(0,2)*rz;
|
|
2993
|
+
double wy = R_camToPan.at<double>(1,0)*rx + R_camToPan.at<double>(1,1)*ry + R_camToPan.at<double>(1,2)*rz;
|
|
2994
|
+
double wz = R_camToPan.at<double>(2,0)*rx + R_camToPan.at<double>(2,1)*ry + R_camToPan.at<double>(2,2)*rz;
|
|
2995
|
+
if (_isLandscape) {
|
|
2996
|
+
double denom = std::sqrt(wy*wy + wz*wz);
|
|
2997
|
+
double s = (denom > 1e-9) ? (-wx / denom) : 0.0;
|
|
2998
|
+
double theta = std::atan2(wy, wz);
|
|
2999
|
+
return cv::Point2d(f * s, -f * theta);
|
|
3000
|
+
} else {
|
|
3001
|
+
// V12 mirror fix kept: −wx so user's-right maps to canvas-right.
|
|
3002
|
+
double theta = std::atan2(-wx, wz);
|
|
3003
|
+
double denom = std::sqrt(wx*wx + wz*wz);
|
|
3004
|
+
double h = (denom > 1e-9) ? (wy / denom) : 0.0;
|
|
3005
|
+
return cv::Point2d(f * theta, -f * h); // Y-flip
|
|
3006
|
+
}
|
|
3007
|
+
};
|
|
3008
|
+
cv::Point2d c00 = projectCorner(0, 0);
|
|
3009
|
+
cv::Point2d c10 = projectCorner((double)src.cols - 1, 0);
|
|
3010
|
+
cv::Point2d c01 = projectCorner(0, (double)src.rows - 1);
|
|
3011
|
+
cv::Point2d c11 = projectCorner((double)src.cols - 1, (double)src.rows - 1);
|
|
3012
|
+
double minX = std::min({c00.x, c10.x, c01.x, c11.x});
|
|
3013
|
+
double maxX = std::max({c00.x, c10.x, c01.x, c11.x});
|
|
3014
|
+
double minY = std::min({c00.y, c10.y, c01.y, c11.y});
|
|
3015
|
+
double maxY = std::max({c00.y, c10.y, c01.y, c11.y});
|
|
3016
|
+
int bboxX = (int)std::floor(minX);
|
|
3017
|
+
int bboxY = (int)std::floor(minY);
|
|
3018
|
+
int bboxW = (int)std::ceil(maxX - minX) + 1;
|
|
3019
|
+
int bboxH = (int)std::ceil(maxY - minY) + 1;
|
|
3020
|
+
if (bboxW <= 0 || bboxH <= 0
|
|
3021
|
+
|| bboxW > (int)_canvas.cols * 2 || bboxH > (int)_canvas.rows * 2) {
|
|
3022
|
+
outImage = cv::Mat(); outMask = cv::Mat();
|
|
3023
|
+
return cv::Point(0, 0);
|
|
3024
|
+
}
|
|
3025
|
+
// V12.4 slit-scan + long-side clip — see v9 engine for the
|
|
3026
|
+
// rationale. Same fractions, same axis-aware logic.
|
|
3027
|
+
static const double kPanStripFraction = 0.70;
|
|
3028
|
+
static const double kLongSideFraction = 0.85;
|
|
3029
|
+
int preCropX = bboxX, preCropY = bboxY, preCropW = bboxW, preCropH = bboxH;
|
|
3030
|
+
{
|
|
3031
|
+
int newW, newH;
|
|
3032
|
+
if (_isLandscape) {
|
|
3033
|
+
newW = std::max(1, (int)(bboxW * kLongSideFraction));
|
|
3034
|
+
newH = std::max(1, (int)(bboxH * kPanStripFraction));
|
|
3035
|
+
} else {
|
|
3036
|
+
newW = std::max(1, (int)(bboxW * kPanStripFraction));
|
|
3037
|
+
newH = std::max(1, (int)(bboxH * kLongSideFraction));
|
|
3038
|
+
}
|
|
3039
|
+
bboxX += (bboxW - newW) / 2;
|
|
3040
|
+
bboxY += (bboxH - newH) / 2;
|
|
3041
|
+
bboxW = newW;
|
|
3042
|
+
bboxH = newH;
|
|
3043
|
+
}
|
|
3044
|
+
// V12.5 telemetry — same line shape as v9 engine, engine tag swapped.
|
|
3045
|
+
NSLog(@"[V12.5-warp] engine=firstwins accepted=%ld isLandscape=%d "
|
|
3046
|
+
@"corners=(%.1f,%.1f),(%.1f,%.1f),(%.1f,%.1f),(%.1f,%.1f) "
|
|
3047
|
+
@"preCrop=(x=%d,y=%d,w=%d,h=%d) "
|
|
3048
|
+
@"postCrop=(x=%d,y=%d,w=%d,h=%d) "
|
|
3049
|
+
@"R_panToCam=[[%.4f,%.4f,%.4f],[%.4f,%.4f,%.4f],[%.4f,%.4f,%.4f]] "
|
|
3050
|
+
@"focalCompose=%.2f",
|
|
3051
|
+
(long)_accepted, (int)_isLandscape,
|
|
3052
|
+
c00.x, c00.y, c10.x, c10.y, c01.x, c01.y, c11.x, c11.y,
|
|
3053
|
+
preCropX, preCropY, preCropW, preCropH,
|
|
3054
|
+
bboxX, bboxY, bboxW, bboxH,
|
|
3055
|
+
R_panToCam.at<double>(0,0), R_panToCam.at<double>(0,1), R_panToCam.at<double>(0,2),
|
|
3056
|
+
R_panToCam.at<double>(1,0), R_panToCam.at<double>(1,1), R_panToCam.at<double>(1,2),
|
|
3057
|
+
R_panToCam.at<double>(2,0), R_panToCam.at<double>(2,1), R_panToCam.at<double>(2,2),
|
|
3058
|
+
f);
|
|
3059
|
+
cv::Mat mapX(bboxH, bboxW, CV_32FC1);
|
|
3060
|
+
cv::Mat mapY(bboxH, bboxW, CV_32FC1);
|
|
3061
|
+
const double r00 = R_panToCam.at<double>(0,0), r01 = R_panToCam.at<double>(0,1), r02 = R_panToCam.at<double>(0,2);
|
|
3062
|
+
const double r10 = R_panToCam.at<double>(1,0), r11 = R_panToCam.at<double>(1,1), r12 = R_panToCam.at<double>(1,2);
|
|
3063
|
+
const double r20 = R_panToCam.at<double>(2,0), r21 = R_panToCam.at<double>(2,1), r22 = R_panToCam.at<double>(2,2);
|
|
3064
|
+
if (_isLandscape) {
|
|
3065
|
+
// Transverse cylinder (axis = pan_X) inverse map.
|
|
3066
|
+
for (int y = 0; y < bboxH; y++) {
|
|
3067
|
+
float *mx = mapX.ptr<float>(y);
|
|
3068
|
+
float *my = mapY.ptr<float>(y);
|
|
3069
|
+
double cylY = (double)(bboxY + y);
|
|
3070
|
+
double theta = -cylY / f;
|
|
3071
|
+
double sinTh = std::sin(theta);
|
|
3072
|
+
double cosTh = std::cos(theta);
|
|
3073
|
+
for (int x = 0; x < bboxW; x++) {
|
|
3074
|
+
double cylX = (double)(bboxX + x);
|
|
3075
|
+
double s = cylX / f;
|
|
3076
|
+
double wx = -s, wy = sinTh, wz = cosTh;
|
|
3077
|
+
double rx = r00*wx + r01*wy + r02*wz;
|
|
3078
|
+
double ry = r10*wx + r11*wy + r12*wz;
|
|
3079
|
+
double rz = r20*wx + r21*wy + r22*wz;
|
|
3080
|
+
if (rz <= 1e-6) { mx[x] = -1.0f; my[x] = -1.0f; }
|
|
3081
|
+
else {
|
|
3082
|
+
double u = fx * rx / rz + cx;
|
|
3083
|
+
double v = fy * ry / rz + cy;
|
|
3084
|
+
if (u < 0 || u >= (double)src.cols || v < 0 || v >= (double)src.rows) {
|
|
3085
|
+
mx[x] = -1.0f; my[x] = -1.0f;
|
|
3086
|
+
} else { mx[x] = (float)u; my[x] = (float)v; }
|
|
3087
|
+
}
|
|
3088
|
+
}
|
|
3089
|
+
}
|
|
3090
|
+
} else {
|
|
3091
|
+
// Vertical cylinder (axis = pan_Y) inverse map.
|
|
3092
|
+
for (int y = 0; y < bboxH; y++) {
|
|
3093
|
+
float *mx = mapX.ptr<float>(y);
|
|
3094
|
+
float *my = mapY.ptr<float>(y);
|
|
3095
|
+
double cylY = (double)(bboxY + y);
|
|
3096
|
+
double h = -cylY / f; // inverse Y-flip
|
|
3097
|
+
for (int x = 0; x < bboxW; x++) {
|
|
3098
|
+
double cylX = (double)(bboxX + x);
|
|
3099
|
+
double theta = cylX / f;
|
|
3100
|
+
double sinT = std::sin(theta);
|
|
3101
|
+
double cosT = std::cos(theta);
|
|
3102
|
+
// Inverse of V12 mirror fix: wx = −sinT.
|
|
3103
|
+
double wx = -sinT, wy = h, wz = cosT;
|
|
3104
|
+
double rx = r00*wx + r01*wy + r02*wz;
|
|
3105
|
+
double ry = r10*wx + r11*wy + r12*wz;
|
|
3106
|
+
double rz = r20*wx + r21*wy + r22*wz;
|
|
3107
|
+
if (rz <= 1e-6) { mx[x] = -1.0f; my[x] = -1.0f; }
|
|
3108
|
+
else {
|
|
3109
|
+
double u = fx * rx / rz + cx;
|
|
3110
|
+
double v = fy * ry / rz + cy;
|
|
3111
|
+
if (u < 0 || u >= (double)src.cols || v < 0 || v >= (double)src.rows) {
|
|
3112
|
+
mx[x] = -1.0f; my[x] = -1.0f;
|
|
3113
|
+
} else { mx[x] = (float)u; my[x] = (float)v; }
|
|
3114
|
+
}
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
}
|
|
3118
|
+
outImage.create(bboxH, bboxW, src.type());
|
|
3119
|
+
cv::remap(src, outImage, mapX, mapY,
|
|
3120
|
+
cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0));
|
|
3121
|
+
outMask.create(bboxH, bboxW, CV_8UC1);
|
|
3122
|
+
outMask.setTo(0);
|
|
3123
|
+
for (int y = 0; y < bboxH; y++) {
|
|
3124
|
+
const float *mx = mapX.ptr<float>(y);
|
|
3125
|
+
uchar *m = outMask.ptr<uchar>(y);
|
|
3126
|
+
for (int x = 0; x < bboxW; x++) if (mx[x] >= 0.0f) m[x] = 255;
|
|
3127
|
+
}
|
|
3128
|
+
return cv::Point(bboxX, bboxY);
|
|
3129
|
+
}
|
|
3130
|
+
|
|
3131
|
+
- (BOOL)convertPixelBuffer:(CVPixelBufferRef)pixelBuffer to:(cv::Mat &)outBGR {
|
|
3132
|
+
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
|
3133
|
+
OSType pf = CVPixelBufferGetPixelFormatType(pixelBuffer);
|
|
3134
|
+
size_t w = CVPixelBufferGetWidth(pixelBuffer);
|
|
3135
|
+
size_t h = CVPixelBufferGetHeight(pixelBuffer);
|
|
3136
|
+
cv::Mat frame;
|
|
3137
|
+
BOOL ok = NO;
|
|
3138
|
+
if (pf == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
|
|
3139
|
+
|| pf == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
|
|
3140
|
+
size_t yStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
|
|
3141
|
+
size_t uvStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
|
|
3142
|
+
uint8_t *yBase = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
|
|
3143
|
+
uint8_t *uvBase = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
|
|
3144
|
+
cv::Mat nv12((int)(h + h / 2), (int)w, CV_8UC1);
|
|
3145
|
+
for (size_t y = 0; y < h; y++) {
|
|
3146
|
+
memcpy(nv12.ptr<uchar>((int)y),
|
|
3147
|
+
yBase + y * yStride, w);
|
|
3148
|
+
}
|
|
3149
|
+
for (size_t y = 0; y < h / 2; y++) {
|
|
3150
|
+
memcpy(nv12.ptr<uchar>((int)(h + y)),
|
|
3151
|
+
uvBase + y * uvStride, w);
|
|
3152
|
+
}
|
|
3153
|
+
cv::cvtColor(nv12, frame, cv::COLOR_YUV2BGR_NV12);
|
|
3154
|
+
ok = YES;
|
|
3155
|
+
} else if (pf == kCVPixelFormatType_32BGRA) {
|
|
3156
|
+
size_t stride = CVPixelBufferGetBytesPerRow(pixelBuffer);
|
|
3157
|
+
uint8_t *base = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer);
|
|
3158
|
+
cv::Mat bgra((int)h, (int)w, CV_8UC4, base, stride);
|
|
3159
|
+
cv::cvtColor(bgra, frame, cv::COLOR_BGRA2BGR);
|
|
3160
|
+
ok = YES;
|
|
3161
|
+
}
|
|
3162
|
+
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
|
3163
|
+
if (!ok) return NO;
|
|
3164
|
+
|
|
3165
|
+
double scale = std::min(
|
|
3166
|
+
(double)_composeWidth / (double)frame.cols,
|
|
3167
|
+
(double)_composeHeight / (double)frame.rows
|
|
3168
|
+
);
|
|
3169
|
+
if (scale > 1.0) scale = 1.0;
|
|
3170
|
+
int outW = std::max(1, (int)std::round(frame.cols * scale));
|
|
3171
|
+
int outH = std::max(1, (int)std::round(frame.rows * scale));
|
|
3172
|
+
if (frame.cols == outW && frame.rows == outH) {
|
|
3173
|
+
outBGR = frame;
|
|
3174
|
+
} else {
|
|
3175
|
+
cv::resize(frame, outBGR, cv::Size(outW, outH), 0, 0, cv::INTER_AREA);
|
|
3176
|
+
}
|
|
3177
|
+
return YES;
|
|
3178
|
+
}
|
|
3179
|
+
|
|
3180
|
+
- (nullable RLISSnapshot *)snapshotWithJpegQuality:(NSInteger)quality
|
|
3181
|
+
error:(NSError **)error
|
|
3182
|
+
{
|
|
3183
|
+
_snapshotSeq += 1;
|
|
3184
|
+
NSString *tmpDir = NSTemporaryDirectory();
|
|
3185
|
+
NSInteger slot = _snapshotSeq % 4;
|
|
3186
|
+
NSString *path = [tmpDir stringByAppendingPathComponent:
|
|
3187
|
+
[NSString stringWithFormat:@"rlss-live-%ld.jpg", (long)slot]];
|
|
3188
|
+
return [self writeOutToPath:path quality:quality applyExposureComp:NO error:error];
|
|
3189
|
+
}
|
|
3190
|
+
|
|
3191
|
+
- (nullable RLISSnapshot *)finalizeAtPath:(NSString *)outputPath
|
|
3192
|
+
jpegQuality:(NSInteger)quality
|
|
3193
|
+
error:(NSError **)error
|
|
3194
|
+
{
|
|
3195
|
+
RLISSnapshot *snap = [self writeOutToPath:outputPath
|
|
3196
|
+
quality:quality
|
|
3197
|
+
applyExposureComp:YES
|
|
3198
|
+
error:error];
|
|
3199
|
+
[self reset];
|
|
3200
|
+
return snap;
|
|
3201
|
+
}
|
|
3202
|
+
|
|
3203
|
+
- (nullable RLISSnapshot *)writeOutToPath:(NSString *)outputPath
|
|
3204
|
+
quality:(NSInteger)quality
|
|
3205
|
+
applyExposureComp:(BOOL)applyExposureComp
|
|
3206
|
+
error:(NSError **)error
|
|
3207
|
+
{
|
|
3208
|
+
if (_accepted == 0) {
|
|
3209
|
+
if (error) {
|
|
3210
|
+
*error = [NSError errorWithDomain:RNImageStitcherIncrementalErrorDomain
|
|
3211
|
+
code:1
|
|
3212
|
+
userInfo:@{NSLocalizedDescriptionKey:
|
|
3213
|
+
@"No strips painted yet."}];
|
|
3214
|
+
}
|
|
3215
|
+
return nil;
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
cv::Rect bbox = cv::boundingRect(_canvasMask);
|
|
3219
|
+
if (bbox.width <= 0 || bbox.height <= 0) {
|
|
3220
|
+
bbox = cv::Rect(0, 0, _canvas.cols, _canvas.rows);
|
|
3221
|
+
}
|
|
3222
|
+
cv::Mat cropped = _canvas(bbox).clone();
|
|
3223
|
+
|
|
3224
|
+
// V12.14.10 — orientation-aware output rotation.
|
|
3225
|
+
//
|
|
3226
|
+
// The engine's canvas is in sensor-Y-as-pan-axis orientation in
|
|
3227
|
+
// both supported modes (1920w × 5000h, dstY grows with pan).
|
|
3228
|
+
// For LANDSCAPE+vertical-pan that's the user-natural output:
|
|
3229
|
+
// 1920 wide × Y tall → wide horizontal strip when displayed in
|
|
3230
|
+
// the portrait-locked app UI (matches what the user saw in
|
|
3231
|
+
// their landscape view).
|
|
3232
|
+
//
|
|
3233
|
+
// For PORTRAIT+horizontal-pan the canvas is the same shape
|
|
3234
|
+
// (1920w × Yh) but the user EXPECTS a wide horizontal strip
|
|
3235
|
+
// showing their horizontal pan extent. Rotate 90° CCW so
|
|
3236
|
+
// the saved JPEG is Y wide × 1920 tall — wide strip in
|
|
3237
|
+
// portrait UI.
|
|
3238
|
+
//
|
|
3239
|
+
// V13.0a — ROTATE_90_CLOCKWISE (was COUNTERCLOCKWISE in V12.14.10).
|
|
3240
|
+
// Ram's V12.14.10 device test showed the saved JPEG appearing
|
|
3241
|
+
// upside-down in the portrait UI; CCW was the wrong direction.
|
|
3242
|
+
// CW maps the canvas's pan-axis growth direction to the user-
|
|
3243
|
+
// perspective rightward direction, which matches the UI's
|
|
3244
|
+
// expected wide-horizontal-strip layout.
|
|
3245
|
+
cv::Mat out;
|
|
3246
|
+
if (_isLandscape) {
|
|
3247
|
+
out = cropped;
|
|
3248
|
+
} else {
|
|
3249
|
+
cv::rotate(cropped, out, cv::ROTATE_90_CLOCKWISE);
|
|
3250
|
+
}
|
|
3251
|
+
|
|
3252
|
+
if (applyExposureComp && !out.empty()) {
|
|
3253
|
+
cv::Mat lab;
|
|
3254
|
+
cv::cvtColor(out, lab, cv::COLOR_BGR2Lab);
|
|
3255
|
+
std::vector<cv::Mat> ch(3);
|
|
3256
|
+
cv::split(lab, ch);
|
|
3257
|
+
cv::Ptr<cv::CLAHE> clahe = cv::createCLAHE(2.0, cv::Size(8, 8));
|
|
3258
|
+
clahe->apply(ch[0], ch[0]);
|
|
3259
|
+
cv::merge(ch, lab);
|
|
3260
|
+
cv::cvtColor(lab, out, cv::COLOR_Lab2BGR);
|
|
3261
|
+
}
|
|
3262
|
+
|
|
3263
|
+
int q = (int)std::clamp((long long)quality, 0LL, 100LL);
|
|
3264
|
+
std::vector<int> params = {cv::IMWRITE_JPEG_QUALITY, q};
|
|
3265
|
+
NSString *cleanPath = [outputPath hasPrefix:@"file://"]
|
|
3266
|
+
? [outputPath substringFromIndex:7] : outputPath;
|
|
3267
|
+
if (!cv::imwrite(std::string([cleanPath UTF8String]), out, params)) {
|
|
3268
|
+
if (error) {
|
|
3269
|
+
*error = [NSError errorWithDomain:RNImageStitcherIncrementalErrorDomain
|
|
3270
|
+
code:2
|
|
3271
|
+
userInfo:@{NSLocalizedDescriptionKey:
|
|
3272
|
+
@"imwrite failed"}];
|
|
3273
|
+
}
|
|
3274
|
+
return nil;
|
|
3275
|
+
}
|
|
3276
|
+
|
|
3277
|
+
RLISSnapshot *snap = [[RLISSnapshot alloc] init];
|
|
3278
|
+
[snap setValue:cleanPath forKey:@"panoramaPath"];
|
|
3279
|
+
[snap setValue:@(out.cols) forKey:@"width"];
|
|
3280
|
+
[snap setValue:@(out.rows) forKey:@"height"];
|
|
3281
|
+
[snap setValue:@(_accepted) forKey:@"acceptedCount"];
|
|
3282
|
+
return snap;
|
|
3283
|
+
}
|
|
3284
|
+
|
|
3285
|
+
@end
|