react-native-image-stitcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/LICENSE +201 -0
- package/NOTICE +21 -0
- package/README.md +189 -0
- package/RNImageStitcher.podspec +76 -0
- package/android/build.gradle +224 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/cpp/CMakeLists.txt +124 -0
- package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
- package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
- package/cpp/ar_frame_pose.h +63 -0
- package/cpp/keyframe_gate.cpp +927 -0
- package/cpp/keyframe_gate.hpp +240 -0
- package/cpp/stitcher.cpp +2207 -0
- package/cpp/stitcher.hpp +275 -0
- package/dist/ar/useARSession.d.ts +102 -0
- package/dist/ar/useARSession.js +133 -0
- package/dist/camera/ARCameraView.d.ts +93 -0
- package/dist/camera/ARCameraView.js +170 -0
- package/dist/camera/Camera.d.ts +134 -0
- package/dist/camera/Camera.js +688 -0
- package/dist/camera/CameraShutter.d.ts +80 -0
- package/dist/camera/CameraShutter.js +237 -0
- package/dist/camera/CameraView.d.ts +65 -0
- package/dist/camera/CameraView.js +117 -0
- package/dist/camera/CaptureControlsBar.d.ts +87 -0
- package/dist/camera/CaptureControlsBar.js +82 -0
- package/dist/camera/CaptureHeader.d.ts +62 -0
- package/dist/camera/CaptureHeader.js +81 -0
- package/dist/camera/CapturePreview.d.ts +70 -0
- package/dist/camera/CapturePreview.js +188 -0
- package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
- package/dist/camera/CaptureStatusOverlay.js +326 -0
- package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
- package/dist/camera/CaptureThumbnailStrip.js +177 -0
- package/dist/camera/IncrementalPanGuide.d.ts +83 -0
- package/dist/camera/IncrementalPanGuide.js +267 -0
- package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
- package/dist/camera/PanoramaBandOverlay.js +399 -0
- package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
- package/dist/camera/PanoramaConfirmModal.js +128 -0
- package/dist/camera/PanoramaGuidance.d.ts +79 -0
- package/dist/camera/PanoramaGuidance.js +246 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
- package/dist/camera/PanoramaSettingsModal.js +611 -0
- package/dist/camera/ViewportCropOverlay.d.ts +46 -0
- package/dist/camera/ViewportCropOverlay.js +67 -0
- package/dist/camera/useCapture.d.ts +111 -0
- package/dist/camera/useCapture.js +160 -0
- package/dist/camera/useDeviceOrientation.d.ts +48 -0
- package/dist/camera/useDeviceOrientation.js +131 -0
- package/dist/camera/useVideoCapture.d.ts +79 -0
- package/dist/camera/useVideoCapture.js +151 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.js +39 -0
- package/dist/quality/normaliseOrientation.d.ts +36 -0
- package/dist/quality/normaliseOrientation.js +62 -0
- package/dist/quality/runQualityCheck.d.ts +41 -0
- package/dist/quality/runQualityCheck.js +98 -0
- package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
- package/dist/sensors/useIMUTranslationGate.js +235 -0
- package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
- package/dist/stitching/IncrementalStitcherView.js +157 -0
- package/dist/stitching/incremental.d.ts +930 -0
- package/dist/stitching/incremental.js +133 -0
- package/dist/stitching/stitchFrames.d.ts +55 -0
- package/dist/stitching/stitchFrames.js +56 -0
- package/dist/stitching/stitchVideo.d.ts +119 -0
- package/dist/stitching/stitchVideo.js +57 -0
- package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
- package/dist/stitching/useIncrementalJSDriver.js +199 -0
- package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
- package/dist/stitching/useIncrementalStitcher.js +172 -0
- package/dist/types.d.ts +58 -0
- package/dist/types.js +15 -0
- package/ios/Package.swift +72 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
- package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
- package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
- package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
- package/package.json +73 -0
- package/react-native.config.js +34 -0
- package/scripts/opencv-version.txt +1 -0
- package/scripts/postinstall-fetch-binaries.js +286 -0
- package/src/ar/useARSession.ts +210 -0
- package/src/camera/.gitkeep +0 -0
- package/src/camera/ARCameraView.tsx +256 -0
- package/src/camera/Camera.tsx +1053 -0
- package/src/camera/CameraShutter.tsx +292 -0
- package/src/camera/CameraView.tsx +157 -0
- package/src/camera/CaptureControlsBar.tsx +204 -0
- package/src/camera/CaptureHeader.tsx +184 -0
- package/src/camera/CapturePreview.tsx +318 -0
- package/src/camera/CaptureStatusOverlay.tsx +391 -0
- package/src/camera/CaptureThumbnailStrip.tsx +277 -0
- package/src/camera/IncrementalPanGuide.tsx +328 -0
- package/src/camera/PanoramaBandOverlay.tsx +498 -0
- package/src/camera/PanoramaConfirmModal.tsx +206 -0
- package/src/camera/PanoramaGuidance.tsx +327 -0
- package/src/camera/PanoramaSettingsModal.tsx +1357 -0
- package/src/camera/ViewportCropOverlay.tsx +81 -0
- package/src/camera/useCapture.ts +279 -0
- package/src/camera/useDeviceOrientation.ts +140 -0
- package/src/camera/useVideoCapture.ts +236 -0
- package/src/index.ts +53 -0
- package/src/quality/.gitkeep +0 -0
- package/src/quality/normaliseOrientation.ts +79 -0
- package/src/quality/runQualityCheck.ts +131 -0
- package/src/sensors/useIMUTranslationGate.ts +347 -0
- package/src/stitching/.gitkeep +0 -0
- package/src/stitching/IncrementalStitcherView.tsx +198 -0
- package/src/stitching/incremental.ts +1021 -0
- package/src/stitching/stitchFrames.ts +88 -0
- package/src/stitching/stitchVideo.ts +153 -0
- package/src/stitching/useIncrementalJSDriver.ts +273 -0
- package/src/stitching/useIncrementalStitcher.ts +252 -0
- package/src/types.ts +78 -0
|
@@ -0,0 +1,2371 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
import android.app.ActivityManager
|
|
5
|
+
import com.facebook.react.bridge.Arguments
|
|
6
|
+
import com.facebook.react.bridge.Promise
|
|
7
|
+
import com.facebook.react.bridge.ReactApplicationContext
|
|
8
|
+
import com.facebook.react.bridge.ReactContextBaseJavaModule
|
|
9
|
+
import com.facebook.react.bridge.ReactMethod
|
|
10
|
+
import com.facebook.react.bridge.ReadableMap
|
|
11
|
+
import com.facebook.react.bridge.WritableMap
|
|
12
|
+
import com.facebook.react.modules.core.DeviceEventManagerModule
|
|
13
|
+
import kotlin.math.max
|
|
14
|
+
import kotlin.math.min
|
|
15
|
+
import kotlinx.coroutines.CoroutineScope
|
|
16
|
+
import kotlinx.coroutines.Dispatchers
|
|
17
|
+
import kotlinx.coroutines.SupervisorJob
|
|
18
|
+
import kotlinx.coroutines.launch
|
|
19
|
+
import org.opencv.calib3d.Calib3d
|
|
20
|
+
import org.opencv.core.Core
|
|
21
|
+
import org.opencv.core.CvType
|
|
22
|
+
import org.opencv.core.DMatch
|
|
23
|
+
import org.opencv.core.KeyPoint
|
|
24
|
+
import org.opencv.core.Mat
|
|
25
|
+
import org.opencv.core.MatOfByte
|
|
26
|
+
import org.opencv.core.MatOfDMatch
|
|
27
|
+
import org.opencv.core.MatOfInt
|
|
28
|
+
import org.opencv.core.MatOfKeyPoint
|
|
29
|
+
import org.opencv.core.MatOfPoint2f
|
|
30
|
+
import org.opencv.core.Point
|
|
31
|
+
import org.opencv.core.Rect
|
|
32
|
+
import org.opencv.core.Scalar
|
|
33
|
+
import org.opencv.core.Size
|
|
34
|
+
import org.opencv.features2d.BFMatcher
|
|
35
|
+
import org.opencv.features2d.ORB
|
|
36
|
+
import org.opencv.imgcodecs.Imgcodecs
|
|
37
|
+
import org.opencv.imgproc.Imgproc
|
|
38
|
+
import java.io.File
|
|
39
|
+
import java.util.concurrent.atomic.AtomicBoolean
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Android twin of iOS' OpenCVIncrementalStitcher + IncrementalStitcher.
|
|
43
|
+
*
|
|
44
|
+
* Why a single file (vs the iOS three-file split):
|
|
45
|
+
* On iOS we cross C++↔ObjC↔Swift boundaries, so the .h/.mm/.swift
|
|
46
|
+
* layering pays for itself. On Android, OpenCV's Java bindings
|
|
47
|
+
* give us cv::* operations directly callable from Kotlin — no JNI
|
|
48
|
+
* layer, no language boundary, the engine logic lives next to the
|
|
49
|
+
* RN module.
|
|
50
|
+
*
|
|
51
|
+
* Why we DON'T need cv::Stitcher (the missing piece on Android):
|
|
52
|
+
* The incremental algorithm only uses ORB + BFMatcher +
|
|
53
|
+
* findHomography + warpPerspective + distanceTransform. All of
|
|
54
|
+
* these ship in the prebuilt `libopencv_java4.so` (features2d,
|
|
55
|
+
* calib3d, imgproc are always-on modules). See the design doc for
|
|
56
|
+
* the full module-level breakdown.
|
|
57
|
+
*
|
|
58
|
+
* What the bridge exposes to JS:
|
|
59
|
+
* - start(options) — spin up the engine
|
|
60
|
+
* - processFrameAtPath() — feed a JPEG path + pose; engine returns
|
|
61
|
+
* the same outcome enum iOS emits as events
|
|
62
|
+
* - finalize(options) — write the final panorama and reset
|
|
63
|
+
* - cancel() — abort without producing output
|
|
64
|
+
* - getState() — pull the latest state on demand
|
|
65
|
+
* - Event "IncrementalStateUpdate" emitted on every
|
|
66
|
+
* processFrameAtPath call
|
|
67
|
+
*
|
|
68
|
+
* What's missing for true live capture on Android:
|
|
69
|
+
* ARCore-backed live frame delivery. The engine itself doesn't
|
|
70
|
+
* care where frames come from; today the only Android caller is
|
|
71
|
+
* the `processFrameAtPath` bridge method. A follow-up will plumb
|
|
72
|
+
* ARCore's per-frame `Frame.acquireCameraImage()` directly into
|
|
73
|
+
* the engine the same way iOS uses ARSession.
|
|
74
|
+
*/
|
|
75
|
+
class IncrementalStitcher(
|
|
76
|
+
private val reactContext: ReactApplicationContext,
|
|
77
|
+
) : ReactContextBaseJavaModule(reactContext) {
|
|
78
|
+
|
|
79
|
+
override fun getName(): String = "IncrementalStitcher"
|
|
80
|
+
|
|
81
|
+
/// Required by RCTEventEmitter contract. No-op on Android because
|
|
82
|
+
/// `DeviceEventManagerModule` does its own listener tracking; we
|
|
83
|
+
/// emit unconditionally and RN drops events when no listener is
|
|
84
|
+
/// attached.
|
|
85
|
+
@ReactMethod
|
|
86
|
+
fun addListener(eventName: String) { /* no-op */ }
|
|
87
|
+
|
|
88
|
+
@ReactMethod
|
|
89
|
+
fun removeListeners(count: Int) { /* no-op */ }
|
|
90
|
+
|
|
91
|
+
/// V7 hybrid engine — selected for engineMode == 'hybrid'.
|
|
92
|
+
private var engine: IncrementalEngine? = null
|
|
93
|
+
/// V12.7 firstwins engine — selected for any engineMode starting
|
|
94
|
+
/// with 'firstwins' (firstwins, firstwins-zoomed, firstwins-rectilinear).
|
|
95
|
+
/// Native engine is identical for firstwins and firstwins-zoomed
|
|
96
|
+
/// (the difference is JS-side viewport zoom only). useRectilinear
|
|
97
|
+
/// is set for 'firstwins-rectilinear'.
|
|
98
|
+
private var firstwinsEngine: IncrementalFirstwinsEngine? = null
|
|
99
|
+
|
|
100
|
+
// ── V16 batch-keyframe mode (Android parity with iOS' V16 Phase 1) ─
|
|
101
|
+
//
|
|
102
|
+
// Selected for engineMode == 'batch-keyframe'. No live engine
|
|
103
|
+
// runs — instead, accepted frames are collected as keyframe paths,
|
|
104
|
+
// and at finalize() time we hand them all to the JNI shim
|
|
105
|
+
// (libimage_stitcher.so) for one-shot cv::Stitcher processing.
|
|
106
|
+
//
|
|
107
|
+
// The MVP gate is frame-count-based ("accept every Nth frame
|
|
108
|
+
// until cap"). iOS uses a pose-based gate (overlap < threshold)
|
|
109
|
+
// — adding that here is a follow-up that needs ARCore-pose
|
|
110
|
+
// accumulation across processFrameAtPath calls. For now, every
|
|
111
|
+
// N-th frame is good enough to validate end-to-end stitching
|
|
112
|
+
// parity.
|
|
113
|
+
private var batchKeyframeMode: Boolean = false
|
|
114
|
+
private val batchKeyframePaths: MutableList<String> = mutableListOf()
|
|
115
|
+
private var batchKeyframeFrameCounter: Int = 0
|
|
116
|
+
/// V16 Phase 2 (Android Fix-1) — per-capture-session subdirectory
|
|
117
|
+
/// under `cacheDir` where this capture's batch-keyframe JPEGs are
|
|
118
|
+
/// written. Created on each batch-keyframe `start()` with a fresh
|
|
119
|
+
/// UUID.
|
|
120
|
+
///
|
|
121
|
+
/// Why:
|
|
122
|
+
/// The V16 Phase-1 MVP wrote every accepted keyframe to
|
|
123
|
+
/// `cacheDir/rlis-keyframe-{N}.jpg` where N restarted at 0 on
|
|
124
|
+
/// each capture. Two captures in a row → second capture's
|
|
125
|
+
/// `rlis-keyframe-0.jpg` overwrites the first's. Worse, RN's
|
|
126
|
+
/// `<Image>` component on Android caches decoded bitmaps keyed
|
|
127
|
+
/// by URI string; the file:// URI was byte-identical across
|
|
128
|
+
/// captures, so the previous capture's bitmap got served for
|
|
129
|
+
/// the new capture's first thumbnail — Ram's "thumbnails come
|
|
130
|
+
/// from the previous capture" symptom (2026-05-12). Also a
|
|
131
|
+
/// data-integrity hazard if a new capture starts while the
|
|
132
|
+
/// previous one's stitcher is still reading the JPEGs from
|
|
133
|
+
/// disk.
|
|
134
|
+
///
|
|
135
|
+
/// Per-session UUID subdir fixes both:
|
|
136
|
+
/// - Each capture's keyframes live at a unique path → no URI
|
|
137
|
+
/// collision → no bitmap-cache reuse across captures.
|
|
138
|
+
/// - Files survive past finalize for post-hoc reprocessing
|
|
139
|
+
/// (Ram's request — same behaviour as iOS' OpenCVKeyframeCollector).
|
|
140
|
+
///
|
|
141
|
+
/// Lifetime:
|
|
142
|
+
/// • Created in `start()` batch-keyframe branch.
|
|
143
|
+
/// • Used by `copyKeyframeToStore()`.
|
|
144
|
+
/// • Persists past `finalize()` for reprocessing.
|
|
145
|
+
/// • Cleaned up on `cancel()` and in `onCatalystInstanceDestroy()`.
|
|
146
|
+
///
|
|
147
|
+
/// Parity: matches iOS `OpenCVKeyframeCollector.sessionDir`
|
|
148
|
+
/// (created with `Library/AppSupport/Captures/{NSUUID}/`).
|
|
149
|
+
private var captureSessionDir: java.io.File? = null
|
|
150
|
+
/// Accept every Nth frame. 10 is the iOS default capture cadence
|
|
151
|
+
/// (5-6 keyframes over a ~2-3 second pan = roughly one every 10
|
|
152
|
+
/// frames at 30fps).
|
|
153
|
+
private var batchKeyframeAcceptStride: Int = 10
|
|
154
|
+
/// Hard cap on keyframes to match iOS' default (V16 Phase 1's
|
|
155
|
+
/// keyframeMaxCount=6). Going higher inflates cv::Stitcher's
|
|
156
|
+
/// MultiBandBlender memory; iOS hit OOM at 7+ on some scenes.
|
|
157
|
+
private var batchKeyframeMaxCount: Int = 6
|
|
158
|
+
/// Batch knobs threaded through to nativeStitchFramePaths at
|
|
159
|
+
/// finalize. Mirror iOS' batchWarperType / batchBlenderType /
|
|
160
|
+
/// batchSeamFinderType / batchEnableInscribedRectCrop ivars.
|
|
161
|
+
private var batchWarperType: String = "plane"
|
|
162
|
+
private var batchBlenderType: String = "multiband"
|
|
163
|
+
private var batchSeamFinderType: String = "graphcut"
|
|
164
|
+
private var batchUseInscribedRectCrop: Boolean = false
|
|
165
|
+
/// Capture orientation at start time. Drives the bake-rotation
|
|
166
|
+
/// table inside the JNI shim. Sourced from configOverrides
|
|
167
|
+
/// (passed from JS), falling back to "portrait".
|
|
168
|
+
private var batchCaptureOrientation: String = "portrait"
|
|
169
|
+
|
|
170
|
+
// ── 2026-05-14: cv::Stitcher pipeline-mode auto-routing ──────────
|
|
171
|
+
//
|
|
172
|
+
// `batchStitchMode` is the JS-supplied setting from
|
|
173
|
+
// PanoramaSettings.stitchMode. Three valid values:
|
|
174
|
+
// 'auto' (default) — at finalize() time, compute translation/
|
|
175
|
+
// rotation totals from the first and last
|
|
176
|
+
// accepted keyframe pose, pick PANORAMA or
|
|
177
|
+
// SCANS by the design-doc 0.55 threshold.
|
|
178
|
+
// 'panorama' — force cv::Stitcher::PANORAMA mode at JNI.
|
|
179
|
+
// 'scans' — force cv::Stitcher::SCANS mode at JNI.
|
|
180
|
+
//
|
|
181
|
+
// batchFirstAcceptedPose / batchLastAcceptedPose are populated by
|
|
182
|
+
// ingestFromARCameraView() on every accepted keyframe. Cleared
|
|
183
|
+
// at start() and consumed at finalize(). They store (tx, ty, tz,
|
|
184
|
+
// qx, qy, qz, qw) — same shape as `KeyframeGate`'s internal
|
|
185
|
+
// last-accepted-pose tracker, but kept locally so we don't have
|
|
186
|
+
// to wire a new accessor through the C++ bridge.
|
|
187
|
+
private var batchStitchMode: String = "auto"
|
|
188
|
+
private var batchFirstAcceptedPose: DoubleArray? = null
|
|
189
|
+
private var batchLastAcceptedPose: DoubleArray? = null
|
|
190
|
+
|
|
191
|
+
/// V16 Phase 1 / P3-F — shared-C++ KeyframeGate. Replaces the
|
|
192
|
+
/// V16-Phase-1 frame-counter MVP placeholder
|
|
193
|
+
/// (handleBatchKeyframeFrame above) with the same pose-driven
|
|
194
|
+
/// 40%-new-content algorithm iOS has used since the V16 ship.
|
|
195
|
+
/// Both platforms call into retailens::KeyframeGate (in
|
|
196
|
+
/// retailens-capture-sdk/cpp/keyframe_gate.cpp) — see that file
|
|
197
|
+
/// for the algorithm.
|
|
198
|
+
///
|
|
199
|
+
/// Lifetime: owned for the life of the module. Closed in
|
|
200
|
+
/// `onCatalystInstanceDestroy()` to release the C++ heap
|
|
201
|
+
/// allocation. `reset()` is called between captures.
|
|
202
|
+
private val keyframeGate = KeyframeGate()
|
|
203
|
+
|
|
204
|
+
/// P3-G diagnostic — rate-limit the per-frame log in
|
|
205
|
+
/// ingestFromARCameraView so we don't spam logcat at 60Hz but
|
|
206
|
+
/// still see the "0 frames captured" mystery resolve to a
|
|
207
|
+
/// specific failure mode.
|
|
208
|
+
private var frameIngestLogTick: Int = 0
|
|
209
|
+
|
|
210
|
+
private val isRunning = AtomicBoolean(false)
|
|
211
|
+
/// Critic #5 fix: serial dispatcher so concurrent
|
|
212
|
+
/// processFrameAtPath() calls can't race on the engine's canvas.
|
|
213
|
+
/// `limitedParallelism(1)` guarantees one-at-a-time execution
|
|
214
|
+
/// while still backing onto the Default pool — matches iOS'
|
|
215
|
+
/// `workQueue` (DispatchQueue.serial).
|
|
216
|
+
@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class)
|
|
217
|
+
private val workScope = CoroutineScope(Dispatchers.Default.limitedParallelism(1))
|
|
218
|
+
|
|
219
|
+
/// 2026-05-16 — realtime+batch fusion (Option A "Replace on
|
|
220
|
+
/// completion") scope. Kept SEPARATE from `workScope` so the
|
|
221
|
+
/// 2-5 s `cv::Stitcher` refinement run that follows a hybrid-
|
|
222
|
+
/// engine finalize() does NOT delay a new start()/processFrame()
|
|
223
|
+
/// that the operator may issue while the refinement is in flight.
|
|
224
|
+
/// The design doc explicitly calls out "operator can continue
|
|
225
|
+
/// browsing / starting another capture during refinement".
|
|
226
|
+
///
|
|
227
|
+
/// Serial: at most one refinement runs at a time (the design's
|
|
228
|
+
/// "cancellation semantics if a new capture starts mid-refine"
|
|
229
|
+
/// is out of scope for this MVP).
|
|
230
|
+
///
|
|
231
|
+
/// 2026-05-16 (Phase 3 critic MED-1) — `SupervisorJob()` keeps
|
|
232
|
+
/// the scope alive when a single refinement coroutine fails.
|
|
233
|
+
/// Every `refineScope.launch { … }` already has a try/catch
|
|
234
|
+
/// around the throwing surface; SupervisorJob is defense-in-
|
|
235
|
+
/// depth for future code added outside that catch.
|
|
236
|
+
@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class)
|
|
237
|
+
private val refineScope = CoroutineScope(
|
|
238
|
+
SupervisorJob() + Dispatchers.Default.limitedParallelism(1)
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
/// Reference to a mounted ARCameraView (if any). Set by the view
|
|
242
|
+
/// when it attaches; the engine flips its `ingestActive` flag
|
|
243
|
+
/// on start/stop so the view feeds frames only during a capture.
|
|
244
|
+
@Volatile private var arCameraViewRef: RNSARCameraView? = null
|
|
245
|
+
|
|
246
|
+
init {
|
|
247
|
+
// Static back-pointer so `RNSARCameraView` can call into
|
|
248
|
+
// the singleton-style bridge module without a DI dance. RN
|
|
249
|
+
// may rebuild module instances across reloads; the view always
|
|
250
|
+
// uses the latest reference.
|
|
251
|
+
bridgeInstance = this
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/// View calls this on attach so the engine can route ingestion
|
|
255
|
+
/// without searching the view tree on every frame.
|
|
256
|
+
internal fun bindArCameraView(view: RNSARCameraView) {
|
|
257
|
+
arCameraViewRef = view
|
|
258
|
+
// If a capture is already running when the view mounts, hot-
|
|
259
|
+
// engage ingestion so the user gets a partial panorama
|
|
260
|
+
// started from this point onward.
|
|
261
|
+
if (isRunning.get()) {
|
|
262
|
+
view.setIncrementalIngestionActive(true)
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
internal fun unbindArCameraView(view: RNSARCameraView) {
|
|
267
|
+
if (arCameraViewRef === view) {
|
|
268
|
+
view.setIncrementalIngestionActive(false)
|
|
269
|
+
arCameraViewRef = null
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
@ReactMethod
|
|
274
|
+
fun start(options: ReadableMap, promise: Promise) {
|
|
275
|
+
if (isRunning.getAndSet(true)) {
|
|
276
|
+
promise.reject(
|
|
277
|
+
"incremental-already-running",
|
|
278
|
+
"An incremental capture is already in progress.",
|
|
279
|
+
)
|
|
280
|
+
return
|
|
281
|
+
}
|
|
282
|
+
try {
|
|
283
|
+
ensureOpenCv()
|
|
284
|
+
val rotation = options.getIntOrDefault("frameRotationDegrees", 90)
|
|
285
|
+
val composeW = options.getIntOrDefault("composeWidth", 960)
|
|
286
|
+
val composeH = options.getIntOrDefault("composeHeight", 720)
|
|
287
|
+
// V12 default canvas: 5000x5000 to match iOS. Old default
|
|
288
|
+
// was 4800x2200 (V7 wide-only); V12 needs square because
|
|
289
|
+
// either pan axis can grow.
|
|
290
|
+
val canvasW = options.getIntOrDefault("canvasWidth", 5000)
|
|
291
|
+
val canvasH = options.getIntOrDefault("canvasHeight", 5000)
|
|
292
|
+
val featherP = options.getIntOrDefault("featherPx", 20)
|
|
293
|
+
val snapQ = max(1, min(100, options.getIntOrDefault("snapshotJpegQuality", 75)))
|
|
294
|
+
// Critic #29: clamp snapshotEveryNAccepts to >= 1 so a
|
|
295
|
+
// value of 0 doesn't mean "snapshot every frame forever".
|
|
296
|
+
val snapN = max(1, options.getIntOrDefault("snapshotEveryNAccepts", 1))
|
|
297
|
+
// V12.7 — engineMode now distinguishes 4 variants. See
|
|
298
|
+
// src/stitching/incremental.ts for the full description.
|
|
299
|
+
// V16 added 'batch-keyframe' as a fifth variant: no live
|
|
300
|
+
// engine, frames are saved as JPEGs and handed to
|
|
301
|
+
// cv::Stitcher (via the JNI shim) at finalize.
|
|
302
|
+
val engineMode = options.getString("engine") ?: "hybrid"
|
|
303
|
+
// 2026-05-15 — Route 'slitscan*' engineModes to the same
|
|
304
|
+
// IncrementalFirstwinsEngine that handles 'firstwins*'.
|
|
305
|
+
// Per IncrementalFirstwinsEngine's docstring (lines 260,
|
|
306
|
+
// 431, 436, 672, 957): "Mirrors iOS' OpenCVSlitScanStitcher.mm
|
|
307
|
+
// exactly". Before this change, 'slitscan' / 'slitscan-rotate'
|
|
308
|
+
// / 'slitscan-both' engineModes fell through to IncrementalEngine
|
|
309
|
+
// (the hybrid engine), producing identical output to picking
|
|
310
|
+
// 'hybrid' — silent platform divergence vs iOS.
|
|
311
|
+
//
|
|
312
|
+
// iOS-parity reference: IncrementalStitcher.swift:556
|
|
313
|
+
// computes `useFirstwinsClass = normalisedMode.hasPrefix("slitscan")`
|
|
314
|
+
// which routes BOTH 'slitscan-rotate' AND 'slitscan-both' AND
|
|
315
|
+
// the deprecated aliases to OpenCVFirstWinsCylindricalStitcher.
|
|
316
|
+
// We mirror that logic here so Android Settings → Engine
|
|
317
|
+
// dropdown actually toggles the underlying engine.
|
|
318
|
+
val isFirstwinsClass =
|
|
319
|
+
engineMode.startsWith("firstwins") ||
|
|
320
|
+
engineMode.startsWith("slitscan")
|
|
321
|
+
val isFirstwins = isFirstwinsClass // legacy name kept
|
|
322
|
+
// for the remainder of
|
|
323
|
+
// start() — refactor to
|
|
324
|
+
// isFirstwinsClass when
|
|
325
|
+
// the engineMode taxonomy
|
|
326
|
+
// is rationalised.
|
|
327
|
+
val useRectilinear =
|
|
328
|
+
engineMode == "firstwins-rectilinear" ||
|
|
329
|
+
engineMode == "slitscan-rotate"
|
|
330
|
+
val isBatchKeyframe = engineMode == "batch-keyframe"
|
|
331
|
+
|
|
332
|
+
val configOverrides: ReadableMap? =
|
|
333
|
+
if (options.hasKey("config")) options.getMap("config") else null
|
|
334
|
+
|
|
335
|
+
if (isBatchKeyframe) {
|
|
336
|
+
// No live engine runs. Reset the keyframe collector
|
|
337
|
+
// state. Read knobs from `config` per the V16 Phase
|
|
338
|
+
// 1 plumbing pattern.
|
|
339
|
+
engine = null
|
|
340
|
+
firstwinsEngine = null
|
|
341
|
+
batchKeyframeMode = true
|
|
342
|
+
batchKeyframePaths.clear()
|
|
343
|
+
batchKeyframeFrameCounter = 0
|
|
344
|
+
// V16 Phase 2 (Android Fix-1) — fresh per-session subdir
|
|
345
|
+
// for this capture's keyframe JPEGs. Replaces the
|
|
346
|
+
// V16-Phase-1 "rlis-keyframe-{N}.jpg in cacheDir"
|
|
347
|
+
// scheme that caused thumbnails from a previous capture
|
|
348
|
+
// to leak into the next one via RN's bitmap cache (see
|
|
349
|
+
// `captureSessionDir` declaration above for the full
|
|
350
|
+
// RCA). Matches iOS' OpenCVKeyframeCollector behaviour.
|
|
351
|
+
captureSessionDir = java.io.File(
|
|
352
|
+
reactContext.cacheDir,
|
|
353
|
+
"rlis-capture-${java.util.UUID.randomUUID()}",
|
|
354
|
+
).also { it.mkdirs() }
|
|
355
|
+
batchKeyframeMaxCount = configOverrides
|
|
356
|
+
?.getIntOrDefault("keyframeMaxCount", 6) ?: 6
|
|
357
|
+
batchWarperType = configOverrides?.getString("warperType")
|
|
358
|
+
?: "plane"
|
|
359
|
+
batchBlenderType = configOverrides?.getString("blenderType")
|
|
360
|
+
?: "multiband"
|
|
361
|
+
batchSeamFinderType = configOverrides?.getString("seamFinderType")
|
|
362
|
+
?: "graphcut"
|
|
363
|
+
batchUseInscribedRectCrop = configOverrides
|
|
364
|
+
?.getBooleanOrDefault("enableMaxInscribedRectCrop", false)
|
|
365
|
+
?: false
|
|
366
|
+
// 2026-05-14 — stitch-mode picker from JS Settings.
|
|
367
|
+
// Default 'auto'. Validated against the closed set
|
|
368
|
+
// {auto, panorama, scans}; unknown values fall back
|
|
369
|
+
// to 'auto'. Reset accumulated-pose state for the
|
|
370
|
+
// new capture so finalize() picks a fresh mode.
|
|
371
|
+
batchStitchMode = (configOverrides?.getString("stitchMode") ?: "auto")
|
|
372
|
+
.let { if (it in setOf("auto", "panorama", "scans")) it else "auto" }
|
|
373
|
+
batchFirstAcceptedPose = null
|
|
374
|
+
batchLastAcceptedPose = null
|
|
375
|
+
// captureOrientation is JS-supplied here (Android
|
|
376
|
+
// doesn't yet have a native ARCore classifier
|
|
377
|
+
// equivalent to iOS' nativeCaptureOrientation; the
|
|
378
|
+
// JS hook is stale but at least it's directional).
|
|
379
|
+
batchCaptureOrientation = options.getString("captureOrientation")
|
|
380
|
+
?: "portrait"
|
|
381
|
+
// P3-F — configure the shared-C++ KeyframeGate for
|
|
382
|
+
// this capture. Same knob set + defaults as iOS:
|
|
383
|
+
// overlapThreshold default 0.4 (40% new content)
|
|
384
|
+
// maxCount default 6
|
|
385
|
+
// Both clamped to safe ranges that iOS also uses (see
|
|
386
|
+
// IncrementalStitcher.swift:608-615).
|
|
387
|
+
val threshold = configOverrides
|
|
388
|
+
?.getDoubleOrDefault("keyframeOverlapThreshold", 0.4) ?: 0.4
|
|
389
|
+
keyframeGate.overlapThreshold = threshold.coerceIn(0.10, 0.80)
|
|
390
|
+
keyframeGate.maxCount = batchKeyframeMaxCount.coerceIn(3, 10)
|
|
391
|
+
|
|
392
|
+
// 2026-05-14 — thread flow-strategy tunables through to the
|
|
393
|
+
// shared C++ gate. Before this commit the Android JNI was
|
|
394
|
+
// missing setFlowNoveltyPercentile + setFlowMaxTranslationM
|
|
395
|
+
// bindings (iOS-only via KeyframeGateBridge), which meant
|
|
396
|
+
// operators flipping these in Settings only affected iOS
|
|
397
|
+
// captures. Now both platforms honour them.
|
|
398
|
+
val pctile = configOverrides
|
|
399
|
+
?.getDoubleOrDefault("flowNoveltyPercentile", 0.85) ?: 0.85
|
|
400
|
+
keyframeGate.flowNoveltyPercentile = pctile.coerceIn(0.50, 0.99)
|
|
401
|
+
// Settings UI exposes flowMaxTranslationCm in CENTIMETRES;
|
|
402
|
+
// C++ API is in METRES. Convert. 0 = disabled.
|
|
403
|
+
val txBudgetCm = configOverrides
|
|
404
|
+
?.getDoubleOrDefault("flowMaxTranslationCm", 0.0) ?: 0.0
|
|
405
|
+
keyframeGate.flowMaxTranslationM = (txBudgetCm / 100.0).coerceAtLeast(0.0)
|
|
406
|
+
|
|
407
|
+
// 2026-05-14 — non-AR mode opt-out for angular fallback.
|
|
408
|
+
// captureSource ∈ {wide, ultrawide} means the host is using
|
|
409
|
+
// vision-camera (no ARKit/ARCore pose). Disable the gate's
|
|
410
|
+
// angular fallback so it doesn't compute on garbage pose.
|
|
411
|
+
val captureSource = configOverrides?.getString("captureSource") ?: "auto"
|
|
412
|
+
val isNonAR = (captureSource == "wide" || captureSource == "ultrawide")
|
|
413
|
+
keyframeGate.disableAngularFallback = isNonAR
|
|
414
|
+
|
|
415
|
+
keyframeGate.enabled = true
|
|
416
|
+
keyframeGate.reset()
|
|
417
|
+
} else if (isFirstwins) {
|
|
418
|
+
batchKeyframeMode = false
|
|
419
|
+
batchKeyframePaths.clear()
|
|
420
|
+
keyframeGate.enabled = false // gate is batch-only; off for live engines
|
|
421
|
+
firstwinsEngine = IncrementalFirstwinsEngine(
|
|
422
|
+
composeWidth = composeW,
|
|
423
|
+
composeHeight = composeH,
|
|
424
|
+
canvasWidth = canvasW,
|
|
425
|
+
canvasHeight = canvasH,
|
|
426
|
+
snapshotJpegQuality = snapQ,
|
|
427
|
+
snapshotEveryNAccepts = snapN,
|
|
428
|
+
frameRotationDegrees = rotation,
|
|
429
|
+
useRectilinear = useRectilinear,
|
|
430
|
+
// Critic #27 fix: writable app-sandbox dir for
|
|
431
|
+
// live-snapshot JPEGs. java.io.tmpdir resolves to
|
|
432
|
+
// /data/local/tmp on Android (rooted-only).
|
|
433
|
+
snapshotCacheDir = reactContext.cacheDir.absolutePath,
|
|
434
|
+
)
|
|
435
|
+
engine = null
|
|
436
|
+
} else {
|
|
437
|
+
batchKeyframeMode = false
|
|
438
|
+
batchKeyframePaths.clear()
|
|
439
|
+
keyframeGate.enabled = false // gate is batch-only; off for hybrid engine
|
|
440
|
+
engine = IncrementalEngine(
|
|
441
|
+
composeWidth = composeW,
|
|
442
|
+
composeHeight = composeH,
|
|
443
|
+
canvasWidth = canvasW,
|
|
444
|
+
canvasHeight = canvasH,
|
|
445
|
+
featherPx = featherP,
|
|
446
|
+
snapshotJpegQuality = snapQ,
|
|
447
|
+
snapshotEveryNAccepts = snapN,
|
|
448
|
+
frameRotationDegrees = rotation,
|
|
449
|
+
)
|
|
450
|
+
firstwinsEngine = null
|
|
451
|
+
}
|
|
452
|
+
// Engage the ARCameraView's per-frame ingestion path if a
|
|
453
|
+
// view is mounted — this is what gives Android parity
|
|
454
|
+
// with iOS' ARSession-driven path. No-op when the view
|
|
455
|
+
// isn't mounted (host is using vision-camera + the gyro
|
|
456
|
+
// driver from useIncrementalAndroidDriver instead).
|
|
457
|
+
arCameraViewRef?.setIncrementalIngestionActive(true)
|
|
458
|
+
|
|
459
|
+
// ── P3-G diagnostic ──────────────────────────────────
|
|
460
|
+
// Surfaces start() state so we can see in logcat: (a)
|
|
461
|
+
// engine mode actually selected, (b) batchKeyframeMode
|
|
462
|
+
// flag, (c) KeyframeGate config, (d) whether the AR
|
|
463
|
+
// view is bound. Each of these is a potential failure
|
|
464
|
+
// point for the "0 keyframes captured" symptom.
|
|
465
|
+
android.util.Log.i(
|
|
466
|
+
"IncrementalStitcher",
|
|
467
|
+
"start() ENTRY: engineMode=$engineMode " +
|
|
468
|
+
"batchKeyframeMode=$batchKeyframeMode " +
|
|
469
|
+
"gate.enabled=${keyframeGate.enabled} " +
|
|
470
|
+
"gate.maxCount=${keyframeGate.maxCount} " +
|
|
471
|
+
"gate.threshold=${keyframeGate.overlapThreshold} " +
|
|
472
|
+
"arCameraViewBound=${arCameraViewRef != null} " +
|
|
473
|
+
"isRunning=${isRunning.get()}",
|
|
474
|
+
)
|
|
475
|
+
|
|
476
|
+
val map = Arguments.createMap()
|
|
477
|
+
map.putBoolean("ok", true)
|
|
478
|
+
promise.resolve(map)
|
|
479
|
+
} catch (t: Throwable) {
|
|
480
|
+
isRunning.set(false)
|
|
481
|
+
promise.reject("incremental-start-failed", t.message, t)
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Feed one frame at a JPEG path into the engine. Pose inputs
|
|
487
|
+
* drive the same FoV-overlap gate as iOS. When a pose source
|
|
488
|
+
* isn't available pass yaw=0, pitch=0, fovHorizDegrees=0 — the
|
|
489
|
+
* engine treats fov<=0 as a sentinel for "no intrinsics" and
|
|
490
|
+
* substitutes a 65° default, so frames will still be processed,
|
|
491
|
+
* just less gated.
|
|
492
|
+
*/
|
|
493
|
+
/**
|
|
494
|
+
* Copy a (non-persistent) source JPEG to a persistent per-keyframe
|
|
495
|
+
* path under the React context's cache dir. The ARCameraView's
|
|
496
|
+
* forwardToIncremental writes every frame to a SINGLE reused tmp
|
|
497
|
+
* file (rlis-arframe.jpg) — adequate for the live engines that
|
|
498
|
+
* decode synchronously, but the batch-keyframe collector
|
|
499
|
+
* accumulates paths for stitching at finalize time, so each
|
|
500
|
+
* keyframe needs its own stable file.
|
|
501
|
+
*
|
|
502
|
+
* Returns the absolute path of the destination on success, or
|
|
503
|
+
* null if the copy failed. Cost ≈ 3-5 ms for a 1080p JPEG on
|
|
504
|
+
* iPhone 16 / Galaxy A35 class hardware.
|
|
505
|
+
*
|
|
506
|
+
* Naming: `rlis-keyframe-{N}.jpg` where N is the next slot index
|
|
507
|
+
* (= batchKeyframePaths.size). Survives until either the next
|
|
508
|
+
* batch-keyframe capture overwrites the same slot or the OS
|
|
509
|
+
* cleans the cache dir. iOS counterpart writes per-session
|
|
510
|
+
* uuid-dirs via OpenCVKeyframeCollector — that's deeper parity
|
|
511
|
+
* for a Phase 3 follow-up; this MVP is just enough to make
|
|
512
|
+
* batch-keyframe work end-to-end on Android.
|
|
513
|
+
*/
|
|
514
|
+
private fun copyKeyframeToStore(srcPath: String): String? {
|
|
515
|
+
// V16 Phase 2 (Android Fix-1) — write into the per-session
|
|
516
|
+
// subdir created by start(). If start() didn't run (defensive
|
|
517
|
+
// — should never happen on the live ingest path), the
|
|
518
|
+
// captureSessionDir is null and we drop the frame; the older
|
|
519
|
+
// "rlis-keyframe-{N}.jpg in cacheDir" fallback is GONE because
|
|
520
|
+
// it was the source of the cross-capture cache bug.
|
|
521
|
+
val dir = captureSessionDir
|
|
522
|
+
if (dir == null) {
|
|
523
|
+
android.util.Log.w(
|
|
524
|
+
"IncrementalStitcher",
|
|
525
|
+
"copyKeyframeToStore: captureSessionDir is null — " +
|
|
526
|
+
"start() should have created it; dropping frame",
|
|
527
|
+
)
|
|
528
|
+
return null
|
|
529
|
+
}
|
|
530
|
+
val destFile = java.io.File(dir, "keyframe-${batchKeyframePaths.size}.jpg")
|
|
531
|
+
return try {
|
|
532
|
+
java.io.File(srcPath).copyTo(destFile, overwrite = true).absolutePath
|
|
533
|
+
} catch (e: Exception) {
|
|
534
|
+
android.util.Log.w(
|
|
535
|
+
"IncrementalStitcher",
|
|
536
|
+
"copyKeyframeToStore: failed to copy $srcPath → " +
|
|
537
|
+
"${destFile.absolutePath}: ${e.message}",
|
|
538
|
+
e,
|
|
539
|
+
)
|
|
540
|
+
null
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// ── V16 Phase 1 → P3-F migration note ────────────────────────
|
|
545
|
+
// The frame-counter placeholder gate `handleBatchKeyframeFrame`
|
|
546
|
+
// that lived here has been REMOVED. Both the AR-driven path
|
|
547
|
+
// (ingestFromARCameraView) and the vision-camera fallback path
|
|
548
|
+
// (processFrameAtPath) now route through the shared-C++
|
|
549
|
+
// `KeyframeGate` (cpp/keyframe_gate.{hpp,cpp}) — same algorithm
|
|
550
|
+
// iOS has used since the V16 ship. See
|
|
551
|
+
// `private val keyframeGate = KeyframeGate()` above for the
|
|
552
|
+
// instance + lifetime.
|
|
553
|
+
//
|
|
554
|
+
// `batchKeyframeAcceptStride` is no longer consulted (the proper
|
|
555
|
+
// gate uses pose-driven overlap, not a frame-counter stride);
|
|
556
|
+
// the field is kept around for now because removing it would
|
|
557
|
+
// touch unrelated init/serialization paths. Wire it back in if
|
|
558
|
+
// we ever add a "force every Nth frame regardless of overlap"
|
|
559
|
+
// override.
|
|
560
|
+
|
|
561
|
+
@ReactMethod
|
|
562
|
+
fun processFrameAtPath(options: ReadableMap, promise: Promise) {
|
|
563
|
+
val hybrid = this.engine
|
|
564
|
+
val firstwins = this.firstwinsEngine
|
|
565
|
+
// batch-keyframe mode runs without a live engine — handle it
|
|
566
|
+
// up-front before the null-check rejects.
|
|
567
|
+
//
|
|
568
|
+
// P3-F: this path uses the same shared-C++ KeyframeGate as
|
|
569
|
+
// the AR view path, but with translation = 0 (gyro-derived
|
|
570
|
+
// poses have only rotation, not position). The gate's
|
|
571
|
+
// internal logic detects the missing plane and uses the
|
|
572
|
+
// camera-forward angular-delta fallback automatically.
|
|
573
|
+
if (batchKeyframeMode) {
|
|
574
|
+
val path = options.getString("path")
|
|
575
|
+
?: return promise.reject("invalid-options", "path required")
|
|
576
|
+
val pose = RNSARFramePose(
|
|
577
|
+
tx = options.getDoubleOrDefault("tx", 0.0) ?: 0.0,
|
|
578
|
+
ty = options.getDoubleOrDefault("ty", 0.0) ?: 0.0,
|
|
579
|
+
tz = options.getDoubleOrDefault("tz", 0.0) ?: 0.0,
|
|
580
|
+
qx = options.getDoubleOrDefault("qx", 0.0) ?: 0.0,
|
|
581
|
+
qy = options.getDoubleOrDefault("qy", 0.0) ?: 0.0,
|
|
582
|
+
qz = options.getDoubleOrDefault("qz", 0.0) ?: 0.0,
|
|
583
|
+
qw = options.getDoubleOrDefault("qw", 1.0) ?: 1.0,
|
|
584
|
+
fx = options.getDoubleOrDefault("fx", 1000.0) ?: 1000.0,
|
|
585
|
+
fy = options.getDoubleOrDefault("fy", 1000.0) ?: 1000.0,
|
|
586
|
+
cx = options.getDoubleOrDefault("cx", 540.0) ?: 540.0,
|
|
587
|
+
cy = options.getDoubleOrDefault("cy", 960.0) ?: 960.0,
|
|
588
|
+
imageWidth = options.getIntOrDefault("imageWidth", 1080),
|
|
589
|
+
imageHeight = options.getIntOrDefault("imageHeight", 1920),
|
|
590
|
+
timestampMs = 0.0,
|
|
591
|
+
trackingState = RNSARSession.TRACKING_TRACKING,
|
|
592
|
+
)
|
|
593
|
+
// Vision-camera path: no plane available (gyro can't fit
|
|
594
|
+
// planes). Pass null → C++ uses angular fallback.
|
|
595
|
+
val decision = keyframeGate.evaluate(pose, null)
|
|
596
|
+
val result = Arguments.createMap()
|
|
597
|
+
// Outcome mapping for iOS-parity JS contract:
|
|
598
|
+
// 1 = accepted, 2 = rejected (gate), 3 = rejected (cap).
|
|
599
|
+
// C++ enum → outcome int:
|
|
600
|
+
val outcome = when {
|
|
601
|
+
decision.accept -> 1
|
|
602
|
+
decision.reason == "max-reached" -> 3
|
|
603
|
+
else -> 2
|
|
604
|
+
}
|
|
605
|
+
result.putInt("outcome", outcome)
|
|
606
|
+
if (decision.accept) {
|
|
607
|
+
batchKeyframePaths.add(path) // vision-camera path
|
|
608
|
+
// gives us a unique
|
|
609
|
+
// per-snapshot file
|
|
610
|
+
// already (no copy
|
|
611
|
+
// needed).
|
|
612
|
+
// Emit the same state event the AR path emits so
|
|
613
|
+
// the JS LiveFrameStrip + "Keyframes n/max" pill
|
|
614
|
+
// work identically on the vision-camera fallback
|
|
615
|
+
// path.
|
|
616
|
+
emitBatchKeyframeAcceptedState(
|
|
617
|
+
thumbnailPath = path,
|
|
618
|
+
keyframeIndex = batchKeyframePaths.size - 1,
|
|
619
|
+
keyframeCount = batchKeyframePaths.size,
|
|
620
|
+
keyframeMax = keyframeGate.maxCount,
|
|
621
|
+
isLandscape = pose.imageWidth >= pose.imageHeight,
|
|
622
|
+
)
|
|
623
|
+
}
|
|
624
|
+
result.putInt("acceptedCount", batchKeyframePaths.size)
|
|
625
|
+
promise.resolve(result)
|
|
626
|
+
return
|
|
627
|
+
}
|
|
628
|
+
if (hybrid == null && firstwins == null) {
|
|
629
|
+
return promise.reject(
|
|
630
|
+
"incremental-not-running",
|
|
631
|
+
"Call start() before processFrameAtPath().",
|
|
632
|
+
)
|
|
633
|
+
}
|
|
634
|
+
val path = options.getString("path")
|
|
635
|
+
?: return promise.reject("invalid-options", "path required")
|
|
636
|
+
val yaw = options.getDoubleOrDefault("yaw", 0.0)
|
|
637
|
+
val pitch = options.getDoubleOrDefault("pitch", 0.0)
|
|
638
|
+
val fovH = options.getDoubleOrDefault("fovHorizDegrees", 65.0)
|
|
639
|
+
val fovV = options.getDoubleOrDefault("fovVertDegrees", 50.0)
|
|
640
|
+
// V6 pose-driven params. Defaults removed per critic finding
|
|
641
|
+
// #3: previously qw=1.0 default meant frames without explicit
|
|
642
|
+
// quaternion produced an identity rotation, and EVERY
|
|
643
|
+
// subsequent frame had R_rel = R_first^T (constant), so
|
|
644
|
+
// strip placement never advanced and `acceptedCount` froze
|
|
645
|
+
// at 1 after the first frame. Now every quaternion field is
|
|
646
|
+
// required; missing → reject as RejectedAlignmentLost so the
|
|
647
|
+
// gyro driver upstream notices instantly.
|
|
648
|
+
if (!options.hasKey("qx") || !options.hasKey("qy")
|
|
649
|
+
|| !options.hasKey("qz") || !options.hasKey("qw")) {
|
|
650
|
+
return promise.reject(
|
|
651
|
+
"invalid-options",
|
|
652
|
+
"qx/qy/qz/qw all required (no identity-quaternion fallback)",
|
|
653
|
+
)
|
|
654
|
+
}
|
|
655
|
+
val qx = options.getDouble("qx")
|
|
656
|
+
val qy = options.getDouble("qy")
|
|
657
|
+
val qz = options.getDouble("qz")
|
|
658
|
+
val qw = options.getDouble("qw")
|
|
659
|
+
val fx = options.getDoubleOrDefault("fx", 0.0)
|
|
660
|
+
val fy = options.getDoubleOrDefault("fy", 0.0)
|
|
661
|
+
val cx = options.getDoubleOrDefault("cx", 0.0)
|
|
662
|
+
val cy = options.getDoubleOrDefault("cy", 0.0)
|
|
663
|
+
val imageWidth = options.getIntOrDefault("imageWidth", 0)
|
|
664
|
+
val imageHeight = options.getIntOrDefault("imageHeight", 0)
|
|
665
|
+
val trackingPoor = options.getBooleanOrDefault("trackingPoor", false)
|
|
666
|
+
|
|
667
|
+
workScope.launch {
|
|
668
|
+
// Critic #4 fix: re-check isRunning synchronously here in
|
|
669
|
+
// case finalize/cancel ran on the JS thread between the
|
|
670
|
+
// null-check above and this dispatch landing. Skip the
|
|
671
|
+
// ingest if we're no longer running — matches iOS' V12.1
|
|
672
|
+
// pattern (synchronous-stop + worker re-check).
|
|
673
|
+
if (!isRunning.get()) {
|
|
674
|
+
promise.resolve(Arguments.createMap().apply { putInt("outcome", -1) })
|
|
675
|
+
return@launch
|
|
676
|
+
}
|
|
677
|
+
try {
|
|
678
|
+
val telemetry: FrameTelemetry
|
|
679
|
+
val state: WritableMap?
|
|
680
|
+
val accepted: Int
|
|
681
|
+
if (firstwins != null) {
|
|
682
|
+
telemetry = firstwins.addFrameAtPath(
|
|
683
|
+
path = path,
|
|
684
|
+
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
685
|
+
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
686
|
+
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
687
|
+
yaw = yaw, pitch = pitch,
|
|
688
|
+
fovHorizDegrees = fovH, fovVertDegrees = fovV,
|
|
689
|
+
trackingPoor = trackingPoor,
|
|
690
|
+
)
|
|
691
|
+
state = firstwins.snapshotIfDue(telemetry)
|
|
692
|
+
accepted = firstwins.acceptedCount
|
|
693
|
+
} else {
|
|
694
|
+
telemetry = hybrid!!.addFrameAtPath(
|
|
695
|
+
path = path,
|
|
696
|
+
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
697
|
+
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
698
|
+
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
699
|
+
yaw = yaw, pitch = pitch,
|
|
700
|
+
fovHorizDegrees = fovH, fovVertDegrees = fovV,
|
|
701
|
+
trackingPoor = trackingPoor,
|
|
702
|
+
)
|
|
703
|
+
state = hybrid.snapshotIfDue(telemetry)
|
|
704
|
+
accepted = hybrid.acceptedCount
|
|
705
|
+
}
|
|
706
|
+
emitState(state)
|
|
707
|
+
val result = Arguments.createMap()
|
|
708
|
+
result.putInt("outcome", telemetry.outcome.ordinal)
|
|
709
|
+
result.putDouble("confidence", telemetry.confidence)
|
|
710
|
+
result.putDouble("overlapPercent", telemetry.overlapPercent)
|
|
711
|
+
result.putDouble("processingMs", telemetry.processingMs)
|
|
712
|
+
result.putInt("acceptedCount", accepted)
|
|
713
|
+
promise.resolve(result)
|
|
714
|
+
} catch (t: Throwable) {
|
|
715
|
+
promise.reject("incremental-process-failed", t.message, t)
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
@ReactMethod
|
|
721
|
+
fun finalize(options: ReadableMap, promise: Promise) {
|
|
722
|
+
val hybrid = this.engine
|
|
723
|
+
val firstwins = this.firstwinsEngine
|
|
724
|
+
if (hybrid == null && firstwins == null && !batchKeyframeMode) {
|
|
725
|
+
return promise.reject(
|
|
726
|
+
"incremental-not-running",
|
|
727
|
+
"No active capture — call start() first.",
|
|
728
|
+
)
|
|
729
|
+
}
|
|
730
|
+
val outputPathOpt = options.getString("outputPath") ?: ""
|
|
731
|
+
val outputPath = if (outputPathOpt.isEmpty()) {
|
|
732
|
+
File(reactContext.cacheDir, "RNImageStitcherIncremental-${System.nanoTime()}.jpg").absolutePath
|
|
733
|
+
} else {
|
|
734
|
+
outputPathOpt
|
|
735
|
+
}
|
|
736
|
+
val quality = options.getIntOrDefault("quality", 90)
|
|
737
|
+
// 2026-05-18 (iOS cross-orientation fix; symmetric on Android) —
|
|
738
|
+
// JS may pass a fresh deviceOrientation at finalize time; if
|
|
739
|
+
// so, override batchCaptureOrientation BEFORE we snapshot it
|
|
740
|
+
// for the stitcher. Empty/missing → keep legacy start-time
|
|
741
|
+
// value. Android cross-orientation was already working per
|
|
742
|
+
// user test (likely because users tested fewer rotation
|
|
743
|
+
// sequences here), but propagating the fresh value uniformly
|
|
744
|
+
// closes the same hole iOS had.
|
|
745
|
+
val freshOrientationOpt = options.getString("captureOrientation") ?: ""
|
|
746
|
+
if (freshOrientationOpt.isNotEmpty()) {
|
|
747
|
+
batchCaptureOrientation = freshOrientationOpt
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
// Disengage the ARCameraView ingestion path FIRST so no late
|
|
751
|
+
// frames slip into the engine while we serialize the canvas.
|
|
752
|
+
arCameraViewRef?.setIncrementalIngestionActive(false)
|
|
753
|
+
// Critic #4 fix: synchronously flip isRunning=false BEFORE
|
|
754
|
+
// dispatching the finalize body, so any in-flight
|
|
755
|
+
// processFrameAtPath workers that are about to launch will
|
|
756
|
+
// bail at the re-check (see processFrameAtPath above).
|
|
757
|
+
// Matches iOS V12.1 fix.
|
|
758
|
+
isRunning.set(false)
|
|
759
|
+
|
|
760
|
+
// V16 batch-keyframe finalize: snapshot the keyframe state
|
|
761
|
+
// synchronously under the same "stop ingestion before
|
|
762
|
+
// dispatching" pattern, then null out the live state.
|
|
763
|
+
val wasBatchKeyframe = batchKeyframeMode
|
|
764
|
+
val keyframePathsSnapshot = batchKeyframePaths.toList()
|
|
765
|
+
val captureOrientationSnapshot = batchCaptureOrientation
|
|
766
|
+
val warperTypeSnapshot = batchWarperType
|
|
767
|
+
val blenderTypeSnapshot = batchBlenderType
|
|
768
|
+
val seamFinderTypeSnapshot = batchSeamFinderType
|
|
769
|
+
val useInscribedRectCropSnapshot = batchUseInscribedRectCrop
|
|
770
|
+
// 2026-05-14 — resolve stitch-mode auto → concrete mode.
|
|
771
|
+
// 'auto' uses the pose deltas accumulated during capture to
|
|
772
|
+
// pick PANORAMA (rotation-heavy) vs SCANS (translation-heavy).
|
|
773
|
+
//
|
|
774
|
+
// Heuristic (matches design doc 2026-05-13-stitch-pipeline-mode-selection):
|
|
775
|
+
// translation_score = ||t_last − t_first|| (meters) / 0.10
|
|
776
|
+
// rotation_score = angle(fwd_last, fwd_first) (radians) / 1.00
|
|
777
|
+
// ratio = translation_score / (translation_score + rotation_score)
|
|
778
|
+
// ratio ≥ 0.55 → SCANS (translation-dominant)
|
|
779
|
+
// ratio < 0.55 → PANORAMA (rotation-dominant)
|
|
780
|
+
// missing poses → SCANS (safer default: bounded canvas).
|
|
781
|
+
//
|
|
782
|
+
// Why biased toward SCANS: PANORAMA on translation diverges
|
|
783
|
+
// catastrophically (3.2 GB compositing canvas observed
|
|
784
|
+
// 2026-05-14 → lmkd kill). SCANS on rotation degrades
|
|
785
|
+
// gracefully (slightly worse seams, never blows up).
|
|
786
|
+
val firstPose = batchFirstAcceptedPose
|
|
787
|
+
val lastPose = batchLastAcceptedPose
|
|
788
|
+
val stitchModeResolved: String = when (batchStitchMode) {
|
|
789
|
+
"panorama" -> "panorama"
|
|
790
|
+
"scans" -> "scans"
|
|
791
|
+
else -> resolveStitchModeAuto(firstPose, lastPose)
|
|
792
|
+
}
|
|
793
|
+
android.util.Log.i(
|
|
794
|
+
"IncrementalStitcher",
|
|
795
|
+
"finalize stitch-mode: configured=$batchStitchMode resolved=$stitchModeResolved " +
|
|
796
|
+
"firstPose=${firstPose != null} lastPose=${lastPose != null}",
|
|
797
|
+
)
|
|
798
|
+
batchKeyframeMode = false
|
|
799
|
+
batchKeyframePaths.clear()
|
|
800
|
+
batchKeyframeFrameCounter = 0
|
|
801
|
+
batchFirstAcceptedPose = null
|
|
802
|
+
batchLastAcceptedPose = null
|
|
803
|
+
|
|
804
|
+
// Null the bridge refs synchronously NOW so any worker that's
|
|
805
|
+
// about to run sees them as gone (V12.1 pattern). We keep
|
|
806
|
+
// local refs to do the actual finalize.
|
|
807
|
+
engine = null
|
|
808
|
+
firstwinsEngine = null
|
|
809
|
+
|
|
810
|
+
workScope.launch {
|
|
811
|
+
try {
|
|
812
|
+
val map = Arguments.createMap()
|
|
813
|
+
if (wasBatchKeyframe) {
|
|
814
|
+
// V16 batch-keyframe: hand keyframe paths to the
|
|
815
|
+
// JNI shim for one-shot cv::Stitcher processing.
|
|
816
|
+
if (keyframePathsSnapshot.size < 2) {
|
|
817
|
+
throw IllegalStateException(
|
|
818
|
+
"Batch-keyframe finalize: only " +
|
|
819
|
+
"${keyframePathsSnapshot.size} keyframe(s) " +
|
|
820
|
+
"captured — at least 2 required."
|
|
821
|
+
)
|
|
822
|
+
}
|
|
823
|
+
// Use the static `bridgeInstance` accessor on
|
|
824
|
+
// BatchStitcher rather than
|
|
825
|
+
// reactContext.getNativeModule — the latter
|
|
826
|
+
// returns null under bridgeless / new-architecture
|
|
827
|
+
// mode even for legacy-registered modules.
|
|
828
|
+
// Empirically: getNativeModule failed on Galaxy
|
|
829
|
+
// A35 with `BatchStitcher module not
|
|
830
|
+
// registered`, despite the module being present
|
|
831
|
+
// in RNImageStitcherPackage.createNativeModules.
|
|
832
|
+
// Same pattern that already works for
|
|
833
|
+
// IncrementalStitcher.bridgeInstance.
|
|
834
|
+
val stitcher = BatchStitcher.bridgeInstance
|
|
835
|
+
?: throw IllegalStateException(
|
|
836
|
+
"BatchStitcher.bridgeInstance is null " +
|
|
837
|
+
"— module hasn't been instantiated yet. " +
|
|
838
|
+
"Check RNImageStitcherPackage registration."
|
|
839
|
+
)
|
|
840
|
+
val dims = stitcher.stitchSync(
|
|
841
|
+
keyframePathsSnapshot.toTypedArray(),
|
|
842
|
+
outputPath,
|
|
843
|
+
quality,
|
|
844
|
+
warperTypeSnapshot,
|
|
845
|
+
blenderTypeSnapshot,
|
|
846
|
+
seamFinderTypeSnapshot,
|
|
847
|
+
captureOrientationSnapshot,
|
|
848
|
+
useInscribedRectCropSnapshot,
|
|
849
|
+
stitchMode = stitchModeResolved,
|
|
850
|
+
)
|
|
851
|
+
// 2026-05-15 (D) — dims layout from native JNI:
|
|
852
|
+
// [0] width, [1] height, [2] framesRequested,
|
|
853
|
+
// [3] framesIncluded, [4] finalThresholdMilli
|
|
854
|
+
// The framesIncluded count is the post-
|
|
855
|
+
// leaveBiggestComponent retained subset. Any
|
|
856
|
+
// delta from acceptedCount = frames the stitcher
|
|
857
|
+
// dropped due to weak feature-matching confidence.
|
|
858
|
+
// Surfaced to JS so the capture-screen UX can
|
|
859
|
+
// show "Stitched N of M frames" when drops > 0.
|
|
860
|
+
val framesRequested =
|
|
861
|
+
if (dims.size > 2) dims[2] else keyframePathsSnapshot.size
|
|
862
|
+
val framesIncluded =
|
|
863
|
+
if (dims.size > 3) dims[3] else keyframePathsSnapshot.size
|
|
864
|
+
val finalConfidenceThresh =
|
|
865
|
+
if (dims.size > 4) dims[4].toDouble() / 1000.0 else -1.0
|
|
866
|
+
map.putString("panoramaPath", outputPath)
|
|
867
|
+
map.putInt("width", dims[0])
|
|
868
|
+
map.putInt("height", dims[1])
|
|
869
|
+
map.putInt("acceptedCount", keyframePathsSnapshot.size)
|
|
870
|
+
map.putInt("framesRequested", framesRequested)
|
|
871
|
+
map.putInt("framesIncluded", framesIncluded)
|
|
872
|
+
map.putInt("framesDropped", framesRequested - framesIncluded)
|
|
873
|
+
map.putDouble("finalConfidenceThresh", finalConfidenceThresh)
|
|
874
|
+
} else if (firstwins != null) {
|
|
875
|
+
val snap = firstwins.finalize(outputPath, quality)
|
|
876
|
+
?: throw IllegalStateException("firstwins.finalize returned null")
|
|
877
|
+
map.putString("panoramaPath", snap.panoramaPath)
|
|
878
|
+
map.putInt("width", snap.width)
|
|
879
|
+
map.putInt("height", snap.height)
|
|
880
|
+
map.putInt("acceptedCount", snap.acceptedCount)
|
|
881
|
+
// Critic #22 fix: explicit native-buffer release.
|
|
882
|
+
firstwins.release()
|
|
883
|
+
} else {
|
|
884
|
+
val snap = hybrid!!.finalize(outputPath, quality)
|
|
885
|
+
map.putString("panoramaPath", snap.panoramaPath)
|
|
886
|
+
map.putInt("width", snap.width)
|
|
887
|
+
map.putInt("height", snap.height)
|
|
888
|
+
map.putInt("acceptedCount", snap.acceptedCount)
|
|
889
|
+
hybrid.release()
|
|
890
|
+
// 2026-05-16 — realtime+batch fusion (Option A
|
|
891
|
+
// "Replace on completion") hook. The live
|
|
892
|
+
// panorama has been written to outputPath; now
|
|
893
|
+
// fire-and-forget an async refinement on the
|
|
894
|
+
// engine's accepted keyframes via the shared C++
|
|
895
|
+
// stitcher.
|
|
896
|
+
//
|
|
897
|
+
// Today's `IncrementalEngine` (the hybrid live
|
|
898
|
+
// engine) does NOT retain per-frame JPEGs — it
|
|
899
|
+
// paints into a single persistent canvas Mat
|
|
900
|
+
// that's torn down by `release()` above. So the
|
|
901
|
+
// keyframe-paths list passed to runHybridAutoRefine
|
|
902
|
+
// is empty for the hybrid branch, which means
|
|
903
|
+
// the auto-trigger detects "< 2 keyframes on
|
|
904
|
+
// disk" and emits `isRefining=false` without
|
|
905
|
+
// running cv::Stitcher. Per the prompt's
|
|
906
|
+
// "no-op when no keyframes on disk" constraint.
|
|
907
|
+
//
|
|
908
|
+
// When a future change wires the hybrid engine
|
|
909
|
+
// to a keyframe collector (parallel to iOS'
|
|
910
|
+
// OpenCVKeyframeCollector), the same hook will
|
|
911
|
+
// light up automatically — just pass the
|
|
912
|
+
// populated list here.
|
|
913
|
+
val keyframePathsForHybrid: List<String> = emptyList()
|
|
914
|
+
val refinedOutputPath = refinedPathFromLive(outputPath)
|
|
915
|
+
runHybridAutoRefine(
|
|
916
|
+
framePaths = keyframePathsForHybrid,
|
|
917
|
+
refinedOutputPath = refinedOutputPath,
|
|
918
|
+
captureOrientation = captureOrientationSnapshot,
|
|
919
|
+
warperType = warperTypeSnapshot,
|
|
920
|
+
blenderType = blenderTypeSnapshot,
|
|
921
|
+
seamFinderType = seamFinderTypeSnapshot,
|
|
922
|
+
useInscribedRectCrop = useInscribedRectCropSnapshot,
|
|
923
|
+
)
|
|
924
|
+
}
|
|
925
|
+
map.putInt("droppedBackpressure", 0)
|
|
926
|
+
promise.resolve(map)
|
|
927
|
+
} catch (t: Throwable) {
|
|
928
|
+
firstwins?.release()
|
|
929
|
+
hybrid?.release()
|
|
930
|
+
promise.reject("incremental-finalize-failed", t.message, t)
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
@ReactMethod
|
|
936
|
+
fun cancel(promise: Promise) {
|
|
937
|
+
// Critic #4 fix: synchronously flip isRunning + null engine
|
|
938
|
+
// refs BEFORE releasing. Any in-flight worker bails at the
|
|
939
|
+
// re-check before touching the now-null engine. Matches
|
|
940
|
+
// iOS V12.1 cancel path.
|
|
941
|
+
arCameraViewRef?.setIncrementalIngestionActive(false)
|
|
942
|
+
isRunning.set(false)
|
|
943
|
+
val hybrid = engine
|
|
944
|
+
val firstwins = firstwinsEngine
|
|
945
|
+
engine = null
|
|
946
|
+
firstwinsEngine = null
|
|
947
|
+
// V16 Phase 2 (Android Fix-1) — clean up the per-session
|
|
948
|
+
// batch-keyframe subdir. iOS-parity: cancel removes the
|
|
949
|
+
// session's saved JPEGs because the operator explicitly
|
|
950
|
+
// aborted, so the keyframes aren't worth preserving for
|
|
951
|
+
// reprocessing. (Successful finalize keeps them — see the
|
|
952
|
+
// ivar declaration.)
|
|
953
|
+
val sessionDirToCleanup = captureSessionDir
|
|
954
|
+
captureSessionDir = null
|
|
955
|
+
batchKeyframeMode = false
|
|
956
|
+
batchKeyframePaths.clear()
|
|
957
|
+
batchKeyframeFrameCounter = 0
|
|
958
|
+
// Defer engine release + session-dir cleanup onto the work
|
|
959
|
+
// queue so we don't race with an ingest that already passed
|
|
960
|
+
// the null-check and is mid-execution on a captured local
|
|
961
|
+
// reference.
|
|
962
|
+
workScope.launch {
|
|
963
|
+
hybrid?.release()
|
|
964
|
+
firstwins?.reset()
|
|
965
|
+
sessionDirToCleanup?.deleteRecursively()
|
|
966
|
+
}
|
|
967
|
+
val map = Arguments.createMap()
|
|
968
|
+
map.putBoolean("ok", true)
|
|
969
|
+
promise.resolve(map)
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
/**
|
|
973
|
+
* Called by `RNSARCameraView` per ARCore frame when it has
|
|
974
|
+
* a fresh JPEG + pose to ingest. Synchronous-feeling from the
|
|
975
|
+
* caller's perspective but actually dispatched onto the engine's
|
|
976
|
+
* own queue so we don't stall the GL render thread. Drops the
|
|
977
|
+
* frame silently if no engine is running (race between view
|
|
978
|
+
* lifecycle and stitcher start/stop).
|
|
979
|
+
*/
|
|
980
|
+
internal fun ingestFromARCameraView(
|
|
981
|
+
path: String,
|
|
982
|
+
tx: Double, ty: Double, tz: Double,
|
|
983
|
+
qx: Double, qy: Double, qz: Double, qw: Double,
|
|
984
|
+
fx: Double, fy: Double, cx: Double, cy: Double,
|
|
985
|
+
imageWidth: Int, imageHeight: Int,
|
|
986
|
+
yaw: Double,
|
|
987
|
+
pitch: Double,
|
|
988
|
+
fovHorizDegrees: Double,
|
|
989
|
+
fovVertDegrees: Double,
|
|
990
|
+
trackingPoor: Boolean,
|
|
991
|
+
) {
|
|
992
|
+
// ── V16 batch-keyframe: AR-driven path ─────────────────────
|
|
993
|
+
//
|
|
994
|
+
// Batch-keyframe mode runs WITHOUT a live engine (engine ==
|
|
995
|
+
// firstwinsEngine == null) — frames accumulate as keyframe
|
|
996
|
+
// paths and the cv::Stitcher pipeline runs at finalize time.
|
|
997
|
+
//
|
|
998
|
+
// P3-F: this branch now calls into the shared-C++
|
|
999
|
+
// KeyframeGate (cpp/keyframe_gate.{hpp,cpp}, same algorithm
|
|
1000
|
+
// iOS uses). The placeholder frame-counter gate that lived
|
|
1001
|
+
// here previously (handleBatchKeyframeFrame) is GONE.
|
|
1002
|
+
//
|
|
1003
|
+
// The ARCameraView's JPEG-encode pipeline writes to a single
|
|
1004
|
+
// REUSED tmp file (rlis-arframe.jpg in cacheDir) — fine for
|
|
1005
|
+
// live engines (decoded into cv::Mat synchronously inside
|
|
1006
|
+
// addFrameAtPath, before next frame arrives), but FATAL for
|
|
1007
|
+
// batch-keyframe (all accepted keyframe paths would point to
|
|
1008
|
+
// the same overwritten file → finalize stitches 6 copies of
|
|
1009
|
+
// the same frame). So we must COPY to a unique path on
|
|
1010
|
+
// accept. We do the gate-evaluate BEFORE the copy so we
|
|
1011
|
+
// skip the ~5 ms JPEG copy on rejected frames.
|
|
1012
|
+
if (batchKeyframeMode) {
|
|
1013
|
+
// Build the POD pose for the gate. tx/ty/tz are passed
|
|
1014
|
+
// through from the AR camera view (camera.pose.tx() etc.);
|
|
1015
|
+
// they're required for the plane-overlap math. Falling
|
|
1016
|
+
// back to the angular path when no plane is latched is
|
|
1017
|
+
// handled internally by the gate (latchedPlane=null arg).
|
|
1018
|
+
val pose = RNSARFramePose(
|
|
1019
|
+
tx = tx, ty = ty, tz = tz,
|
|
1020
|
+
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
1021
|
+
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
1022
|
+
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
1023
|
+
timestampMs = 0.0, // not used by the gate
|
|
1024
|
+
trackingState = RNSARSession.TRACKING_TRACKING,
|
|
1025
|
+
)
|
|
1026
|
+
// Fetch the latched plane (if any) from the AR session
|
|
1027
|
+
// and convert to a column-major 16-float matrix matching
|
|
1028
|
+
// the C++ PlaneTransform layout. ARCore's
|
|
1029
|
+
// Pose.toMatrix(out, offset) gives us exactly that layout
|
|
1030
|
+
// (same as iOS simd_float4x4).
|
|
1031
|
+
val planeMatrix: FloatArray? =
|
|
1032
|
+
RNSARSession.instance?.latchedPlaneTransform?.let { p ->
|
|
1033
|
+
FloatArray(16).also { p.toMatrix(it, 0) }
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
val decision = keyframeGate.evaluate(pose, planeMatrix)
|
|
1037
|
+
|
|
1038
|
+
// ── P3-G diagnostic ──────────────────────────────────
|
|
1039
|
+
// Rate-limit at the same cadence as the plane evaluator
|
|
1040
|
+
// (every 30 frames ≈ 2 Hz at 60Hz frame rate) but ALWAYS
|
|
1041
|
+
// log accepts (rare, important signal).
|
|
1042
|
+
if (decision.accept || (frameIngestLogTick++ % 30 == 0)) {
|
|
1043
|
+
android.util.Log.i(
|
|
1044
|
+
"IncrementalStitcher",
|
|
1045
|
+
"ingestFromARCameraView batch: " +
|
|
1046
|
+
"accept=${decision.accept} reason=${decision.reason} " +
|
|
1047
|
+
"newContent=${"%.3f".format(decision.newContentFraction)} " +
|
|
1048
|
+
"gateCount=${decision.acceptedCount} " +
|
|
1049
|
+
"paths.size=${batchKeyframePaths.size} " +
|
|
1050
|
+
"planeAvailable=${planeMatrix != null}",
|
|
1051
|
+
)
|
|
1052
|
+
}
|
|
1053
|
+
if (!decision.accept) {
|
|
1054
|
+
// Frame rejected by the gate — could be overlap-too-
|
|
1055
|
+
// high (most common), max-reached, or projection-
|
|
1056
|
+
// degenerate. Drop silently; the next frame will be
|
|
1057
|
+
// evaluated. TODO(P1-followup): emit a state event
|
|
1058
|
+
// so the JS UI can surface the reason in the pill.
|
|
1059
|
+
return
|
|
1060
|
+
}
|
|
1061
|
+
// Accepted — copy the (reused) tmp JPEG to a persistent
|
|
1062
|
+
// per-keyframe path so subsequent frames overwriting the
|
|
1063
|
+
// source don't clobber it.
|
|
1064
|
+
val persistentPath = copyKeyframeToStore(path)
|
|
1065
|
+
if (persistentPath == null) {
|
|
1066
|
+
android.util.Log.w(
|
|
1067
|
+
"IncrementalStitcher",
|
|
1068
|
+
"ingestFromARCameraView batch: ACCEPTED but copy FAILED — frame dropped",
|
|
1069
|
+
)
|
|
1070
|
+
// Copy failed — drop the frame. Logged inside
|
|
1071
|
+
// copyKeyframeToStore. Counter was already incremented
|
|
1072
|
+
// inside the gate; that's fine — the next acceptable
|
|
1073
|
+
// frame will still be accepted on its own merits.
|
|
1074
|
+
return
|
|
1075
|
+
}
|
|
1076
|
+
batchKeyframePaths.add(persistentPath)
|
|
1077
|
+
// 2026-05-14 — capture pose at every accept for the
|
|
1078
|
+
// stitch-mode auto-decision at finalize(). First accept
|
|
1079
|
+
// anchors the "from" pose; every subsequent accept
|
|
1080
|
+
// updates "to". Cleared at start(). Order: tx, ty, tz,
|
|
1081
|
+
// qx, qy, qz, qw (same as the Pose struct in C++ KeyframeGate).
|
|
1082
|
+
val poseSnapshot = doubleArrayOf(tx, ty, tz, qx, qy, qz, qw)
|
|
1083
|
+
if (batchFirstAcceptedPose == null) batchFirstAcceptedPose = poseSnapshot
|
|
1084
|
+
batchLastAcceptedPose = poseSnapshot
|
|
1085
|
+
android.util.Log.i(
|
|
1086
|
+
"IncrementalStitcher",
|
|
1087
|
+
"ingestFromARCameraView batch: ACCEPTED keyframe #${batchKeyframePaths.size}" +
|
|
1088
|
+
" → $persistentPath",
|
|
1089
|
+
)
|
|
1090
|
+
// Emit a state event so the JS-side LiveFrameStrip
|
|
1091
|
+
// renders the thumbnail strip + the "Keyframes: n/max"
|
|
1092
|
+
// pill updates in real time. iOS counterpart:
|
|
1093
|
+
// emitBatchKeyframeAcceptedState in
|
|
1094
|
+
// IncrementalStitcher.swift — same field set
|
|
1095
|
+
// so the JS subscriber doesn't branch on platform.
|
|
1096
|
+
emitBatchKeyframeAcceptedState(
|
|
1097
|
+
thumbnailPath = persistentPath,
|
|
1098
|
+
keyframeIndex = batchKeyframePaths.size - 1,
|
|
1099
|
+
keyframeCount = batchKeyframePaths.size,
|
|
1100
|
+
keyframeMax = keyframeGate.maxCount,
|
|
1101
|
+
isLandscape = imageWidth >= imageHeight,
|
|
1102
|
+
)
|
|
1103
|
+
return
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
val hybrid = this.engine
|
|
1107
|
+
val firstwins = this.firstwinsEngine
|
|
1108
|
+
if (hybrid == null && firstwins == null) return
|
|
1109
|
+
workScope.launch {
|
|
1110
|
+
val state: WritableMap? = if (firstwins != null) {
|
|
1111
|
+
val tele = firstwins.addFrameAtPath(
|
|
1112
|
+
path = path,
|
|
1113
|
+
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
1114
|
+
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
1115
|
+
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
1116
|
+
yaw = yaw, pitch = pitch,
|
|
1117
|
+
fovHorizDegrees = fovHorizDegrees,
|
|
1118
|
+
fovVertDegrees = fovVertDegrees,
|
|
1119
|
+
trackingPoor = trackingPoor,
|
|
1120
|
+
)
|
|
1121
|
+
firstwins.snapshotIfDue(tele)
|
|
1122
|
+
} else {
|
|
1123
|
+
val tele = hybrid!!.addFrameAtPath(
|
|
1124
|
+
path = path,
|
|
1125
|
+
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
1126
|
+
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
1127
|
+
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
1128
|
+
yaw = yaw, pitch = pitch,
|
|
1129
|
+
fovHorizDegrees = fovHorizDegrees,
|
|
1130
|
+
fovVertDegrees = fovVertDegrees,
|
|
1131
|
+
trackingPoor = trackingPoor,
|
|
1132
|
+
)
|
|
1133
|
+
hybrid.snapshotIfDue(tele)
|
|
1134
|
+
}
|
|
1135
|
+
emitState(state)
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
@ReactMethod
|
|
1140
|
+
fun getState(promise: Promise) {
|
|
1141
|
+
val state = firstwinsEngine?.lastState ?: engine?.lastState
|
|
1142
|
+
if (state == null) {
|
|
1143
|
+
promise.resolve(null)
|
|
1144
|
+
return
|
|
1145
|
+
}
|
|
1146
|
+
promise.resolve(state)
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
// ── V15.0e — AR plane detection bridge (iOS-parity) ──────────────
|
|
1150
|
+
//
|
|
1151
|
+
// iOS exposes these on the IncrementalStitcherBridge (NOT on the
|
|
1152
|
+
// ARSession module) so the JS code calls
|
|
1153
|
+
// getIncrementalNativeModule().getARPlaneStatus()
|
|
1154
|
+
// (see retailens-capture-sdk/src/stitching/incremental.ts:535).
|
|
1155
|
+
// Both methods delegate to the AR session singleton — same pattern
|
|
1156
|
+
// as iOS' IncrementalStitcherBridge.swift, where the bridge holds
|
|
1157
|
+
// the RN @objc surface and the singleton holds the AR algorithm.
|
|
1158
|
+
|
|
1159
|
+
/**
|
|
1160
|
+
* Poll-friendly plane-status read. Called by JS at 2 Hz while
|
|
1161
|
+
* planeSource = 'ARKitDetected' (the default). When the AR session
|
|
1162
|
+
* native module isn't registered (e.g. plain stitching tests
|
|
1163
|
+
* without an active AR session), returns a stable "searching"
|
|
1164
|
+
* default so the JS gate never throws.
|
|
1165
|
+
*/
|
|
1166
|
+
@ReactMethod
|
|
1167
|
+
fun getARPlaneStatus(promise: Promise) {
|
|
1168
|
+
val session = RNSARSession.instance
|
|
1169
|
+
if (session == null) {
|
|
1170
|
+
// Safe default: no AR session = no plane to lock onto.
|
|
1171
|
+
// Shape MUST match the iOS contract so JS doesn't branch.
|
|
1172
|
+
val map = Arguments.createMap()
|
|
1173
|
+
map.putString("status", "searching")
|
|
1174
|
+
map.putBoolean("hasPlane", false)
|
|
1175
|
+
map.putDouble("bestAlignment", -1.0)
|
|
1176
|
+
map.putDouble("threshold", 0.6)
|
|
1177
|
+
promise.resolve(map)
|
|
1178
|
+
return
|
|
1179
|
+
}
|
|
1180
|
+
promise.resolve(session.buildARPlaneStatusMap())
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
/**
|
|
1184
|
+
* Force re-evaluation of plane detection. Used by the JS
|
|
1185
|
+
* hold-to-scan press handler in AuditCaptureScreen.tsx:529 (which
|
|
1186
|
+
* `.catch(()=>{})`s the result). Returns `latched=false`
|
|
1187
|
+
* synchronously; JS sees the new state on the next 2 Hz
|
|
1188
|
+
* getARPlaneStatus poll (~16 ms later, when the GL render thread
|
|
1189
|
+
* runs evaluatePlanesForFrame on the next ARCore frame). See
|
|
1190
|
+
* detailed semantic note in RNSARSession.buildARPlaneStatusMap.
|
|
1191
|
+
*/
|
|
1192
|
+
@ReactMethod
|
|
1193
|
+
fun relatchARPlane(promise: Promise) {
|
|
1194
|
+
RNSARSession.instance?.clearPlaneLatch()
|
|
1195
|
+
val map = Arguments.createMap()
|
|
1196
|
+
map.putBoolean("latched", false)
|
|
1197
|
+
promise.resolve(map)
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
/**
|
|
1201
|
+
* iOS-parity bridge method (was missing from Android — flagged
|
|
1202
|
+
* in the parity audit as Section C gap #1 / Section F gap #2).
|
|
1203
|
+
*
|
|
1204
|
+
* Arms the KeyframeGate to force-accept the NEXT frame regardless
|
|
1205
|
+
* of overlap. Used by the JS shutter-release path so we don't
|
|
1206
|
+
* truncate the trailing edge of the scan. iOS counterpart:
|
|
1207
|
+
* IncrementalStitcherBridge.swift markNextFrameAsLastKeyframe.
|
|
1208
|
+
*
|
|
1209
|
+
* Always resolves with `{ ok: true }`. No-op when the gate is
|
|
1210
|
+
* disabled (which is fine — the live engines don't need a
|
|
1211
|
+
* force-last; only batch-keyframe does).
|
|
1212
|
+
*/
|
|
1213
|
+
@ReactMethod
|
|
1214
|
+
fun markNextFrameAsLastKeyframe(promise: Promise) {
|
|
1215
|
+
keyframeGate.forceAcceptNext = true
|
|
1216
|
+
val map = Arguments.createMap()
|
|
1217
|
+
map.putBoolean("ok", true)
|
|
1218
|
+
promise.resolve(map)
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
/**
|
|
1222
|
+
* 2026-05-18 (Iss 3) — GC stale keyframe-session directories under
|
|
1223
|
+
* the SDK's cacheDir. Scans `cacheDir` for `rlis-capture-*`
|
|
1224
|
+
* subdirectories (created by start() above) and removes those whose
|
|
1225
|
+
* newest file mtime is older than `olderThanMs` (default 24h).
|
|
1226
|
+
*
|
|
1227
|
+
* iOS sibling: `IncrementalStitcher.swift::cleanupKeyframes`.
|
|
1228
|
+
*
|
|
1229
|
+
* Resolves with `{ sessionsDeleted, bytesFreed }`. Never rejects —
|
|
1230
|
+
* filesystem failures (missing dir, permission errors) resolve with
|
|
1231
|
+
* zero counts so the host can call this unconditionally on launch.
|
|
1232
|
+
*
|
|
1233
|
+
* Note: Android's OS already evicts cacheDir entries under storage
|
|
1234
|
+
* pressure, so this is a "be a good citizen and free space sooner"
|
|
1235
|
+
* helper rather than a hard requirement. Still useful so the user's
|
|
1236
|
+
* disk-usage report doesn't show 100s of MB of stale captures.
|
|
1237
|
+
*/
|
|
1238
|
+
@ReactMethod
|
|
1239
|
+
fun cleanupKeyframes(options: ReadableMap?, promise: Promise) {
|
|
1240
|
+
val olderThanMs = options?.getDoubleOrDefault(
|
|
1241
|
+
"olderThanMs", 24.0 * 3600.0 * 1000.0,
|
|
1242
|
+
) ?: (24.0 * 3600.0 * 1000.0)
|
|
1243
|
+
val cutoffMs = System.currentTimeMillis() - olderThanMs.toLong()
|
|
1244
|
+
var sessionsDeleted = 0
|
|
1245
|
+
var bytesFreed = 0L
|
|
1246
|
+
try {
|
|
1247
|
+
val cache = reactContext.cacheDir ?: throw IllegalStateException("no cacheDir")
|
|
1248
|
+
val sessions = cache.listFiles { f -> f.isDirectory && f.name.startsWith("rlis-capture-") }
|
|
1249
|
+
?: emptyArray()
|
|
1250
|
+
for (sessionDir in sessions) {
|
|
1251
|
+
// Newest mtime across the session's files (flat tree today,
|
|
1252
|
+
// walked recursively for future-proofing).
|
|
1253
|
+
var newestMtime = 0L
|
|
1254
|
+
var bytes = 0L
|
|
1255
|
+
sessionDir.walkTopDown().forEach { f ->
|
|
1256
|
+
if (f.isFile) {
|
|
1257
|
+
if (f.lastModified() > newestMtime) newestMtime = f.lastModified()
|
|
1258
|
+
bytes += f.length()
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
if (newestMtime == 0L) {
|
|
1262
|
+
// Empty session — fall back to the dir's own mtime.
|
|
1263
|
+
newestMtime = sessionDir.lastModified()
|
|
1264
|
+
}
|
|
1265
|
+
if (newestMtime in 1 until cutoffMs) {
|
|
1266
|
+
if (sessionDir.deleteRecursively()) {
|
|
1267
|
+
sessionsDeleted += 1
|
|
1268
|
+
bytesFreed += bytes
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
} catch (e: Exception) {
|
|
1273
|
+
android.util.Log.w(
|
|
1274
|
+
"IncrementalStitcher",
|
|
1275
|
+
"cleanupKeyframes: ${e.message}",
|
|
1276
|
+
)
|
|
1277
|
+
}
|
|
1278
|
+
val map = Arguments.createMap()
|
|
1279
|
+
map.putInt("sessionsDeleted", sessionsDeleted)
|
|
1280
|
+
map.putDouble("bytesFreed", bytesFreed.toDouble())
|
|
1281
|
+
promise.resolve(map)
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
/**
|
|
1285
|
+
* 2026-05-18 (Iss 3) — return the current capture's keyframe
|
|
1286
|
+
* session directory. Empty string when no capture is in flight
|
|
1287
|
+
* (or not in batch-keyframe mode).
|
|
1288
|
+
*
|
|
1289
|
+
* iOS sibling: `IncrementalStitcher.swift::currentKeyframeDir`.
|
|
1290
|
+
*/
|
|
1291
|
+
@ReactMethod
|
|
1292
|
+
fun getKeyframeDir(promise: Promise) {
|
|
1293
|
+
val path = if (batchKeyframeMode) {
|
|
1294
|
+
captureSessionDir?.absolutePath ?: ""
|
|
1295
|
+
} else {
|
|
1296
|
+
""
|
|
1297
|
+
}
|
|
1298
|
+
val map = Arguments.createMap()
|
|
1299
|
+
map.putString("path", path)
|
|
1300
|
+
promise.resolve(map)
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
/**
|
|
1304
|
+
* 2026-05-16 — realtime+batch fusion (Option A "Replace on
|
|
1305
|
+
* completion") entry point. Run the shared C++ `cv::Stitcher`
|
|
1306
|
+
* pipeline over a caller-supplied list of keyframe JPEGs and
|
|
1307
|
+
* write a refined panorama to `outputPath`.
|
|
1308
|
+
*
|
|
1309
|
+
* Pre-conditions:
|
|
1310
|
+
* - `framePaths.length >= 2`
|
|
1311
|
+
* - Each path must exist on disk
|
|
1312
|
+
*
|
|
1313
|
+
* Routing: delegates to `BatchStitcher.stitchSync(...)` —
|
|
1314
|
+
* the same shared-JNI shim the batch-keyframe finalize uses.
|
|
1315
|
+
* Quality defaults match the batch-keyframe finalize:
|
|
1316
|
+
* warperType = "spherical"
|
|
1317
|
+
* blenderType = "multiband"
|
|
1318
|
+
* seamFinderType = "graphcut"
|
|
1319
|
+
* captureOrientation = "portrait"
|
|
1320
|
+
* useInscribedRectCrop = false
|
|
1321
|
+
* stitchMode = "auto"
|
|
1322
|
+
* jpegQuality = 90
|
|
1323
|
+
*
|
|
1324
|
+
* Threading: dispatches onto `refineScope` so the JS promise
|
|
1325
|
+
* doesn't block the @ReactMethod thread for the 2-5 s the
|
|
1326
|
+
* stitcher takes. iOS-parity behaviour.
|
|
1327
|
+
*
|
|
1328
|
+
* iOS sibling: `IncrementalStitcher.swift::refinePanorama`.
|
|
1329
|
+
*
|
|
1330
|
+
* See: docs/site-content/design/2026-05-14-realtime-batch-fusion.md
|
|
1331
|
+
*/
|
|
1332
|
+
@ReactMethod
|
|
1333
|
+
fun refinePanorama(options: ReadableMap, promise: Promise) {
|
|
1334
|
+
val framePathsArr = options.getArray("framePaths")
|
|
1335
|
+
if (framePathsArr == null || framePathsArr.size() < 2) {
|
|
1336
|
+
promise.reject(
|
|
1337
|
+
"incremental-refine-invalid-input",
|
|
1338
|
+
"refinePanorama requires at least 2 framePaths (got " +
|
|
1339
|
+
"${framePathsArr?.size() ?: 0}).",
|
|
1340
|
+
)
|
|
1341
|
+
return
|
|
1342
|
+
}
|
|
1343
|
+
val framePaths = Array(framePathsArr.size()) {
|
|
1344
|
+
stripFileScheme(framePathsArr.getString(it) ?: "")
|
|
1345
|
+
}
|
|
1346
|
+
val outputPathOpt = options.getString("outputPath")
|
|
1347
|
+
if (outputPathOpt.isNullOrEmpty()) {
|
|
1348
|
+
promise.reject(
|
|
1349
|
+
"incremental-refine-invalid-input",
|
|
1350
|
+
"refinePanorama requires a non-empty outputPath.",
|
|
1351
|
+
)
|
|
1352
|
+
return
|
|
1353
|
+
}
|
|
1354
|
+
val outputPath = stripFileScheme(outputPathOpt)
|
|
1355
|
+
val config: ReadableMap? =
|
|
1356
|
+
if (options.hasKey("config")) options.getMap("config") else null
|
|
1357
|
+
val warperType = config?.getString("warperType") ?: "spherical"
|
|
1358
|
+
val blenderType = config?.getString("blenderType") ?: "multiband"
|
|
1359
|
+
val seamFinderType = config?.getString("seamFinderType") ?: "graphcut"
|
|
1360
|
+
val captureOrientation = config?.getString("captureOrientation") ?: "portrait"
|
|
1361
|
+
val useInscribedRectCrop =
|
|
1362
|
+
config?.getBooleanOrDefault("useInscribedRectCrop", false) ?: false
|
|
1363
|
+
val stitchMode = (config?.getString("stitchMode") ?: "auto")
|
|
1364
|
+
.let { if (it in setOf("auto", "panorama", "scans")) it else "auto" }
|
|
1365
|
+
val jpegQuality = max(1, min(100,
|
|
1366
|
+
config?.getIntOrDefault("jpegQuality", 90) ?: 90))
|
|
1367
|
+
|
|
1368
|
+
// Pre-flight existence check — same defensive layer iOS has.
|
|
1369
|
+
for (p in framePaths) {
|
|
1370
|
+
if (!File(p).exists()) {
|
|
1371
|
+
promise.reject(
|
|
1372
|
+
"incremental-refine-missing-keyframe",
|
|
1373
|
+
"refinePanorama: keyframe missing on disk — $p",
|
|
1374
|
+
)
|
|
1375
|
+
return
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
refineScope.launch {
|
|
1380
|
+
try {
|
|
1381
|
+
val stitcher = BatchStitcher.bridgeInstance
|
|
1382
|
+
?: throw IllegalStateException(
|
|
1383
|
+
"BatchStitcher.bridgeInstance is null — " +
|
|
1384
|
+
"module hasn't been instantiated yet.",
|
|
1385
|
+
)
|
|
1386
|
+
// "auto" mode is meaningful only when we have first/
|
|
1387
|
+
// last keyframe poses to consult; the explicit
|
|
1388
|
+
// refinePanorama entry point has no pose context, so
|
|
1389
|
+
// collapse 'auto' → 'scans' here (the safer fallback,
|
|
1390
|
+
// identical to resolveStitchModeAuto's null-pose
|
|
1391
|
+
// branch). Concrete modes pass through unchanged.
|
|
1392
|
+
val effectiveMode = if (stitchMode == "auto") "scans" else stitchMode
|
|
1393
|
+
val dims = stitcher.stitchSync(
|
|
1394
|
+
framePaths,
|
|
1395
|
+
outputPath,
|
|
1396
|
+
jpegQuality,
|
|
1397
|
+
warperType,
|
|
1398
|
+
blenderType,
|
|
1399
|
+
seamFinderType,
|
|
1400
|
+
captureOrientation,
|
|
1401
|
+
useInscribedRectCrop,
|
|
1402
|
+
stitchMode = effectiveMode,
|
|
1403
|
+
)
|
|
1404
|
+
val framesRequested =
|
|
1405
|
+
if (dims.size > 2) dims[2] else framePaths.size
|
|
1406
|
+
val framesIncluded =
|
|
1407
|
+
if (dims.size > 3) dims[3] else framePaths.size
|
|
1408
|
+
val finalConfidenceThresh =
|
|
1409
|
+
if (dims.size > 4) dims[4].toDouble() / 1000.0 else -1.0
|
|
1410
|
+
val map = Arguments.createMap().apply {
|
|
1411
|
+
putString("panoramaPath", outputPath)
|
|
1412
|
+
putInt("width", dims[0])
|
|
1413
|
+
putInt("height", dims[1])
|
|
1414
|
+
putInt("framesRequested", framesRequested)
|
|
1415
|
+
putInt("framesIncluded", framesIncluded)
|
|
1416
|
+
putInt("framesDropped", framesRequested - framesIncluded)
|
|
1417
|
+
putDouble("finalConfidenceThresh", finalConfidenceThresh)
|
|
1418
|
+
}
|
|
1419
|
+
promise.resolve(map)
|
|
1420
|
+
} catch (t: Throwable) {
|
|
1421
|
+
promise.reject("incremental-refine-failed", t.message, t)
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
/**
|
|
1427
|
+
* 2026-05-16 — realtime+batch fusion auto-trigger called from
|
|
1428
|
+
* the hybrid-engine branch of `finalize()`. Fire-and-forget;
|
|
1429
|
+
* the finalize() promise has ALREADY resolved with the live
|
|
1430
|
+
* panorama before this is invoked.
|
|
1431
|
+
*
|
|
1432
|
+
* 1. Emits a state event with `isRefining = true` so the
|
|
1433
|
+
* host renders a "Refining…" pill.
|
|
1434
|
+
* 2. Runs `BatchStitcher.stitchSync(...)` on the supplied
|
|
1435
|
+
* keyframe paths.
|
|
1436
|
+
* 3. On success: emits a state event with `isRefining = false`
|
|
1437
|
+
* AND `refinedPanoramaPath = <path>`.
|
|
1438
|
+
* 4. On failure: emits a state event with `isRefining = false`
|
|
1439
|
+
* and no refined path. Host keeps showing the live
|
|
1440
|
+
* panorama; failure does not affect audit save.
|
|
1441
|
+
*
|
|
1442
|
+
* NO-OP when `framePaths.size < 2` or any path is missing on
|
|
1443
|
+
* disk — matches the design doc's "if keyframes are NOT on
|
|
1444
|
+
* disk, auto-trigger is a no-op" contract. Today's hybrid
|
|
1445
|
+
* engine retains no per-frame JPEGs so this is the
|
|
1446
|
+
* always-no-op path; the hook is wired in advance of a future
|
|
1447
|
+
* keyframe-collector enhancement.
|
|
1448
|
+
*/
|
|
1449
|
+
internal fun runHybridAutoRefine(
|
|
1450
|
+
framePaths: List<String>,
|
|
1451
|
+
refinedOutputPath: String,
|
|
1452
|
+
captureOrientation: String,
|
|
1453
|
+
warperType: String,
|
|
1454
|
+
blenderType: String,
|
|
1455
|
+
seamFinderType: String,
|
|
1456
|
+
useInscribedRectCrop: Boolean,
|
|
1457
|
+
) {
|
|
1458
|
+
if (framePaths.size < 2) {
|
|
1459
|
+
android.util.Log.i(
|
|
1460
|
+
"IncrementalStitcher",
|
|
1461
|
+
"[refine.auto] skipped: framePaths.size=${framePaths.size} " +
|
|
1462
|
+
"(hybrid engine retains no per-frame JPEGs)",
|
|
1463
|
+
)
|
|
1464
|
+
emitRefinementState(isRefining = false, refinedPanoramaPath = null)
|
|
1465
|
+
return
|
|
1466
|
+
}
|
|
1467
|
+
for (p in framePaths) {
|
|
1468
|
+
if (!File(p).exists()) {
|
|
1469
|
+
android.util.Log.i(
|
|
1470
|
+
"IncrementalStitcher",
|
|
1471
|
+
"[refine.auto] skipped: missing keyframe $p",
|
|
1472
|
+
)
|
|
1473
|
+
emitRefinementState(isRefining = false, refinedPanoramaPath = null)
|
|
1474
|
+
return
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
emitRefinementState(isRefining = true, refinedPanoramaPath = null)
|
|
1478
|
+
refineScope.launch {
|
|
1479
|
+
try {
|
|
1480
|
+
val stitcher = BatchStitcher.bridgeInstance
|
|
1481
|
+
?: throw IllegalStateException(
|
|
1482
|
+
"BatchStitcher.bridgeInstance is null at auto-refine time",
|
|
1483
|
+
)
|
|
1484
|
+
stitcher.stitchSync(
|
|
1485
|
+
framePaths.toTypedArray(),
|
|
1486
|
+
refinedOutputPath,
|
|
1487
|
+
90,
|
|
1488
|
+
warperType,
|
|
1489
|
+
blenderType,
|
|
1490
|
+
seamFinderType,
|
|
1491
|
+
captureOrientation,
|
|
1492
|
+
useInscribedRectCrop,
|
|
1493
|
+
stitchMode = "scans",
|
|
1494
|
+
)
|
|
1495
|
+
android.util.Log.i(
|
|
1496
|
+
"IncrementalStitcher",
|
|
1497
|
+
"[refine.auto] success path=$refinedOutputPath",
|
|
1498
|
+
)
|
|
1499
|
+
emitRefinementState(
|
|
1500
|
+
isRefining = false,
|
|
1501
|
+
refinedPanoramaPath = refinedOutputPath,
|
|
1502
|
+
)
|
|
1503
|
+
} catch (t: Throwable) {
|
|
1504
|
+
android.util.Log.w(
|
|
1505
|
+
"IncrementalStitcher",
|
|
1506
|
+
"[refine.auto] refinement failed (live output kept): ${t.message}",
|
|
1507
|
+
)
|
|
1508
|
+
emitRefinementState(isRefining = false, refinedPanoramaPath = null)
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
/**
|
|
1514
|
+
* 2026-05-16 — emit a refinement-related state event. Reuses
|
|
1515
|
+
* the same IncrementalStateUpdate channel the live
|
|
1516
|
+
* engines emit on; JS reads `isRefining` and `refinedPanoramaPath`
|
|
1517
|
+
* directly from the event payload (no schema change required on
|
|
1518
|
+
* the JS dispatch side).
|
|
1519
|
+
*/
|
|
1520
|
+
private fun emitRefinementState(
|
|
1521
|
+
isRefining: Boolean,
|
|
1522
|
+
refinedPanoramaPath: String?,
|
|
1523
|
+
) {
|
|
1524
|
+
val state = Arguments.createMap().apply {
|
|
1525
|
+
putNull("panoramaPath")
|
|
1526
|
+
putInt("width", 0)
|
|
1527
|
+
putInt("height", 0)
|
|
1528
|
+
putInt("acceptedCount", 0)
|
|
1529
|
+
putInt("outcome", 0) // AcceptedHigh
|
|
1530
|
+
putDouble("confidence", 1.0)
|
|
1531
|
+
putDouble("overlapPercent", -1.0)
|
|
1532
|
+
putInt("processingMs", 0)
|
|
1533
|
+
putBoolean("isLandscape", false)
|
|
1534
|
+
putInt("paintedExtent", 0)
|
|
1535
|
+
putInt("panExtent", 0)
|
|
1536
|
+
putInt("keyframeMax", 0)
|
|
1537
|
+
putBoolean("isRefining", isRefining)
|
|
1538
|
+
if (refinedPanoramaPath != null) {
|
|
1539
|
+
putString("refinedPanoramaPath", refinedPanoramaPath)
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
emitState(state)
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
/**
|
|
1546
|
+
* 2026-05-16 — given the live panorama path, derive a sibling
|
|
1547
|
+
* path for the refined output. Same algorithm iOS uses:
|
|
1548
|
+
* /…/<base>.jpg → /…/<base>-refined.jpg
|
|
1549
|
+
*/
|
|
1550
|
+
private fun refinedPathFromLive(livePath: String): String {
|
|
1551
|
+
val cleaned = stripFileScheme(livePath)
|
|
1552
|
+
val file = File(cleaned)
|
|
1553
|
+
val parent = file.parentFile ?: File(reactContext.cacheDir, "panoramas")
|
|
1554
|
+
val name = file.name
|
|
1555
|
+
val dot = name.lastIndexOf('.')
|
|
1556
|
+
val refinedName = if (dot >= 0) {
|
|
1557
|
+
"${name.substring(0, dot)}-refined${name.substring(dot)}"
|
|
1558
|
+
} else {
|
|
1559
|
+
"$name-refined"
|
|
1560
|
+
}
|
|
1561
|
+
return File(parent, refinedName).absolutePath
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
/**
|
|
1565
|
+
* Poll the process' memory footprint in MB. Android parity for
|
|
1566
|
+
* iOS' `getMemoryFootprintMB` (which polls Mach `phys_footprint`
|
|
1567
|
+
* via `task_info(TASK_VM_INFO)` — see
|
|
1568
|
+
* `IncrementalStitcherBridge.swift:231-259`).
|
|
1569
|
+
*
|
|
1570
|
+
* Returns the **total PSS** (proportional set size) of this
|
|
1571
|
+
* process in MB. PSS is the metric Android's Low-Memory-Killer
|
|
1572
|
+
* (`lmkd`) ranks against, so it's the right one-true-number for
|
|
1573
|
+
* the on-screen memory pill: it's "how close are we to being
|
|
1574
|
+
* killed by the system?".
|
|
1575
|
+
*
|
|
1576
|
+
* Total PSS = USS (private) + sum(shared / refcount). Read via
|
|
1577
|
+
* `ActivityManager.getProcessMemoryInfo()`, which is the same API
|
|
1578
|
+
* Android Studio's profiler uses. Granularity is 1 KB; we
|
|
1579
|
+
* divide by 1024 to MB so the JS side displays a number directly
|
|
1580
|
+
* comparable to the iOS phys_footprint value.
|
|
1581
|
+
*
|
|
1582
|
+
* Returns -1.0 on failure (very rare — `getProcessMemoryInfo()`
|
|
1583
|
+
* is generally infallible since Android 5.0 because PSS is read
|
|
1584
|
+
* from `/proc/self/smaps` synchronously on the calling thread).
|
|
1585
|
+
*/
|
|
1586
|
+
@ReactMethod
|
|
1587
|
+
fun getMemoryFootprintMB(promise: Promise) {
|
|
1588
|
+
try {
|
|
1589
|
+
val am = reactContext.getSystemService(ActivityManager::class.java)
|
|
1590
|
+
if (am == null) {
|
|
1591
|
+
promise.resolve(-1.0)
|
|
1592
|
+
return
|
|
1593
|
+
}
|
|
1594
|
+
val pid = android.os.Process.myPid()
|
|
1595
|
+
val infos = am.getProcessMemoryInfo(intArrayOf(pid))
|
|
1596
|
+
if (infos == null || infos.isEmpty()) {
|
|
1597
|
+
promise.resolve(-1.0)
|
|
1598
|
+
return
|
|
1599
|
+
}
|
|
1600
|
+
// totalPss is in KB. Divide by 1024 → MB. Use Double so
|
|
1601
|
+
// the JS overlay can render fractional MB if it wants.
|
|
1602
|
+
val mb = infos[0].totalPss.toDouble() / 1024.0
|
|
1603
|
+
promise.resolve(mb)
|
|
1604
|
+
} catch (t: Throwable) {
|
|
1605
|
+
android.util.Log.w(
|
|
1606
|
+
"IncrementalStitcher",
|
|
1607
|
+
"getMemoryFootprintMB: failed: ${t.message}",
|
|
1608
|
+
)
|
|
1609
|
+
promise.resolve(-1.0)
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
|
|
1613
|
+
/**
|
|
1614
|
+
* Release the C++ KeyframeGate heap allocation when RN tears
|
|
1615
|
+
* down the bridge module (e.g. on a JS reload). Without this,
|
|
1616
|
+
* each reload leaks ~100 bytes of native heap — small but
|
|
1617
|
+
* unbounded over a long dev session.
|
|
1618
|
+
*/
|
|
1619
|
+
override fun onCatalystInstanceDestroy() {
|
|
1620
|
+
try {
|
|
1621
|
+
keyframeGate.close()
|
|
1622
|
+
} catch (t: Throwable) {
|
|
1623
|
+
android.util.Log.w(
|
|
1624
|
+
"IncrementalStitcher",
|
|
1625
|
+
"onCatalystInstanceDestroy: keyframeGate.close failed: ${t.message}",
|
|
1626
|
+
)
|
|
1627
|
+
}
|
|
1628
|
+
// V16 Phase 2 (Android Fix-1) — best-effort cleanup of the
|
|
1629
|
+
// current per-session subdir. Prevents leftover dirs
|
|
1630
|
+
// accumulating across dev-time RN reloads. OS cache cleanup
|
|
1631
|
+
// would eventually reclaim cacheDir entries anyway, but this
|
|
1632
|
+
// makes the dev loop tidy.
|
|
1633
|
+
try {
|
|
1634
|
+
captureSessionDir?.deleteRecursively()
|
|
1635
|
+
} catch (t: Throwable) {
|
|
1636
|
+
// Ignore — not critical at teardown.
|
|
1637
|
+
}
|
|
1638
|
+
captureSessionDir = null
|
|
1639
|
+
super.onCatalystInstanceDestroy()
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
private fun emitState(state: WritableMap?) {
|
|
1643
|
+
if (state == null) return
|
|
1644
|
+
// Re-emit to JS via the standard DeviceEventEmitter pattern.
|
|
1645
|
+
// RN drops events when no listener is attached, so we don't
|
|
1646
|
+
// need our own gating.
|
|
1647
|
+
reactContext
|
|
1648
|
+
.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
|
|
1649
|
+
.emit("IncrementalStateUpdate", state)
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
/**
|
|
1653
|
+
* Emit a state event when a batch-keyframe is accepted. Carries
|
|
1654
|
+
* the on-disk thumbnail path so JS can render it in the
|
|
1655
|
+
* LiveFrameStrip + advance the "Keyframes: N/M" pill.
|
|
1656
|
+
*
|
|
1657
|
+
* iOS-parity field set — mirrors
|
|
1658
|
+
* IncrementalStitcher.swift::emitBatchKeyframeAcceptedState
|
|
1659
|
+
* exactly (same field names, types, order) so the JS subscriber
|
|
1660
|
+
* in incremental.ts doesn't need to branch on platform.
|
|
1661
|
+
*/
|
|
1662
|
+
private fun emitBatchKeyframeAcceptedState(
|
|
1663
|
+
thumbnailPath: String,
|
|
1664
|
+
keyframeIndex: Int,
|
|
1665
|
+
keyframeCount: Int,
|
|
1666
|
+
keyframeMax: Int,
|
|
1667
|
+
isLandscape: Boolean,
|
|
1668
|
+
) {
|
|
1669
|
+
val state = Arguments.createMap()
|
|
1670
|
+
state.putNull("panoramaPath")
|
|
1671
|
+
state.putInt("width", 0)
|
|
1672
|
+
state.putInt("height", 0)
|
|
1673
|
+
state.putInt("acceptedCount", keyframeCount)
|
|
1674
|
+
// Outcome 0 = AcceptedHigh — matches the FrameOutcome enum
|
|
1675
|
+
// ordinal that the live engines emit. Keeps the iOS
|
|
1676
|
+
// IncrementalOutcome contract: batch-keyframe
|
|
1677
|
+
// accepts all carry outcome=acceptedHigh.
|
|
1678
|
+
state.putInt("outcome", 0)
|
|
1679
|
+
state.putDouble("confidence", 1.0)
|
|
1680
|
+
state.putDouble("overlapPercent", -1.0)
|
|
1681
|
+
state.putInt("processingMs", 0)
|
|
1682
|
+
state.putBoolean("isLandscape", isLandscape)
|
|
1683
|
+
state.putInt("paintedExtent", 0) // batch-keyframe doesn't
|
|
1684
|
+
state.putInt("panExtent", 0) // paint a live canvas
|
|
1685
|
+
state.putInt("keyframeMax", keyframeMax)
|
|
1686
|
+
// Batch-keyframe extras the live-engine schema doesn't carry
|
|
1687
|
+
// — JS reads these directly from the event payload (matches
|
|
1688
|
+
// iOS' direct-userInfo-write at the bottom of the iOS
|
|
1689
|
+
// emitter).
|
|
1690
|
+
state.putString("batchKeyframeThumbnailPath", thumbnailPath)
|
|
1691
|
+
state.putInt("batchKeyframeIndex", keyframeIndex)
|
|
1692
|
+
emitState(state)
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
// ── OpenCV bootstrap ────────────────────────────────────────────
|
|
1696
|
+
|
|
1697
|
+
/**
|
|
1698
|
+
* 2026-05-14 — stitch-mode auto-resolution.
|
|
1699
|
+
*
|
|
1700
|
+
* Inputs are the first and last accepted-keyframe poses captured
|
|
1701
|
+
* during this batch capture. Each pose is `[tx, ty, tz, qx, qy,
|
|
1702
|
+
* qz, qw]` in the AR-session world frame. When either pose is
|
|
1703
|
+
* null (e.g., < 2 keyframes were accepted, OR the capture used a
|
|
1704
|
+
* non-AR camera path where ARKit/ARCore poses aren't available)
|
|
1705
|
+
* we default to 'scans' — that's the safer of the two: SCANS on
|
|
1706
|
+
* pure rotation produces a slightly-less-sharp output, while
|
|
1707
|
+
* PANORAMA on translation produces an unbounded compositing
|
|
1708
|
+
* canvas and the lmkd kill we observed 2026-05-14.
|
|
1709
|
+
*
|
|
1710
|
+
* Heuristic (see design doc 2026-05-13-stitch-pipeline-mode-selection):
|
|
1711
|
+
* translation_score = ||t_last − t_first|| / 0.10 (10 cm → 1.0)
|
|
1712
|
+
* rotation_score = angle(fwd_last, fwd_first) / 1.00 (1 rad ≈ 57° → 1.0)
|
|
1713
|
+
* ratio = translation_score / (translation_score + rotation_score)
|
|
1714
|
+
* ratio ≥ 0.55 → SCANS (biased toward SCANS for safety)
|
|
1715
|
+
* ratio < 0.55 → PANORAMA
|
|
1716
|
+
*
|
|
1717
|
+
* Returns "panorama" or "scans" — never "auto".
|
|
1718
|
+
*/
|
|
1719
|
+
private fun resolveStitchModeAuto(
|
|
1720
|
+
firstPose: DoubleArray?,
|
|
1721
|
+
lastPose: DoubleArray?,
|
|
1722
|
+
): String {
|
|
1723
|
+
if (firstPose == null || lastPose == null) return "scans"
|
|
1724
|
+
if (firstPose.size != 7 || lastPose.size != 7) return "scans"
|
|
1725
|
+
|
|
1726
|
+
// Translation magnitude (Euclidean, in metres).
|
|
1727
|
+
val dtx = lastPose[0] - firstPose[0]
|
|
1728
|
+
val dty = lastPose[1] - firstPose[1]
|
|
1729
|
+
val dtz = lastPose[2] - firstPose[2]
|
|
1730
|
+
val tMeters = kotlin.math.sqrt(dtx * dtx + dty * dty + dtz * dtz)
|
|
1731
|
+
|
|
1732
|
+
// Rotation magnitude — angle between camera-forward vectors.
|
|
1733
|
+
// Camera-forward in body frame is (0, 0, -1) for ARKit/ARCore
|
|
1734
|
+
// conventions; rotated by the pose quaternion gives the world-
|
|
1735
|
+
// frame forward direction. Angle between the first and last
|
|
1736
|
+
// camera-forward vectors is the total rotation around any axis.
|
|
1737
|
+
val fwdFirst = qrotForward(firstPose[3], firstPose[4], firstPose[5], firstPose[6])
|
|
1738
|
+
val fwdLast = qrotForward(lastPose[3], lastPose[4], lastPose[5], lastPose[6])
|
|
1739
|
+
val dot = (fwdFirst[0] * fwdLast[0] + fwdFirst[1] * fwdLast[1] + fwdFirst[2] * fwdLast[2])
|
|
1740
|
+
.coerceIn(-1.0, 1.0)
|
|
1741
|
+
val rRadians = kotlin.math.acos(dot)
|
|
1742
|
+
|
|
1743
|
+
// Normalisation: 10 cm of translation ≈ 1 rad of rotation as
|
|
1744
|
+
// "equivalent magnitude" for the ratio. Empirically: shelf
|
|
1745
|
+
// scans cover ~30 cm of translation with ~10° (0.17 rad) of
|
|
1746
|
+
// rotation. ratio = (0.30/0.10) / (3.0 + 0.17) = 0.95 → SCANS.
|
|
1747
|
+
// Pure 90° rotation panorama: 0 translation, 1.57 rad rotation.
|
|
1748
|
+
// ratio = 0 / (0 + 1.57) = 0.0 → PANORAMA.
|
|
1749
|
+
val tScore = tMeters / 0.10
|
|
1750
|
+
val rScore = rRadians / 1.00
|
|
1751
|
+
val denom = tScore + rScore
|
|
1752
|
+
if (denom <= 1e-9) return "scans" // degenerate — no motion at all
|
|
1753
|
+
val ratio = tScore / denom
|
|
1754
|
+
|
|
1755
|
+
android.util.Log.i(
|
|
1756
|
+
"IncrementalStitcher",
|
|
1757
|
+
"stitch-mode auto: t=${"%.3f".format(tMeters)}m " +
|
|
1758
|
+
"r=${"%.3f".format(rRadians)}rad " +
|
|
1759
|
+
"ratio=${"%.3f".format(ratio)} " +
|
|
1760
|
+
"→ ${if (ratio >= 0.55) "scans" else "panorama"}",
|
|
1761
|
+
)
|
|
1762
|
+
return if (ratio >= 0.55) "scans" else "panorama"
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
/**
|
|
1766
|
+
* Rotate the camera-forward unit vector (0, 0, -1) by a unit
|
|
1767
|
+
* quaternion (qx, qy, qz, qw). Closed-form expansion of
|
|
1768
|
+
* v' = q · v · q⁻¹. Same convention as `qrot` in
|
|
1769
|
+
* cpp/keyframe_gate.cpp.
|
|
1770
|
+
*/
|
|
1771
|
+
private fun qrotForward(qx: Double, qy: Double, qz: Double, qw: Double): DoubleArray {
|
|
1772
|
+
// v = (0, 0, -1). q · v · q⁻¹ closed-form:
|
|
1773
|
+
// result = v + 2 * qw * (q_xyz × v) + 2 * q_xyz × (q_xyz × v)
|
|
1774
|
+
// Pre-computed for v=(0,0,-1):
|
|
1775
|
+
// q_xyz × v = (qy * -1 - qz * 0, qz * 0 - qx * -1, qx * 0 - qy * 0)
|
|
1776
|
+
// = (-qy, qx, 0)
|
|
1777
|
+
// q_xyz × (q_xyz × v):
|
|
1778
|
+
// = (qy*0 - qz*qx, qz*(-qy) - qx*0, qx*qx - qy*(-qy))
|
|
1779
|
+
// = (-qz*qx, -qz*qy, qx² + qy²)
|
|
1780
|
+
// result = (0 + 2*qw*(-qy) + 2*(-qz*qx),
|
|
1781
|
+
// 0 + 2*qw*qx + 2*(-qz*qy),
|
|
1782
|
+
// -1 + 2*qw*0 + 2*(qx² + qy²))
|
|
1783
|
+
return doubleArrayOf(
|
|
1784
|
+
-2.0 * (qw * qy + qz * qx),
|
|
1785
|
+
2.0 * (qw * qx - qz * qy),
|
|
1786
|
+
-1.0 + 2.0 * (qx * qx + qy * qy),
|
|
1787
|
+
)
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
private fun ensureOpenCv() {
|
|
1791
|
+
if (!opencvInitialised.get()) {
|
|
1792
|
+
try {
|
|
1793
|
+
System.loadLibrary("opencv_java4")
|
|
1794
|
+
opencvInitialised.set(true)
|
|
1795
|
+
} catch (e: UnsatisfiedLinkError) {
|
|
1796
|
+
throw IllegalStateException(
|
|
1797
|
+
"OpenCV native library 'opencv_java4' failed to load",
|
|
1798
|
+
e,
|
|
1799
|
+
)
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1804
|
+
companion object {
|
|
1805
|
+
@JvmStatic
|
|
1806
|
+
private val opencvInitialised = AtomicBoolean(false)
|
|
1807
|
+
|
|
1808
|
+
/// Static back-pointer used by the camera view to reach the
|
|
1809
|
+
/// active bridge module instance without a DI dance. Set
|
|
1810
|
+
/// in `init {}` of the most recently constructed instance.
|
|
1811
|
+
@JvmStatic
|
|
1812
|
+
@Volatile
|
|
1813
|
+
var bridgeInstance: IncrementalStitcher? = null
|
|
1814
|
+
private set
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
|
|
1819
|
+
// ── ReadableMap helpers ─────────────────────────────────────────────
|
|
1820
|
+
|
|
1821
|
+
private fun ReadableMap.getIntOrDefault(key: String, default: Int): Int =
|
|
1822
|
+
if (hasKey(key) && !isNull(key)) getInt(key) else default
|
|
1823
|
+
|
|
1824
|
+
private fun ReadableMap.getDoubleOrDefault(key: String, default: Double): Double =
|
|
1825
|
+
if (hasKey(key) && !isNull(key)) getDouble(key) else default
|
|
1826
|
+
|
|
1827
|
+
private fun ReadableMap.getBooleanOrDefault(key: String, default: Boolean): Boolean =
|
|
1828
|
+
if (hasKey(key) && !isNull(key)) getBoolean(key) else default
|
|
1829
|
+
|
|
1830
|
+
|
|
1831
|
+
// ── Frame outcome — mirrors iOS RLISFrameOutcome ────────────────────
|
|
1832
|
+
|
|
1833
|
+
internal enum class FrameOutcome {
|
|
1834
|
+
AcceptedHigh,
|
|
1835
|
+
AcceptedMedium,
|
|
1836
|
+
SkippedTooClose,
|
|
1837
|
+
RejectedTooFar,
|
|
1838
|
+
RejectedSceneUniform,
|
|
1839
|
+
RejectedAlignmentLost,
|
|
1840
|
+
SkippedTrackingPoor,
|
|
1841
|
+
/** V12.11 Step D — operator panned BACKWARDS past the running
|
|
1842
|
+
* max along the pan axis. Engine has SKIPPED the paste; host
|
|
1843
|
+
* should auto-finalize. Rectilinear-only. */
|
|
1844
|
+
RejectedReverseDirection,
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
|
|
1848
|
+
internal data class FrameTelemetry(
|
|
1849
|
+
val outcome: FrameOutcome,
|
|
1850
|
+
val overlapPercent: Double,
|
|
1851
|
+
val matchCount: Int,
|
|
1852
|
+
val inlierRatio: Double,
|
|
1853
|
+
val confidence: Double,
|
|
1854
|
+
val processingMs: Double,
|
|
1855
|
+
/** V12.12 — engine-detected orientation. Mirrors iOS'
|
|
1856
|
+
* `RLISFrameTelemetry.isLandscape`. TRUE for landscape capture
|
|
1857
|
+
* (vertical pan), FALSE for portrait (horizontal pan). Stays
|
|
1858
|
+
* at the FIRST-FRAME determination thereafter. */
|
|
1859
|
+
val isLandscape: Boolean = false,
|
|
1860
|
+
)
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
internal data class StitcherSnapshot(
|
|
1864
|
+
val panoramaPath: String,
|
|
1865
|
+
val width: Int,
|
|
1866
|
+
val height: Int,
|
|
1867
|
+
val acceptedCount: Int,
|
|
1868
|
+
)
|
|
1869
|
+
|
|
1870
|
+
|
|
1871
|
+
/**
|
|
1872
|
+
* Pure-OpenCV implementation of the incremental algorithm. No RN
|
|
1873
|
+
* dependency — this class can be unit-tested with synthetic Mat
|
|
1874
|
+
* inputs. Exact algorithmic mirror of iOS' OpenCVIncrementalStitcher.mm.
|
|
1875
|
+
*/
|
|
1876
|
+
internal class IncrementalEngine(
|
|
1877
|
+
val composeWidth: Int,
|
|
1878
|
+
val composeHeight: Int,
|
|
1879
|
+
val canvasWidth: Int,
|
|
1880
|
+
val canvasHeight: Int,
|
|
1881
|
+
val featherPx: Int,
|
|
1882
|
+
val snapshotJpegQuality: Int,
|
|
1883
|
+
val snapshotEveryNAccepts: Int,
|
|
1884
|
+
/// 0/90/180/270 — rotation applied to each ingested frame before
|
|
1885
|
+
/// any other processing. See iOS' equivalent for the full
|
|
1886
|
+
/// rationale. JS computes from device orientation.
|
|
1887
|
+
val frameRotationDegrees: Int,
|
|
1888
|
+
) {
|
|
1889
|
+
private val canvas: Mat = Mat.zeros(canvasHeight, canvasWidth, CvType.CV_8UC3)
|
|
1890
|
+
private val canvasMask: Mat = Mat.zeros(canvasHeight, canvasWidth, CvType.CV_8UC1)
|
|
1891
|
+
|
|
1892
|
+
/// V7 pose-driven state — sensor-native compute path. Mirrors iOS.
|
|
1893
|
+
private var firstRotationArkit: Mat = Mat()
|
|
1894
|
+
private var kCompose: Mat = Mat()
|
|
1895
|
+
private var tCanvas: Mat = Mat.eye(3, 3, CvType.CV_64F)
|
|
1896
|
+
private val mArkitToCv: Mat = Mat(3, 3, CvType.CV_64F).apply {
|
|
1897
|
+
// diag(1, -1, -1) — ARKit/ARCore (Y-up, -Z forward) → OpenCV.
|
|
1898
|
+
setTo(Scalar(0.0))
|
|
1899
|
+
put(0, 0, 1.0); put(1, 1, -1.0); put(2, 2, -1.0)
|
|
1900
|
+
}
|
|
1901
|
+
|
|
1902
|
+
private var lastAcceptedYaw: Double = 0.0
|
|
1903
|
+
private var lastAcceptedPitch: Double = 0.0
|
|
1904
|
+
private var hasFirstFrame: Boolean = false
|
|
1905
|
+
private var acceptsSinceSnapshot: Int = 0
|
|
1906
|
+
var acceptedCount: Int = 0
|
|
1907
|
+
private set
|
|
1908
|
+
private var snapshotSeq: Int = 0
|
|
1909
|
+
var lastState: WritableMap? = null
|
|
1910
|
+
private set
|
|
1911
|
+
|
|
1912
|
+
/**
|
|
1913
|
+
* Read the JPEG at `path`, downscale to compose-resolution, run
|
|
1914
|
+
* the same algorithm as the iOS engine.
|
|
1915
|
+
*/
|
|
1916
|
+
fun addFrameAtPath(
|
|
1917
|
+
path: String,
|
|
1918
|
+
qx: Double,
|
|
1919
|
+
qy: Double,
|
|
1920
|
+
qz: Double,
|
|
1921
|
+
qw: Double,
|
|
1922
|
+
fx: Double,
|
|
1923
|
+
fy: Double,
|
|
1924
|
+
cx: Double,
|
|
1925
|
+
cy: Double,
|
|
1926
|
+
imageWidth: Int,
|
|
1927
|
+
imageHeight: Int,
|
|
1928
|
+
yaw: Double,
|
|
1929
|
+
pitch: Double,
|
|
1930
|
+
fovHorizDegrees: Double,
|
|
1931
|
+
fovVertDegrees: Double,
|
|
1932
|
+
trackingPoor: Boolean,
|
|
1933
|
+
): FrameTelemetry {
|
|
1934
|
+
val t0 = System.nanoTime()
|
|
1935
|
+
if (trackingPoor) {
|
|
1936
|
+
return FrameTelemetry(
|
|
1937
|
+
FrameOutcome.SkippedTrackingPoor, -1.0, 0, 0.0, 0.0,
|
|
1938
|
+
msSince(t0),
|
|
1939
|
+
)
|
|
1940
|
+
}
|
|
1941
|
+
|
|
1942
|
+
val cleaned = stripFileScheme(path)
|
|
1943
|
+
val srcRaw = Imgcodecs.imread(cleaned, Imgcodecs.IMREAD_COLOR)
|
|
1944
|
+
if (srcRaw.empty()) {
|
|
1945
|
+
return FrameTelemetry(
|
|
1946
|
+
FrameOutcome.SkippedTrackingPoor, -1.0, 0, 0.0, 0.0,
|
|
1947
|
+
msSince(t0),
|
|
1948
|
+
)
|
|
1949
|
+
}
|
|
1950
|
+
// V7: NO input rotation. ARCore (and the JS gyro fallback)
|
|
1951
|
+
// deliver sensor-native landscape frames; we keep them in
|
|
1952
|
+
// that frame through the entire compute pipeline. Output
|
|
1953
|
+
// rotation for display happens at snapshot/finalize time.
|
|
1954
|
+
// See iOS' equivalent fix for the architectural rationale.
|
|
1955
|
+
val frame = downsampleToCompose(srcRaw)
|
|
1956
|
+
if (frame !== srcRaw) srcRaw.release()
|
|
1957
|
+
|
|
1958
|
+
// Build R_new from quaternion.
|
|
1959
|
+
val rNew = quaternionToRotationMat(qx, qy, qz, qw)
|
|
1960
|
+
|
|
1961
|
+
if (!hasFirstFrame) {
|
|
1962
|
+
firstRotationArkit = rNew.clone()
|
|
1963
|
+
// V7: K is in COMPOSE pixel coordinates. Sensor intrinsics
|
|
1964
|
+
// get scaled by the same uniform factor we used to downsample
|
|
1965
|
+
// the frame, so K · ray → pixel produces the right pixel
|
|
1966
|
+
// in compose space directly. No rotation chain needed.
|
|
1967
|
+
val sx = frame.cols().toDouble() / maxOf(1, imageWidth)
|
|
1968
|
+
val sy = frame.rows().toDouble() / maxOf(1, imageHeight)
|
|
1969
|
+
val s = 0.5 * (sx + sy)
|
|
1970
|
+
kCompose = Mat(3, 3, CvType.CV_64F).apply {
|
|
1971
|
+
setTo(Scalar(0.0))
|
|
1972
|
+
put(0, 0, fx * s); put(0, 2, cx * s)
|
|
1973
|
+
put(1, 1, fy * s); put(1, 2, cy * s)
|
|
1974
|
+
put(2, 2, 1.0)
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
// Place first frame at canvas centre.
|
|
1978
|
+
val ox = (canvas.cols() - frame.cols()) / 2
|
|
1979
|
+
val oy = (canvas.rows() - frame.rows()) / 2
|
|
1980
|
+
val roi = Rect(ox, oy, frame.cols(), frame.rows())
|
|
1981
|
+
frame.copyTo(canvas.submat(roi))
|
|
1982
|
+
canvasMask.submat(roi).setTo(Scalar(255.0))
|
|
1983
|
+
tCanvas = Mat.eye(3, 3, CvType.CV_64F)
|
|
1984
|
+
tCanvas.put(0, 2, ox.toDouble())
|
|
1985
|
+
tCanvas.put(1, 2, oy.toDouble())
|
|
1986
|
+
|
|
1987
|
+
lastAcceptedYaw = yaw
|
|
1988
|
+
lastAcceptedPitch = pitch
|
|
1989
|
+
hasFirstFrame = true
|
|
1990
|
+
acceptedCount = 1
|
|
1991
|
+
frame.release()
|
|
1992
|
+
return FrameTelemetry(
|
|
1993
|
+
FrameOutcome.AcceptedHigh, 0.0, 0, 0.0, 1.0, msSince(t0),
|
|
1994
|
+
)
|
|
1995
|
+
}
|
|
1996
|
+
|
|
1997
|
+
val overlap = computeOverlapPct(
|
|
1998
|
+
yaw - lastAcceptedYaw, pitch - lastAcceptedPitch,
|
|
1999
|
+
fovHorizDegrees, fovVertDegrees,
|
|
2000
|
+
)
|
|
2001
|
+
if (overlap > MAX_OVERLAP_PCT) {
|
|
2002
|
+
frame.release()
|
|
2003
|
+
return FrameTelemetry(
|
|
2004
|
+
FrameOutcome.SkippedTooClose, overlap, 0, 0.0, 0.0, msSince(t0),
|
|
2005
|
+
)
|
|
2006
|
+
}
|
|
2007
|
+
if (overlap < MIN_OVERLAP_PCT) {
|
|
2008
|
+
frame.release()
|
|
2009
|
+
return FrameTelemetry(
|
|
2010
|
+
FrameOutcome.RejectedTooFar, overlap, 0, 0.0, 0.0, msSince(t0),
|
|
2011
|
+
)
|
|
2012
|
+
}
|
|
2013
|
+
|
|
2014
|
+
// V7 pose-driven homography (sensor-native compose space):
|
|
2015
|
+
// R_rel_cv = M · R_first⁻¹ · R_new · M
|
|
2016
|
+
// H_compose = K_compose · R_rel_cv · K_compose⁻¹
|
|
2017
|
+
// H_canvas = T_canvas · H_compose
|
|
2018
|
+
// No R2S/S chain — the v6 bug was applying input rotation
|
|
2019
|
+
// and undoing it via the chain; v7 keeps everything in
|
|
2020
|
+
// sensor-native compose space and rotates only at output.
|
|
2021
|
+
val firstInv = Mat()
|
|
2022
|
+
Core.transpose(firstRotationArkit, firstInv)
|
|
2023
|
+
val tmp1 = Mat(); Core.gemm(mArkitToCv, firstInv, 1.0, Mat(), 0.0, tmp1)
|
|
2024
|
+
val tmp2 = Mat(); Core.gemm(tmp1, rNew, 1.0, Mat(), 0.0, tmp2)
|
|
2025
|
+
val rRelCv = Mat(); Core.gemm(tmp2, mArkitToCv, 1.0, Mat(), 0.0, rRelCv)
|
|
2026
|
+
firstInv.release(); tmp1.release(); tmp2.release()
|
|
2027
|
+
|
|
2028
|
+
val kInv = kCompose.inv()
|
|
2029
|
+
val hcTmp = Mat(); Core.gemm(kCompose, rRelCv, 1.0, Mat(), 0.0, hcTmp)
|
|
2030
|
+
val hCompose = Mat(); Core.gemm(hcTmp, kInv, 1.0, Mat(), 0.0, hCompose)
|
|
2031
|
+
kInv.release(); hcTmp.release(); rRelCv.release(); rNew.release()
|
|
2032
|
+
|
|
2033
|
+
val hCanvas = Mat(); Core.gemm(tCanvas, hCompose, 1.0, Mat(), 0.0, hCanvas)
|
|
2034
|
+
hCompose.release()
|
|
2035
|
+
|
|
2036
|
+
warpAndBlend(frame, hCanvas)
|
|
2037
|
+
hCanvas.release()
|
|
2038
|
+
frame.release()
|
|
2039
|
+
|
|
2040
|
+
lastAcceptedYaw = yaw
|
|
2041
|
+
lastAcceptedPitch = pitch
|
|
2042
|
+
acceptedCount++
|
|
2043
|
+
|
|
2044
|
+
// Confidence as in iOS — function of how centred the overlap
|
|
2045
|
+
// is in the [10, 75]% acceptance window.
|
|
2046
|
+
val midOverlap = 0.5 * (MIN_OVERLAP_PCT + MAX_OVERLAP_PCT)
|
|
2047
|
+
val overlapDistance = kotlin.math.abs(overlap - midOverlap) /
|
|
2048
|
+
(MAX_OVERLAP_PCT - midOverlap)
|
|
2049
|
+
val confidence = maxOf(0.0, 1.0 - overlapDistance)
|
|
2050
|
+
val outcome = if (confidence >= 0.6) FrameOutcome.AcceptedHigh
|
|
2051
|
+
else FrameOutcome.AcceptedMedium
|
|
2052
|
+
|
|
2053
|
+
return FrameTelemetry(
|
|
2054
|
+
outcome, overlap, -1, -1.0, confidence, msSince(t0),
|
|
2055
|
+
)
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
/** Write a JPEG snapshot if accept-counter has hit the configured cadence. */
|
|
2059
|
+
fun snapshotIfDue(tele: FrameTelemetry): WritableMap? {
|
|
2060
|
+
val isAccept = tele.outcome == FrameOutcome.AcceptedHigh
|
|
2061
|
+
|| tele.outcome == FrameOutcome.AcceptedMedium
|
|
2062
|
+
var snapshotPath: String? = null
|
|
2063
|
+
var snapW = 0
|
|
2064
|
+
var snapH = 0
|
|
2065
|
+
if (isAccept) {
|
|
2066
|
+
acceptsSinceSnapshot++
|
|
2067
|
+
if (acceptsSinceSnapshot >= snapshotEveryNAccepts) {
|
|
2068
|
+
acceptsSinceSnapshot = 0
|
|
2069
|
+
snapshotSeq++
|
|
2070
|
+
val slot = snapshotSeq % 4
|
|
2071
|
+
val tmpPath = "${System.getProperty("java.io.tmpdir") ?: "/data/local/tmp"}" +
|
|
2072
|
+
"/rlis-live-$slot.jpg"
|
|
2073
|
+
// tightCrop = true for live snapshots: the canvas is
|
|
2074
|
+
// 4800x2200, but most of it is empty until the pan
|
|
2075
|
+
// covers it. Without a tight crop, every snapshot
|
|
2076
|
+
// was a ~24 MB JPEG that RN's <Image> couldn't keep
|
|
2077
|
+
// up with. Tight-cropped snapshots are 50–500 KB.
|
|
2078
|
+
val snap = writeJpeg(tmpPath, snapshotJpegQuality, tightCrop = true)
|
|
2079
|
+
if (snap != null) {
|
|
2080
|
+
snapshotPath = snap.panoramaPath
|
|
2081
|
+
snapW = snap.width
|
|
2082
|
+
snapH = snap.height
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
val map = Arguments.createMap().apply {
|
|
2088
|
+
putInt("width", snapW)
|
|
2089
|
+
putInt("height", snapH)
|
|
2090
|
+
putInt("acceptedCount", acceptedCount)
|
|
2091
|
+
putInt("outcome", tele.outcome.ordinal)
|
|
2092
|
+
putDouble("confidence", tele.confidence)
|
|
2093
|
+
putDouble("overlapPercent", tele.overlapPercent)
|
|
2094
|
+
putDouble("processingMs", tele.processingMs)
|
|
2095
|
+
if (snapshotPath != null) putString("panoramaPath", snapshotPath)
|
|
2096
|
+
}
|
|
2097
|
+
lastState = map
|
|
2098
|
+
return map
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
fun finalize(outputPath: String, quality: Int): StitcherSnapshot {
|
|
2102
|
+
val cleaned = stripFileScheme(outputPath)
|
|
2103
|
+
val snap = writeJpeg(cleaned, quality, tightCrop = true)
|
|
2104
|
+
?: throw IllegalStateException(
|
|
2105
|
+
"No frames have been accepted yet, or write failed: $cleaned",
|
|
2106
|
+
)
|
|
2107
|
+
release()
|
|
2108
|
+
return snap
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
fun release() {
|
|
2112
|
+
canvas.release()
|
|
2113
|
+
canvasMask.release()
|
|
2114
|
+
firstRotationArkit.release()
|
|
2115
|
+
kCompose.release()
|
|
2116
|
+
tCanvas.release()
|
|
2117
|
+
mArkitToCv.release()
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
// ── internal helpers ────────────────────────────────────────────
|
|
2121
|
+
|
|
2122
|
+
private fun downsampleToCompose(src: Mat): Mat {
|
|
2123
|
+
// Uniform scale that fits inside the compose-dim budget — the
|
|
2124
|
+
// smaller of the two ratios wins so neither axis distorts.
|
|
2125
|
+
val sw = src.cols().toDouble()
|
|
2126
|
+
val sh = src.rows().toDouble()
|
|
2127
|
+
var scale = minOf(composeWidth.toDouble() / sw, composeHeight.toDouble() / sh)
|
|
2128
|
+
if (scale > 1.0) scale = 1.0 // never upscale
|
|
2129
|
+
val outW = maxOf(1, (sw * scale).toInt())
|
|
2130
|
+
val outH = maxOf(1, (sh * scale).toInt())
|
|
2131
|
+
if (src.cols() == outW && src.rows() == outH) return src
|
|
2132
|
+
val out = Mat()
|
|
2133
|
+
Imgproc.resize(src, out, Size(outW.toDouble(), outH.toDouble()), 0.0, 0.0, Imgproc.INTER_AREA)
|
|
2134
|
+
return out
|
|
2135
|
+
}
|
|
2136
|
+
|
|
2137
|
+
// `placeFirstFrame` was dropped in v6 — the first-frame logic is
|
|
2138
|
+
// now inlined in `addFrameAtPath` so the engine can capture the
|
|
2139
|
+
// reference pose + intrinsics in the same place it positions the
|
|
2140
|
+
// frame on the canvas.
|
|
2141
|
+
|
|
2142
|
+
private fun warpAndBlend(frame: Mat, worldH: Mat) {
|
|
2143
|
+
val canvasSize = Size(canvasWidth.toDouble(), canvasHeight.toDouble())
|
|
2144
|
+
|
|
2145
|
+
val warped = Mat()
|
|
2146
|
+
Imgproc.warpPerspective(
|
|
2147
|
+
frame, warped, worldH, canvasSize,
|
|
2148
|
+
Imgproc.INTER_LINEAR, Core.BORDER_CONSTANT, Scalar(0.0, 0.0, 0.0),
|
|
2149
|
+
)
|
|
2150
|
+
|
|
2151
|
+
val frameOnesMask = Mat(frame.rows(), frame.cols(), CvType.CV_8UC1, Scalar(255.0))
|
|
2152
|
+
val warpedMask = Mat()
|
|
2153
|
+
Imgproc.warpPerspective(
|
|
2154
|
+
frameOnesMask, warpedMask, worldH, canvasSize,
|
|
2155
|
+
Imgproc.INTER_NEAREST, Core.BORDER_CONSTANT, Scalar(0.0),
|
|
2156
|
+
)
|
|
2157
|
+
frameOnesMask.release()
|
|
2158
|
+
|
|
2159
|
+
// Hard midline seam (replaces v4 ratio-feather). Same fix
|
|
2160
|
+
// as iOS v5: each output pixel comes from exactly one frame,
|
|
2161
|
+
// so misalignment between frames can't produce ghosts. The
|
|
2162
|
+
// seam is placed where each pixel is equidistant from both
|
|
2163
|
+
// frames' outer edges (the "middle" of the overlap), then
|
|
2164
|
+
// softened with a small Gaussian to hide the pixel-perfect
|
|
2165
|
+
// cut.
|
|
2166
|
+
val distNew = Mat()
|
|
2167
|
+
Imgproc.distanceTransform(warpedMask, distNew, Imgproc.DIST_L2, 3)
|
|
2168
|
+
val distCanvas = Mat()
|
|
2169
|
+
Imgproc.distanceTransform(canvasMask, distCanvas, Imgproc.DIST_L2, 3)
|
|
2170
|
+
|
|
2171
|
+
// alpha8: 255 where new is deeper, 0 where canvas is deeper.
|
|
2172
|
+
val alpha8 = Mat()
|
|
2173
|
+
Core.compare(distNew, distCanvas, alpha8, Core.CMP_GE)
|
|
2174
|
+
|
|
2175
|
+
// First-touch regions need new frame to write unconditionally.
|
|
2176
|
+
val noPriorMask = Mat()
|
|
2177
|
+
Core.compare(canvasMask, Scalar(0.0), noPriorMask, Core.CMP_EQ)
|
|
2178
|
+
alpha8.setTo(Scalar(255.0), noPriorMask)
|
|
2179
|
+
noPriorMask.release()
|
|
2180
|
+
|
|
2181
|
+
val alpha = Mat()
|
|
2182
|
+
alpha8.convertTo(alpha, CvType.CV_32F, 1.0 / 255.0)
|
|
2183
|
+
alpha8.release()
|
|
2184
|
+
Imgproc.GaussianBlur(alpha, alpha, Size(7.0, 7.0), 0.0)
|
|
2185
|
+
distNew.release(); distCanvas.release()
|
|
2186
|
+
|
|
2187
|
+
val alphaChannels = mutableListOf(alpha, alpha, alpha)
|
|
2188
|
+
val alpha3 = Mat()
|
|
2189
|
+
Core.merge(alphaChannels, alpha3)
|
|
2190
|
+
val invAlpha3 = Mat()
|
|
2191
|
+
Core.subtract(Mat.ones(alpha3.size(), alpha3.type()).apply {
|
|
2192
|
+
setTo(Scalar(1.0, 1.0, 1.0))
|
|
2193
|
+
}, alpha3, invAlpha3)
|
|
2194
|
+
|
|
2195
|
+
val warpedF = Mat(); warped.convertTo(warpedF, CvType.CV_32FC3)
|
|
2196
|
+
val canvasF = Mat(); canvas.convertTo(canvasF, CvType.CV_32FC3)
|
|
2197
|
+
val blendedF = Mat()
|
|
2198
|
+
Core.multiply(warpedF, alpha3, warpedF)
|
|
2199
|
+
Core.multiply(canvasF, invAlpha3, canvasF)
|
|
2200
|
+
Core.add(warpedF, canvasF, blendedF)
|
|
2201
|
+
warpedF.release(); canvasF.release()
|
|
2202
|
+
alpha.release(); alpha3.release(); invAlpha3.release()
|
|
2203
|
+
|
|
2204
|
+
val blended8 = Mat()
|
|
2205
|
+
blendedF.convertTo(blended8, CvType.CV_8UC3)
|
|
2206
|
+
blendedF.release()
|
|
2207
|
+
// Only write where warpedMask is set; rest of canvas is unchanged.
|
|
2208
|
+
blended8.copyTo(canvas, warpedMask)
|
|
2209
|
+
blended8.release()
|
|
2210
|
+
|
|
2211
|
+
Core.bitwise_or(canvasMask, warpedMask, canvasMask)
|
|
2212
|
+
warpedMask.release()
|
|
2213
|
+
warped.release()
|
|
2214
|
+
}
|
|
2215
|
+
|
|
2216
|
+
private fun writeJpeg(
|
|
2217
|
+
outputPath: String,
|
|
2218
|
+
quality: Int,
|
|
2219
|
+
tightCrop: Boolean,
|
|
2220
|
+
): StitcherSnapshot? {
|
|
2221
|
+
if (acceptedCount == 0) return null
|
|
2222
|
+
var crop = Rect(0, 0, canvas.cols(), canvas.rows())
|
|
2223
|
+
if (tightCrop) {
|
|
2224
|
+
val nonZero = MatOfPoint2f()
|
|
2225
|
+
// boundingRect on the mask matrix gives us the tight crop;
|
|
2226
|
+
// OpenCV Java's API takes a Mat of points, but for an
|
|
2227
|
+
// image mask we use Imgproc.boundingRect on a contour.
|
|
2228
|
+
// Cheaper path: walk the mask once.
|
|
2229
|
+
val contoured = Imgproc.boundingRect(MaskNonZeroContour(canvasMask))
|
|
2230
|
+
if (contoured.width > 0 && contoured.height > 0) {
|
|
2231
|
+
crop = contoured
|
|
2232
|
+
}
|
|
2233
|
+
nonZero.release()
|
|
2234
|
+
}
|
|
2235
|
+
val cropped = Mat(canvas, crop)
|
|
2236
|
+
// V7.1 GRAVITY-DERIVED OUTPUT ROTATION. Mirrors iOS — see
|
|
2237
|
+
// OpenCVIncrementalStitcher.mm for the full derivation. The
|
|
2238
|
+
// rotation comes from the AR pose (which knows gravity) so
|
|
2239
|
+
// we don't need a device-orientation hook (which was the
|
|
2240
|
+
// source of the v7 "sideways for landscape" bug).
|
|
2241
|
+
var rotationDeg = 0
|
|
2242
|
+
if (hasFirstFrame && !firstRotationArkit.empty()) {
|
|
2243
|
+
val gravWorld = Mat(3, 1, CvType.CV_64F).apply {
|
|
2244
|
+
put(0, 0, 0.0); put(1, 0, -1.0); put(2, 0, 0.0)
|
|
2245
|
+
}
|
|
2246
|
+
val firstT = Mat()
|
|
2247
|
+
Core.transpose(firstRotationArkit, firstT)
|
|
2248
|
+
val gravArkit = Mat(); Core.gemm(firstT, gravWorld, 1.0, Mat(), 0.0, gravArkit)
|
|
2249
|
+
val gravCv = Mat(); Core.gemm(mArkitToCv, gravArkit, 1.0, Mat(), 0.0, gravCv)
|
|
2250
|
+
val gx = gravCv.get(0, 0)[0]
|
|
2251
|
+
val gy = gravCv.get(1, 0)[0]
|
|
2252
|
+
val angle = kotlin.math.atan2(gx, gy) * 180.0 / Math.PI
|
|
2253
|
+
rotationDeg = (kotlin.math.round(angle / 90.0).toInt()) * 90
|
|
2254
|
+
rotationDeg = ((rotationDeg % 360) + 360) % 360
|
|
2255
|
+
gravWorld.release(); firstT.release(); gravArkit.release(); gravCv.release()
|
|
2256
|
+
}
|
|
2257
|
+
val out = when (rotationDeg) {
|
|
2258
|
+
90 -> Mat().also { Core.rotate(cropped, it, Core.ROTATE_90_CLOCKWISE) }
|
|
2259
|
+
180 -> Mat().also { Core.rotate(cropped, it, Core.ROTATE_180) }
|
|
2260
|
+
270 -> Mat().also { Core.rotate(cropped, it, Core.ROTATE_90_COUNTERCLOCKWISE) }
|
|
2261
|
+
else -> cropped
|
|
2262
|
+
}
|
|
2263
|
+
val outW = out.cols()
|
|
2264
|
+
val outH = out.rows()
|
|
2265
|
+
val params = MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, quality)
|
|
2266
|
+
val ok = Imgcodecs.imwrite(outputPath, out, params)
|
|
2267
|
+
if (out !== cropped) out.release()
|
|
2268
|
+
cropped.release()
|
|
2269
|
+
if (!ok) return null
|
|
2270
|
+
return StitcherSnapshot(
|
|
2271
|
+
panoramaPath = outputPath,
|
|
2272
|
+
width = outW,
|
|
2273
|
+
height = outH,
|
|
2274
|
+
acceptedCount = acceptedCount,
|
|
2275
|
+
)
|
|
2276
|
+
}
|
|
2277
|
+
|
|
2278
|
+
private fun msSince(t0Nanos: Long): Double =
|
|
2279
|
+
(System.nanoTime() - t0Nanos) / 1_000_000.0
|
|
2280
|
+
|
|
2281
|
+
companion object {
|
|
2282
|
+
// v3 thresholds — relaxed match-count + inlier minimums for
|
|
2283
|
+
// light-texture shelf scenes; tighter det range because the
|
|
2284
|
+
// affine fit produces a much narrower legitimate scale band.
|
|
2285
|
+
private const val MIN_OVERLAP_PCT = 10.0
|
|
2286
|
+
private const val MAX_OVERLAP_PCT = 75.0
|
|
2287
|
+
private const val MIN_MATCHES_ACCEPT = 10
|
|
2288
|
+
private const val MIN_INLIER_RATIO_ACCEPT = 0.18
|
|
2289
|
+
private const val HIGH_CONF_MATCHES = 60
|
|
2290
|
+
private const val HIGH_CONF_INLIER_RATIO = 0.55
|
|
2291
|
+
private const val ORB_MAX_FEATURES = 1000
|
|
2292
|
+
private const val ORB_SCALE_FACTOR = 1.2f
|
|
2293
|
+
private const val ORB_LEVELS = 8
|
|
2294
|
+
private const val ORB_EDGE_THRESHOLD = 31
|
|
2295
|
+
private const val LOWE_RATIO = 0.75f
|
|
2296
|
+
private const val RANSAC_REPROJ_THRESH = 5.0
|
|
2297
|
+
private const val HOM_DET_MIN = 0.7
|
|
2298
|
+
private const val HOM_DET_MAX = 1.4
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
|
|
2303
|
+
/// Helper for OpenCV's odd boundingRect signature on a binary mask.
|
|
2304
|
+
/// Wraps the mask's non-zero pixel coords as a MatOfPoint that
|
|
2305
|
+
/// `Imgproc.boundingRect` will accept.
|
|
2306
|
+
private fun MaskNonZeroContour(mask: Mat): org.opencv.core.MatOfPoint {
|
|
2307
|
+
val locations = Mat()
|
|
2308
|
+
Core.findNonZero(mask, locations)
|
|
2309
|
+
if (locations.empty()) {
|
|
2310
|
+
locations.release()
|
|
2311
|
+
return org.opencv.core.MatOfPoint()
|
|
2312
|
+
}
|
|
2313
|
+
val pts = mutableListOf<Point>()
|
|
2314
|
+
// findNonZero returns CV_32SC2 with N rows, 1 col. Each entry is
|
|
2315
|
+
// a (col, row) pair; build a point list for boundingRect.
|
|
2316
|
+
val n = locations.rows()
|
|
2317
|
+
val buf = IntArray(2)
|
|
2318
|
+
for (i in 0 until n) {
|
|
2319
|
+
locations.get(i, 0, buf)
|
|
2320
|
+
pts.add(Point(buf[0].toDouble(), buf[1].toDouble()))
|
|
2321
|
+
}
|
|
2322
|
+
locations.release()
|
|
2323
|
+
val out = org.opencv.core.MatOfPoint()
|
|
2324
|
+
out.fromList(pts)
|
|
2325
|
+
return out
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
|
|
2329
|
+
// computeOverlapPct, stripFileScheme — same code as iOS's static helpers,
|
|
2330
|
+
// transcribed to Kotlin.
|
|
2331
|
+
|
|
2332
|
+
internal fun computeOverlapPct(
|
|
2333
|
+
deltaYaw: Double,
|
|
2334
|
+
deltaPitch: Double,
|
|
2335
|
+
fovHorizDegrees: Double,
|
|
2336
|
+
fovVertDegrees: Double,
|
|
2337
|
+
): Double {
|
|
2338
|
+
val absYaw = kotlin.math.abs(deltaYaw)
|
|
2339
|
+
val absPitch = kotlin.math.abs(deltaPitch)
|
|
2340
|
+
var fovH = fovHorizDegrees * Math.PI / 180.0
|
|
2341
|
+
var fovV = fovVertDegrees * Math.PI / 180.0
|
|
2342
|
+
if (fovH <= 1e-6) fovH = 65.0 * Math.PI / 180.0
|
|
2343
|
+
if (fovV <= 1e-6) fovV = 50.0 * Math.PI / 180.0
|
|
2344
|
+
val overlap = if (absYaw >= absPitch) {
|
|
2345
|
+
1.0 - absYaw / fovH
|
|
2346
|
+
} else {
|
|
2347
|
+
1.0 - absPitch / fovV
|
|
2348
|
+
}
|
|
2349
|
+
return overlap.coerceIn(0.0, 1.0) * 100.0
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
|
|
2353
|
+
internal fun stripFileScheme(path: String): String =
|
|
2354
|
+
if (path.startsWith("file://")) path.removePrefix("file://") else path
|
|
2355
|
+
|
|
2356
|
+
|
|
2357
|
+
/// Quaternion → 3x3 rotation matrix, mirroring iOS `quaternionToRotationMat`.
|
|
2358
|
+
internal fun quaternionToRotationMat(qx0: Double, qy0: Double, qz0: Double, qw0: Double): Mat {
|
|
2359
|
+
var qx = qx0; var qy = qy0; var qz = qz0; var qw = qw0
|
|
2360
|
+
val n = kotlin.math.sqrt(qx*qx + qy*qy + qz*qz + qw*qw)
|
|
2361
|
+
if (n > 1e-9) { qx /= n; qy /= n; qz /= n; qw /= n }
|
|
2362
|
+
val r = Mat(3, 3, CvType.CV_64F)
|
|
2363
|
+
r.put(0, 0, 1 - 2*(qy*qy + qz*qz)); r.put(0, 1, 2*(qx*qy - qw*qz)); r.put(0, 2, 2*(qx*qz + qw*qy))
|
|
2364
|
+
r.put(1, 0, 2*(qx*qy + qw*qz)); r.put(1, 1, 1 - 2*(qx*qx + qz*qz)); r.put(1, 2, 2*(qy*qz - qw*qx))
|
|
2365
|
+
r.put(2, 0, 2*(qx*qz - qw*qy)); r.put(2, 1, 2*(qy*qz + qw*qx)); r.put(2, 2, 1 - 2*(qx*qx + qy*qy))
|
|
2366
|
+
return r
|
|
2367
|
+
}
|
|
2368
|
+
|
|
2369
|
+
|
|
2370
|
+
// `sensorRotationMatrix` was removed in V7 — the rotation chain it
|
|
2371
|
+
// powered is no longer in the homography path. See iOS' equivalent.
|