react-native-image-stitcher 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/CHANGELOG.md +96 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +21 -0
  4. package/README.md +189 -0
  5. package/RNImageStitcher.podspec +76 -0
  6. package/android/build.gradle +224 -0
  7. package/android/src/main/AndroidManifest.xml +3 -0
  8. package/android/src/main/cpp/CMakeLists.txt +124 -0
  9. package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
  10. package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
  11. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
  12. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
  13. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
  14. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
  15. package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
  16. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
  17. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
  18. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
  19. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
  20. package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
  21. package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
  22. package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
  23. package/cpp/ar_frame_pose.h +63 -0
  24. package/cpp/keyframe_gate.cpp +927 -0
  25. package/cpp/keyframe_gate.hpp +240 -0
  26. package/cpp/stitcher.cpp +2207 -0
  27. package/cpp/stitcher.hpp +275 -0
  28. package/dist/ar/useARSession.d.ts +102 -0
  29. package/dist/ar/useARSession.js +133 -0
  30. package/dist/camera/ARCameraView.d.ts +93 -0
  31. package/dist/camera/ARCameraView.js +170 -0
  32. package/dist/camera/Camera.d.ts +134 -0
  33. package/dist/camera/Camera.js +688 -0
  34. package/dist/camera/CameraShutter.d.ts +80 -0
  35. package/dist/camera/CameraShutter.js +237 -0
  36. package/dist/camera/CameraView.d.ts +65 -0
  37. package/dist/camera/CameraView.js +117 -0
  38. package/dist/camera/CaptureControlsBar.d.ts +87 -0
  39. package/dist/camera/CaptureControlsBar.js +82 -0
  40. package/dist/camera/CaptureHeader.d.ts +62 -0
  41. package/dist/camera/CaptureHeader.js +81 -0
  42. package/dist/camera/CapturePreview.d.ts +70 -0
  43. package/dist/camera/CapturePreview.js +188 -0
  44. package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
  45. package/dist/camera/CaptureStatusOverlay.js +326 -0
  46. package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
  47. package/dist/camera/CaptureThumbnailStrip.js +177 -0
  48. package/dist/camera/IncrementalPanGuide.d.ts +83 -0
  49. package/dist/camera/IncrementalPanGuide.js +267 -0
  50. package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
  51. package/dist/camera/PanoramaBandOverlay.js +399 -0
  52. package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
  53. package/dist/camera/PanoramaConfirmModal.js +128 -0
  54. package/dist/camera/PanoramaGuidance.d.ts +79 -0
  55. package/dist/camera/PanoramaGuidance.js +246 -0
  56. package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
  57. package/dist/camera/PanoramaSettingsModal.js +611 -0
  58. package/dist/camera/ViewportCropOverlay.d.ts +46 -0
  59. package/dist/camera/ViewportCropOverlay.js +67 -0
  60. package/dist/camera/useCapture.d.ts +111 -0
  61. package/dist/camera/useCapture.js +160 -0
  62. package/dist/camera/useDeviceOrientation.d.ts +48 -0
  63. package/dist/camera/useDeviceOrientation.js +131 -0
  64. package/dist/camera/useVideoCapture.d.ts +79 -0
  65. package/dist/camera/useVideoCapture.js +151 -0
  66. package/dist/index.d.ts +26 -0
  67. package/dist/index.js +39 -0
  68. package/dist/quality/normaliseOrientation.d.ts +36 -0
  69. package/dist/quality/normaliseOrientation.js +62 -0
  70. package/dist/quality/runQualityCheck.d.ts +41 -0
  71. package/dist/quality/runQualityCheck.js +98 -0
  72. package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
  73. package/dist/sensors/useIMUTranslationGate.js +235 -0
  74. package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
  75. package/dist/stitching/IncrementalStitcherView.js +157 -0
  76. package/dist/stitching/incremental.d.ts +930 -0
  77. package/dist/stitching/incremental.js +133 -0
  78. package/dist/stitching/stitchFrames.d.ts +55 -0
  79. package/dist/stitching/stitchFrames.js +56 -0
  80. package/dist/stitching/stitchVideo.d.ts +119 -0
  81. package/dist/stitching/stitchVideo.js +57 -0
  82. package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
  83. package/dist/stitching/useIncrementalJSDriver.js +199 -0
  84. package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
  85. package/dist/stitching/useIncrementalStitcher.js +172 -0
  86. package/dist/types.d.ts +58 -0
  87. package/dist/types.js +15 -0
  88. package/ios/Package.swift +72 -0
  89. package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
  90. package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
  91. package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
  92. package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
  93. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
  94. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
  95. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
  96. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
  97. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
  98. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
  99. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
  100. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
  101. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
  102. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
  104. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
  105. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
  106. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
  107. package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
  108. package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
  109. package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
  110. package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
  111. package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
  112. package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
  113. package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
  114. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
  115. package/package.json +73 -0
  116. package/react-native.config.js +34 -0
  117. package/scripts/opencv-version.txt +1 -0
  118. package/scripts/postinstall-fetch-binaries.js +286 -0
  119. package/src/ar/useARSession.ts +210 -0
  120. package/src/camera/.gitkeep +0 -0
  121. package/src/camera/ARCameraView.tsx +256 -0
  122. package/src/camera/Camera.tsx +1053 -0
  123. package/src/camera/CameraShutter.tsx +292 -0
  124. package/src/camera/CameraView.tsx +157 -0
  125. package/src/camera/CaptureControlsBar.tsx +204 -0
  126. package/src/camera/CaptureHeader.tsx +184 -0
  127. package/src/camera/CapturePreview.tsx +318 -0
  128. package/src/camera/CaptureStatusOverlay.tsx +391 -0
  129. package/src/camera/CaptureThumbnailStrip.tsx +277 -0
  130. package/src/camera/IncrementalPanGuide.tsx +328 -0
  131. package/src/camera/PanoramaBandOverlay.tsx +498 -0
  132. package/src/camera/PanoramaConfirmModal.tsx +206 -0
  133. package/src/camera/PanoramaGuidance.tsx +327 -0
  134. package/src/camera/PanoramaSettingsModal.tsx +1357 -0
  135. package/src/camera/ViewportCropOverlay.tsx +81 -0
  136. package/src/camera/useCapture.ts +279 -0
  137. package/src/camera/useDeviceOrientation.ts +140 -0
  138. package/src/camera/useVideoCapture.ts +236 -0
  139. package/src/index.ts +53 -0
  140. package/src/quality/.gitkeep +0 -0
  141. package/src/quality/normaliseOrientation.ts +79 -0
  142. package/src/quality/runQualityCheck.ts +131 -0
  143. package/src/sensors/useIMUTranslationGate.ts +347 -0
  144. package/src/stitching/.gitkeep +0 -0
  145. package/src/stitching/IncrementalStitcherView.tsx +198 -0
  146. package/src/stitching/incremental.ts +1021 -0
  147. package/src/stitching/stitchFrames.ts +88 -0
  148. package/src/stitching/stitchVideo.ts +153 -0
  149. package/src/stitching/useIncrementalJSDriver.ts +273 -0
  150. package/src/stitching/useIncrementalStitcher.ts +252 -0
  151. package/src/types.ts +78 -0
@@ -0,0 +1,2371 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ package io.imagestitcher.rn
3
+
4
+ import android.app.ActivityManager
5
+ import com.facebook.react.bridge.Arguments
6
+ import com.facebook.react.bridge.Promise
7
+ import com.facebook.react.bridge.ReactApplicationContext
8
+ import com.facebook.react.bridge.ReactContextBaseJavaModule
9
+ import com.facebook.react.bridge.ReactMethod
10
+ import com.facebook.react.bridge.ReadableMap
11
+ import com.facebook.react.bridge.WritableMap
12
+ import com.facebook.react.modules.core.DeviceEventManagerModule
13
+ import kotlin.math.max
14
+ import kotlin.math.min
15
+ import kotlinx.coroutines.CoroutineScope
16
+ import kotlinx.coroutines.Dispatchers
17
+ import kotlinx.coroutines.SupervisorJob
18
+ import kotlinx.coroutines.launch
19
+ import org.opencv.calib3d.Calib3d
20
+ import org.opencv.core.Core
21
+ import org.opencv.core.CvType
22
+ import org.opencv.core.DMatch
23
+ import org.opencv.core.KeyPoint
24
+ import org.opencv.core.Mat
25
+ import org.opencv.core.MatOfByte
26
+ import org.opencv.core.MatOfDMatch
27
+ import org.opencv.core.MatOfInt
28
+ import org.opencv.core.MatOfKeyPoint
29
+ import org.opencv.core.MatOfPoint2f
30
+ import org.opencv.core.Point
31
+ import org.opencv.core.Rect
32
+ import org.opencv.core.Scalar
33
+ import org.opencv.core.Size
34
+ import org.opencv.features2d.BFMatcher
35
+ import org.opencv.features2d.ORB
36
+ import org.opencv.imgcodecs.Imgcodecs
37
+ import org.opencv.imgproc.Imgproc
38
+ import java.io.File
39
+ import java.util.concurrent.atomic.AtomicBoolean
40
+
41
+ /**
42
+ * Android twin of iOS' OpenCVIncrementalStitcher + IncrementalStitcher.
43
+ *
44
+ * Why a single file (vs the iOS three-file split):
45
+ * On iOS we cross C++↔ObjC↔Swift boundaries, so the .h/.mm/.swift
46
+ * layering pays for itself. On Android, OpenCV's Java bindings
47
+ * give us cv::* operations directly callable from Kotlin — no JNI
48
+ * layer, no language boundary, the engine logic lives next to the
49
+ * RN module.
50
+ *
51
+ * Why we DON'T need cv::Stitcher (the missing piece on Android):
52
+ * The incremental algorithm only uses ORB + BFMatcher +
53
+ * findHomography + warpPerspective + distanceTransform. All of
54
+ * these ship in the prebuilt `libopencv_java4.so` (features2d,
55
+ * calib3d, imgproc are always-on modules). See the design doc for
56
+ * the full module-level breakdown.
57
+ *
58
+ * What the bridge exposes to JS:
59
+ * - start(options) — spin up the engine
60
+ * - processFrameAtPath() — feed a JPEG path + pose; engine returns
61
+ * the same outcome enum iOS emits as events
62
+ * - finalize(options) — write the final panorama and reset
63
+ * - cancel() — abort without producing output
64
+ * - getState() — pull the latest state on demand
65
+ * - Event "IncrementalStateUpdate" emitted on every
66
+ * processFrameAtPath call
67
+ *
68
+ * What's missing for true live capture on Android:
69
+ * ARCore-backed live frame delivery. The engine itself doesn't
70
+ * care where frames come from; today the only Android caller is
71
+ * the `processFrameAtPath` bridge method. A follow-up will plumb
72
+ * ARCore's per-frame `Frame.acquireCameraImage()` directly into
73
+ * the engine the same way iOS uses ARSession.
74
+ */
75
+ class IncrementalStitcher(
76
+ private val reactContext: ReactApplicationContext,
77
+ ) : ReactContextBaseJavaModule(reactContext) {
78
+
79
+ override fun getName(): String = "IncrementalStitcher"
80
+
81
+ /// Required by RCTEventEmitter contract. No-op on Android because
82
+ /// `DeviceEventManagerModule` does its own listener tracking; we
83
+ /// emit unconditionally and RN drops events when no listener is
84
+ /// attached.
85
+ @ReactMethod
86
+ fun addListener(eventName: String) { /* no-op */ }
87
+
88
+ @ReactMethod
89
+ fun removeListeners(count: Int) { /* no-op */ }
90
+
91
+ /// V7 hybrid engine — selected for engineMode == 'hybrid'.
92
+ private var engine: IncrementalEngine? = null
93
+ /// V12.7 firstwins engine — selected for any engineMode starting
94
+ /// with 'firstwins' (firstwins, firstwins-zoomed, firstwins-rectilinear).
95
+ /// Native engine is identical for firstwins and firstwins-zoomed
96
+ /// (the difference is JS-side viewport zoom only). useRectilinear
97
+ /// is set for 'firstwins-rectilinear'.
98
+ private var firstwinsEngine: IncrementalFirstwinsEngine? = null
99
+
100
+ // ── V16 batch-keyframe mode (Android parity with iOS' V16 Phase 1) ─
101
+ //
102
+ // Selected for engineMode == 'batch-keyframe'. No live engine
103
+ // runs — instead, accepted frames are collected as keyframe paths,
104
+ // and at finalize() time we hand them all to the JNI shim
105
+ // (libimage_stitcher.so) for one-shot cv::Stitcher processing.
106
+ //
107
+ // The MVP gate is frame-count-based ("accept every Nth frame
108
+ // until cap"). iOS uses a pose-based gate (overlap < threshold)
109
+ // — adding that here is a follow-up that needs ARCore-pose
110
+ // accumulation across processFrameAtPath calls. For now, every
111
+ // N-th frame is good enough to validate end-to-end stitching
112
+ // parity.
113
+ private var batchKeyframeMode: Boolean = false
114
+ private val batchKeyframePaths: MutableList<String> = mutableListOf()
115
+ private var batchKeyframeFrameCounter: Int = 0
116
+ /// V16 Phase 2 (Android Fix-1) — per-capture-session subdirectory
117
+ /// under `cacheDir` where this capture's batch-keyframe JPEGs are
118
+ /// written. Created on each batch-keyframe `start()` with a fresh
119
+ /// UUID.
120
+ ///
121
+ /// Why:
122
+ /// The V16 Phase-1 MVP wrote every accepted keyframe to
123
+ /// `cacheDir/rlis-keyframe-{N}.jpg` where N restarted at 0 on
124
+ /// each capture. Two captures in a row → second capture's
125
+ /// `rlis-keyframe-0.jpg` overwrites the first's. Worse, RN's
126
+ /// `<Image>` component on Android caches decoded bitmaps keyed
127
+ /// by URI string; the file:// URI was byte-identical across
128
+ /// captures, so the previous capture's bitmap got served for
129
+ /// the new capture's first thumbnail — Ram's "thumbnails come
130
+ /// from the previous capture" symptom (2026-05-12). Also a
131
+ /// data-integrity hazard if a new capture starts while the
132
+ /// previous one's stitcher is still reading the JPEGs from
133
+ /// disk.
134
+ ///
135
+ /// Per-session UUID subdir fixes both:
136
+ /// - Each capture's keyframes live at a unique path → no URI
137
+ /// collision → no bitmap-cache reuse across captures.
138
+ /// - Files survive past finalize for post-hoc reprocessing
139
+ /// (Ram's request — same behaviour as iOS' OpenCVKeyframeCollector).
140
+ ///
141
+ /// Lifetime:
142
+ /// • Created in `start()` batch-keyframe branch.
143
+ /// • Used by `copyKeyframeToStore()`.
144
+ /// • Persists past `finalize()` for reprocessing.
145
+ /// • Cleaned up on `cancel()` and in `onCatalystInstanceDestroy()`.
146
+ ///
147
+ /// Parity: matches iOS `OpenCVKeyframeCollector.sessionDir`
148
+ /// (created with `Library/AppSupport/Captures/{NSUUID}/`).
149
+ private var captureSessionDir: java.io.File? = null
150
+ /// Accept every Nth frame. 10 is the iOS default capture cadence
151
+ /// (5-6 keyframes over a ~2-3 second pan = roughly one every 10
152
+ /// frames at 30fps).
153
+ private var batchKeyframeAcceptStride: Int = 10
154
+ /// Hard cap on keyframes to match iOS' default (V16 Phase 1's
155
+ /// keyframeMaxCount=6). Going higher inflates cv::Stitcher's
156
+ /// MultiBandBlender memory; iOS hit OOM at 7+ on some scenes.
157
+ private var batchKeyframeMaxCount: Int = 6
158
+ /// Batch knobs threaded through to nativeStitchFramePaths at
159
+ /// finalize. Mirror iOS' batchWarperType / batchBlenderType /
160
+ /// batchSeamFinderType / batchEnableInscribedRectCrop ivars.
161
+ private var batchWarperType: String = "plane"
162
+ private var batchBlenderType: String = "multiband"
163
+ private var batchSeamFinderType: String = "graphcut"
164
+ private var batchUseInscribedRectCrop: Boolean = false
165
+ /// Capture orientation at start time. Drives the bake-rotation
166
+ /// table inside the JNI shim. Sourced from configOverrides
167
+ /// (passed from JS), falling back to "portrait".
168
+ private var batchCaptureOrientation: String = "portrait"
169
+
170
+ // ── 2026-05-14: cv::Stitcher pipeline-mode auto-routing ──────────
171
+ //
172
+ // `batchStitchMode` is the JS-supplied setting from
173
+ // PanoramaSettings.stitchMode. Three valid values:
174
+ // 'auto' (default) — at finalize() time, compute translation/
175
+ // rotation totals from the first and last
176
+ // accepted keyframe pose, pick PANORAMA or
177
+ // SCANS by the design-doc 0.55 threshold.
178
+ // 'panorama' — force cv::Stitcher::PANORAMA mode at JNI.
179
+ // 'scans' — force cv::Stitcher::SCANS mode at JNI.
180
+ //
181
+ // batchFirstAcceptedPose / batchLastAcceptedPose are populated by
182
+ // ingestFromARCameraView() on every accepted keyframe. Cleared
183
+ // at start() and consumed at finalize(). They store (tx, ty, tz,
184
+ // qx, qy, qz, qw) — same shape as `KeyframeGate`'s internal
185
+ // last-accepted-pose tracker, but kept locally so we don't have
186
+ // to wire a new accessor through the C++ bridge.
187
+ private var batchStitchMode: String = "auto"
188
+ private var batchFirstAcceptedPose: DoubleArray? = null
189
+ private var batchLastAcceptedPose: DoubleArray? = null
190
+
191
+ /// V16 Phase 1 / P3-F — shared-C++ KeyframeGate. Replaces the
192
+ /// V16-Phase-1 frame-counter MVP placeholder
193
+ /// (handleBatchKeyframeFrame above) with the same pose-driven
194
+ /// 40%-new-content algorithm iOS has used since the V16 ship.
195
+ /// Both platforms call into retailens::KeyframeGate (in
196
+ /// retailens-capture-sdk/cpp/keyframe_gate.cpp) — see that file
197
+ /// for the algorithm.
198
+ ///
199
+ /// Lifetime: owned for the life of the module. Closed in
200
+ /// `onCatalystInstanceDestroy()` to release the C++ heap
201
+ /// allocation. `reset()` is called between captures.
202
+ private val keyframeGate = KeyframeGate()
203
+
204
+ /// P3-G diagnostic — rate-limit the per-frame log in
205
+ /// ingestFromARCameraView so we don't spam logcat at 60Hz but
206
+ /// still see the "0 frames captured" mystery resolve to a
207
+ /// specific failure mode.
208
+ private var frameIngestLogTick: Int = 0
209
+
210
+ private val isRunning = AtomicBoolean(false)
211
+ /// Critic #5 fix: serial dispatcher so concurrent
212
+ /// processFrameAtPath() calls can't race on the engine's canvas.
213
+ /// `limitedParallelism(1)` guarantees one-at-a-time execution
214
+ /// while still backing onto the Default pool — matches iOS'
215
+ /// `workQueue` (DispatchQueue.serial).
216
+ @OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class)
217
+ private val workScope = CoroutineScope(Dispatchers.Default.limitedParallelism(1))
218
+
219
+ /// 2026-05-16 — realtime+batch fusion (Option A "Replace on
220
+ /// completion") scope. Kept SEPARATE from `workScope` so the
221
+ /// 2-5 s `cv::Stitcher` refinement run that follows a hybrid-
222
+ /// engine finalize() does NOT delay a new start()/processFrame()
223
+ /// that the operator may issue while the refinement is in flight.
224
+ /// The design doc explicitly calls out "operator can continue
225
+ /// browsing / starting another capture during refinement".
226
+ ///
227
+ /// Serial: at most one refinement runs at a time (the design's
228
+ /// "cancellation semantics if a new capture starts mid-refine"
229
+ /// is out of scope for this MVP).
230
+ ///
231
+ /// 2026-05-16 (Phase 3 critic MED-1) — `SupervisorJob()` keeps
232
+ /// the scope alive when a single refinement coroutine fails.
233
+ /// Every `refineScope.launch { … }` already has a try/catch
234
+ /// around the throwing surface; SupervisorJob is defense-in-
235
+ /// depth for future code added outside that catch.
236
+ @OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class)
237
+ private val refineScope = CoroutineScope(
238
+ SupervisorJob() + Dispatchers.Default.limitedParallelism(1)
239
+ )
240
+
241
+ /// Reference to a mounted ARCameraView (if any). Set by the view
242
+ /// when it attaches; the engine flips its `ingestActive` flag
243
+ /// on start/stop so the view feeds frames only during a capture.
244
+ @Volatile private var arCameraViewRef: RNSARCameraView? = null
245
+
246
+ init {
247
+ // Static back-pointer so `RNSARCameraView` can call into
248
+ // the singleton-style bridge module without a DI dance. RN
249
+ // may rebuild module instances across reloads; the view always
250
+ // uses the latest reference.
251
+ bridgeInstance = this
252
+ }
253
+
254
+ /// View calls this on attach so the engine can route ingestion
255
+ /// without searching the view tree on every frame.
256
+ internal fun bindArCameraView(view: RNSARCameraView) {
257
+ arCameraViewRef = view
258
+ // If a capture is already running when the view mounts, hot-
259
+ // engage ingestion so the user gets a partial panorama
260
+ // started from this point onward.
261
+ if (isRunning.get()) {
262
+ view.setIncrementalIngestionActive(true)
263
+ }
264
+ }
265
+
266
+ internal fun unbindArCameraView(view: RNSARCameraView) {
267
+ if (arCameraViewRef === view) {
268
+ view.setIncrementalIngestionActive(false)
269
+ arCameraViewRef = null
270
+ }
271
+ }
272
+
273
+ @ReactMethod
274
+ fun start(options: ReadableMap, promise: Promise) {
275
+ if (isRunning.getAndSet(true)) {
276
+ promise.reject(
277
+ "incremental-already-running",
278
+ "An incremental capture is already in progress.",
279
+ )
280
+ return
281
+ }
282
+ try {
283
+ ensureOpenCv()
284
+ val rotation = options.getIntOrDefault("frameRotationDegrees", 90)
285
+ val composeW = options.getIntOrDefault("composeWidth", 960)
286
+ val composeH = options.getIntOrDefault("composeHeight", 720)
287
+ // V12 default canvas: 5000x5000 to match iOS. Old default
288
+ // was 4800x2200 (V7 wide-only); V12 needs square because
289
+ // either pan axis can grow.
290
+ val canvasW = options.getIntOrDefault("canvasWidth", 5000)
291
+ val canvasH = options.getIntOrDefault("canvasHeight", 5000)
292
+ val featherP = options.getIntOrDefault("featherPx", 20)
293
+ val snapQ = max(1, min(100, options.getIntOrDefault("snapshotJpegQuality", 75)))
294
+ // Critic #29: clamp snapshotEveryNAccepts to >= 1 so a
295
+ // value of 0 doesn't mean "snapshot every frame forever".
296
+ val snapN = max(1, options.getIntOrDefault("snapshotEveryNAccepts", 1))
297
+ // V12.7 — engineMode now distinguishes 4 variants. See
298
+ // src/stitching/incremental.ts for the full description.
299
+ // V16 added 'batch-keyframe' as a fifth variant: no live
300
+ // engine, frames are saved as JPEGs and handed to
301
+ // cv::Stitcher (via the JNI shim) at finalize.
302
+ val engineMode = options.getString("engine") ?: "hybrid"
303
+ // 2026-05-15 — Route 'slitscan*' engineModes to the same
304
+ // IncrementalFirstwinsEngine that handles 'firstwins*'.
305
+ // Per IncrementalFirstwinsEngine's docstring (lines 260,
306
+ // 431, 436, 672, 957): "Mirrors iOS' OpenCVSlitScanStitcher.mm
307
+ // exactly". Before this change, 'slitscan' / 'slitscan-rotate'
308
+ // / 'slitscan-both' engineModes fell through to IncrementalEngine
309
+ // (the hybrid engine), producing identical output to picking
310
+ // 'hybrid' — silent platform divergence vs iOS.
311
+ //
312
+ // iOS-parity reference: IncrementalStitcher.swift:556
313
+ // computes `useFirstwinsClass = normalisedMode.hasPrefix("slitscan")`
314
+ // which routes BOTH 'slitscan-rotate' AND 'slitscan-both' AND
315
+ // the deprecated aliases to OpenCVFirstWinsCylindricalStitcher.
316
+ // We mirror that logic here so Android Settings → Engine
317
+ // dropdown actually toggles the underlying engine.
318
+ val isFirstwinsClass =
319
+ engineMode.startsWith("firstwins") ||
320
+ engineMode.startsWith("slitscan")
321
+ val isFirstwins = isFirstwinsClass // legacy name kept
322
+ // for the remainder of
323
+ // start() — refactor to
324
+ // isFirstwinsClass when
325
+ // the engineMode taxonomy
326
+ // is rationalised.
327
+ val useRectilinear =
328
+ engineMode == "firstwins-rectilinear" ||
329
+ engineMode == "slitscan-rotate"
330
+ val isBatchKeyframe = engineMode == "batch-keyframe"
331
+
332
+ val configOverrides: ReadableMap? =
333
+ if (options.hasKey("config")) options.getMap("config") else null
334
+
335
+ if (isBatchKeyframe) {
336
+ // No live engine runs. Reset the keyframe collector
337
+ // state. Read knobs from `config` per the V16 Phase
338
+ // 1 plumbing pattern.
339
+ engine = null
340
+ firstwinsEngine = null
341
+ batchKeyframeMode = true
342
+ batchKeyframePaths.clear()
343
+ batchKeyframeFrameCounter = 0
344
+ // V16 Phase 2 (Android Fix-1) — fresh per-session subdir
345
+ // for this capture's keyframe JPEGs. Replaces the
346
+ // V16-Phase-1 "rlis-keyframe-{N}.jpg in cacheDir"
347
+ // scheme that caused thumbnails from a previous capture
348
+ // to leak into the next one via RN's bitmap cache (see
349
+ // `captureSessionDir` declaration above for the full
350
+ // RCA). Matches iOS' OpenCVKeyframeCollector behaviour.
351
+ captureSessionDir = java.io.File(
352
+ reactContext.cacheDir,
353
+ "rlis-capture-${java.util.UUID.randomUUID()}",
354
+ ).also { it.mkdirs() }
355
+ batchKeyframeMaxCount = configOverrides
356
+ ?.getIntOrDefault("keyframeMaxCount", 6) ?: 6
357
+ batchWarperType = configOverrides?.getString("warperType")
358
+ ?: "plane"
359
+ batchBlenderType = configOverrides?.getString("blenderType")
360
+ ?: "multiband"
361
+ batchSeamFinderType = configOverrides?.getString("seamFinderType")
362
+ ?: "graphcut"
363
+ batchUseInscribedRectCrop = configOverrides
364
+ ?.getBooleanOrDefault("enableMaxInscribedRectCrop", false)
365
+ ?: false
366
+ // 2026-05-14 — stitch-mode picker from JS Settings.
367
+ // Default 'auto'. Validated against the closed set
368
+ // {auto, panorama, scans}; unknown values fall back
369
+ // to 'auto'. Reset accumulated-pose state for the
370
+ // new capture so finalize() picks a fresh mode.
371
+ batchStitchMode = (configOverrides?.getString("stitchMode") ?: "auto")
372
+ .let { if (it in setOf("auto", "panorama", "scans")) it else "auto" }
373
+ batchFirstAcceptedPose = null
374
+ batchLastAcceptedPose = null
375
+ // captureOrientation is JS-supplied here (Android
376
+ // doesn't yet have a native ARCore classifier
377
+ // equivalent to iOS' nativeCaptureOrientation; the
378
+ // JS hook is stale but at least it's directional).
379
+ batchCaptureOrientation = options.getString("captureOrientation")
380
+ ?: "portrait"
381
+ // P3-F — configure the shared-C++ KeyframeGate for
382
+ // this capture. Same knob set + defaults as iOS:
383
+ // overlapThreshold default 0.4 (40% new content)
384
+ // maxCount default 6
385
+ // Both clamped to safe ranges that iOS also uses (see
386
+ // IncrementalStitcher.swift:608-615).
387
+ val threshold = configOverrides
388
+ ?.getDoubleOrDefault("keyframeOverlapThreshold", 0.4) ?: 0.4
389
+ keyframeGate.overlapThreshold = threshold.coerceIn(0.10, 0.80)
390
+ keyframeGate.maxCount = batchKeyframeMaxCount.coerceIn(3, 10)
391
+
392
+ // 2026-05-14 — thread flow-strategy tunables through to the
393
+ // shared C++ gate. Before this commit the Android JNI was
394
+ // missing setFlowNoveltyPercentile + setFlowMaxTranslationM
395
+ // bindings (iOS-only via KeyframeGateBridge), which meant
396
+ // operators flipping these in Settings only affected iOS
397
+ // captures. Now both platforms honour them.
398
+ val pctile = configOverrides
399
+ ?.getDoubleOrDefault("flowNoveltyPercentile", 0.85) ?: 0.85
400
+ keyframeGate.flowNoveltyPercentile = pctile.coerceIn(0.50, 0.99)
401
+ // Settings UI exposes flowMaxTranslationCm in CENTIMETRES;
402
+ // C++ API is in METRES. Convert. 0 = disabled.
403
+ val txBudgetCm = configOverrides
404
+ ?.getDoubleOrDefault("flowMaxTranslationCm", 0.0) ?: 0.0
405
+ keyframeGate.flowMaxTranslationM = (txBudgetCm / 100.0).coerceAtLeast(0.0)
406
+
407
+ // 2026-05-14 — non-AR mode opt-out for angular fallback.
408
+ // captureSource ∈ {wide, ultrawide} means the host is using
409
+ // vision-camera (no ARKit/ARCore pose). Disable the gate's
410
+ // angular fallback so it doesn't compute on garbage pose.
411
+ val captureSource = configOverrides?.getString("captureSource") ?: "auto"
412
+ val isNonAR = (captureSource == "wide" || captureSource == "ultrawide")
413
+ keyframeGate.disableAngularFallback = isNonAR
414
+
415
+ keyframeGate.enabled = true
416
+ keyframeGate.reset()
417
+ } else if (isFirstwins) {
418
+ batchKeyframeMode = false
419
+ batchKeyframePaths.clear()
420
+ keyframeGate.enabled = false // gate is batch-only; off for live engines
421
+ firstwinsEngine = IncrementalFirstwinsEngine(
422
+ composeWidth = composeW,
423
+ composeHeight = composeH,
424
+ canvasWidth = canvasW,
425
+ canvasHeight = canvasH,
426
+ snapshotJpegQuality = snapQ,
427
+ snapshotEveryNAccepts = snapN,
428
+ frameRotationDegrees = rotation,
429
+ useRectilinear = useRectilinear,
430
+ // Critic #27 fix: writable app-sandbox dir for
431
+ // live-snapshot JPEGs. java.io.tmpdir resolves to
432
+ // /data/local/tmp on Android (rooted-only).
433
+ snapshotCacheDir = reactContext.cacheDir.absolutePath,
434
+ )
435
+ engine = null
436
+ } else {
437
+ batchKeyframeMode = false
438
+ batchKeyframePaths.clear()
439
+ keyframeGate.enabled = false // gate is batch-only; off for hybrid engine
440
+ engine = IncrementalEngine(
441
+ composeWidth = composeW,
442
+ composeHeight = composeH,
443
+ canvasWidth = canvasW,
444
+ canvasHeight = canvasH,
445
+ featherPx = featherP,
446
+ snapshotJpegQuality = snapQ,
447
+ snapshotEveryNAccepts = snapN,
448
+ frameRotationDegrees = rotation,
449
+ )
450
+ firstwinsEngine = null
451
+ }
452
+ // Engage the ARCameraView's per-frame ingestion path if a
453
+ // view is mounted — this is what gives Android parity
454
+ // with iOS' ARSession-driven path. No-op when the view
455
+ // isn't mounted (host is using vision-camera + the gyro
456
+ // driver from useIncrementalAndroidDriver instead).
457
+ arCameraViewRef?.setIncrementalIngestionActive(true)
458
+
459
+ // ── P3-G diagnostic ──────────────────────────────────
460
+ // Surfaces start() state so we can see in logcat: (a)
461
+ // engine mode actually selected, (b) batchKeyframeMode
462
+ // flag, (c) KeyframeGate config, (d) whether the AR
463
+ // view is bound. Each of these is a potential failure
464
+ // point for the "0 keyframes captured" symptom.
465
+ android.util.Log.i(
466
+ "IncrementalStitcher",
467
+ "start() ENTRY: engineMode=$engineMode " +
468
+ "batchKeyframeMode=$batchKeyframeMode " +
469
+ "gate.enabled=${keyframeGate.enabled} " +
470
+ "gate.maxCount=${keyframeGate.maxCount} " +
471
+ "gate.threshold=${keyframeGate.overlapThreshold} " +
472
+ "arCameraViewBound=${arCameraViewRef != null} " +
473
+ "isRunning=${isRunning.get()}",
474
+ )
475
+
476
+ val map = Arguments.createMap()
477
+ map.putBoolean("ok", true)
478
+ promise.resolve(map)
479
+ } catch (t: Throwable) {
480
+ isRunning.set(false)
481
+ promise.reject("incremental-start-failed", t.message, t)
482
+ }
483
+ }
484
+
485
+ /**
486
+ * Feed one frame at a JPEG path into the engine. Pose inputs
487
+ * drive the same FoV-overlap gate as iOS. When a pose source
488
+ * isn't available pass yaw=0, pitch=0, fovHorizDegrees=0 — the
489
+ * engine treats fov<=0 as a sentinel for "no intrinsics" and
490
+ * substitutes a 65° default, so frames will still be processed,
491
+ * just less gated.
492
+ */
493
+ /**
494
+ * Copy a (non-persistent) source JPEG to a persistent per-keyframe
495
+ * path under the React context's cache dir. The ARCameraView's
496
+ * forwardToIncremental writes every frame to a SINGLE reused tmp
497
+ * file (rlis-arframe.jpg) — adequate for the live engines that
498
+ * decode synchronously, but the batch-keyframe collector
499
+ * accumulates paths for stitching at finalize time, so each
500
+ * keyframe needs its own stable file.
501
+ *
502
+ * Returns the absolute path of the destination on success, or
503
+ * null if the copy failed. Cost ≈ 3-5 ms for a 1080p JPEG on
504
+ * iPhone 16 / Galaxy A35 class hardware.
505
+ *
506
+ * Naming: `rlis-keyframe-{N}.jpg` where N is the next slot index
507
+ * (= batchKeyframePaths.size). Survives until either the next
508
+ * batch-keyframe capture overwrites the same slot or the OS
509
+ * cleans the cache dir. iOS counterpart writes per-session
510
+ * uuid-dirs via OpenCVKeyframeCollector — that's deeper parity
511
+ * for a Phase 3 follow-up; this MVP is just enough to make
512
+ * batch-keyframe work end-to-end on Android.
513
+ */
514
+ private fun copyKeyframeToStore(srcPath: String): String? {
515
+ // V16 Phase 2 (Android Fix-1) — write into the per-session
516
+ // subdir created by start(). If start() didn't run (defensive
517
+ // — should never happen on the live ingest path), the
518
+ // captureSessionDir is null and we drop the frame; the older
519
+ // "rlis-keyframe-{N}.jpg in cacheDir" fallback is GONE because
520
+ // it was the source of the cross-capture cache bug.
521
+ val dir = captureSessionDir
522
+ if (dir == null) {
523
+ android.util.Log.w(
524
+ "IncrementalStitcher",
525
+ "copyKeyframeToStore: captureSessionDir is null — " +
526
+ "start() should have created it; dropping frame",
527
+ )
528
+ return null
529
+ }
530
+ val destFile = java.io.File(dir, "keyframe-${batchKeyframePaths.size}.jpg")
531
+ return try {
532
+ java.io.File(srcPath).copyTo(destFile, overwrite = true).absolutePath
533
+ } catch (e: Exception) {
534
+ android.util.Log.w(
535
+ "IncrementalStitcher",
536
+ "copyKeyframeToStore: failed to copy $srcPath → " +
537
+ "${destFile.absolutePath}: ${e.message}",
538
+ e,
539
+ )
540
+ null
541
+ }
542
+ }
543
+
544
+ // ── V16 Phase 1 → P3-F migration note ────────────────────────
545
+ // The frame-counter placeholder gate `handleBatchKeyframeFrame`
546
+ // that lived here has been REMOVED. Both the AR-driven path
547
+ // (ingestFromARCameraView) and the vision-camera fallback path
548
+ // (processFrameAtPath) now route through the shared-C++
549
+ // `KeyframeGate` (cpp/keyframe_gate.{hpp,cpp}) — same algorithm
550
+ // iOS has used since the V16 ship. See
551
+ // `private val keyframeGate = KeyframeGate()` above for the
552
+ // instance + lifetime.
553
+ //
554
+ // `batchKeyframeAcceptStride` is no longer consulted (the proper
555
+ // gate uses pose-driven overlap, not a frame-counter stride);
556
+ // the field is kept around for now because removing it would
557
+ // touch unrelated init/serialization paths. Wire it back in if
558
+ // we ever add a "force every Nth frame regardless of overlap"
559
+ // override.
560
+
561
+ @ReactMethod
562
+ fun processFrameAtPath(options: ReadableMap, promise: Promise) {
563
+ val hybrid = this.engine
564
+ val firstwins = this.firstwinsEngine
565
+ // batch-keyframe mode runs without a live engine — handle it
566
+ // up-front before the null-check rejects.
567
+ //
568
+ // P3-F: this path uses the same shared-C++ KeyframeGate as
569
+ // the AR view path, but with translation = 0 (gyro-derived
570
+ // poses have only rotation, not position). The gate's
571
+ // internal logic detects the missing plane and uses the
572
+ // camera-forward angular-delta fallback automatically.
573
+ if (batchKeyframeMode) {
574
+ val path = options.getString("path")
575
+ ?: return promise.reject("invalid-options", "path required")
576
+ val pose = RNSARFramePose(
577
+ tx = options.getDoubleOrDefault("tx", 0.0) ?: 0.0,
578
+ ty = options.getDoubleOrDefault("ty", 0.0) ?: 0.0,
579
+ tz = options.getDoubleOrDefault("tz", 0.0) ?: 0.0,
580
+ qx = options.getDoubleOrDefault("qx", 0.0) ?: 0.0,
581
+ qy = options.getDoubleOrDefault("qy", 0.0) ?: 0.0,
582
+ qz = options.getDoubleOrDefault("qz", 0.0) ?: 0.0,
583
+ qw = options.getDoubleOrDefault("qw", 1.0) ?: 1.0,
584
+ fx = options.getDoubleOrDefault("fx", 1000.0) ?: 1000.0,
585
+ fy = options.getDoubleOrDefault("fy", 1000.0) ?: 1000.0,
586
+ cx = options.getDoubleOrDefault("cx", 540.0) ?: 540.0,
587
+ cy = options.getDoubleOrDefault("cy", 960.0) ?: 960.0,
588
+ imageWidth = options.getIntOrDefault("imageWidth", 1080),
589
+ imageHeight = options.getIntOrDefault("imageHeight", 1920),
590
+ timestampMs = 0.0,
591
+ trackingState = RNSARSession.TRACKING_TRACKING,
592
+ )
593
+ // Vision-camera path: no plane available (gyro can't fit
594
+ // planes). Pass null → C++ uses angular fallback.
595
+ val decision = keyframeGate.evaluate(pose, null)
596
+ val result = Arguments.createMap()
597
+ // Outcome mapping for iOS-parity JS contract:
598
+ // 1 = accepted, 2 = rejected (gate), 3 = rejected (cap).
599
+ // C++ enum → outcome int:
600
+ val outcome = when {
601
+ decision.accept -> 1
602
+ decision.reason == "max-reached" -> 3
603
+ else -> 2
604
+ }
605
+ result.putInt("outcome", outcome)
606
+ if (decision.accept) {
607
+ batchKeyframePaths.add(path) // vision-camera path
608
+ // gives us a unique
609
+ // per-snapshot file
610
+ // already (no copy
611
+ // needed).
612
+ // Emit the same state event the AR path emits so
613
+ // the JS LiveFrameStrip + "Keyframes n/max" pill
614
+ // work identically on the vision-camera fallback
615
+ // path.
616
+ emitBatchKeyframeAcceptedState(
617
+ thumbnailPath = path,
618
+ keyframeIndex = batchKeyframePaths.size - 1,
619
+ keyframeCount = batchKeyframePaths.size,
620
+ keyframeMax = keyframeGate.maxCount,
621
+ isLandscape = pose.imageWidth >= pose.imageHeight,
622
+ )
623
+ }
624
+ result.putInt("acceptedCount", batchKeyframePaths.size)
625
+ promise.resolve(result)
626
+ return
627
+ }
628
+ if (hybrid == null && firstwins == null) {
629
+ return promise.reject(
630
+ "incremental-not-running",
631
+ "Call start() before processFrameAtPath().",
632
+ )
633
+ }
634
+ val path = options.getString("path")
635
+ ?: return promise.reject("invalid-options", "path required")
636
+ val yaw = options.getDoubleOrDefault("yaw", 0.0)
637
+ val pitch = options.getDoubleOrDefault("pitch", 0.0)
638
+ val fovH = options.getDoubleOrDefault("fovHorizDegrees", 65.0)
639
+ val fovV = options.getDoubleOrDefault("fovVertDegrees", 50.0)
640
+ // V6 pose-driven params. Defaults removed per critic finding
641
+ // #3: previously qw=1.0 default meant frames without explicit
642
+ // quaternion produced an identity rotation, and EVERY
643
+ // subsequent frame had R_rel = R_first^T (constant), so
644
+ // strip placement never advanced and `acceptedCount` froze
645
+ // at 1 after the first frame. Now every quaternion field is
646
+ // required; missing → reject as RejectedAlignmentLost so the
647
+ // gyro driver upstream notices instantly.
648
+ if (!options.hasKey("qx") || !options.hasKey("qy")
649
+ || !options.hasKey("qz") || !options.hasKey("qw")) {
650
+ return promise.reject(
651
+ "invalid-options",
652
+ "qx/qy/qz/qw all required (no identity-quaternion fallback)",
653
+ )
654
+ }
655
+ val qx = options.getDouble("qx")
656
+ val qy = options.getDouble("qy")
657
+ val qz = options.getDouble("qz")
658
+ val qw = options.getDouble("qw")
659
+ val fx = options.getDoubleOrDefault("fx", 0.0)
660
+ val fy = options.getDoubleOrDefault("fy", 0.0)
661
+ val cx = options.getDoubleOrDefault("cx", 0.0)
662
+ val cy = options.getDoubleOrDefault("cy", 0.0)
663
+ val imageWidth = options.getIntOrDefault("imageWidth", 0)
664
+ val imageHeight = options.getIntOrDefault("imageHeight", 0)
665
+ val trackingPoor = options.getBooleanOrDefault("trackingPoor", false)
666
+
667
+ workScope.launch {
668
+ // Critic #4 fix: re-check isRunning synchronously here in
669
+ // case finalize/cancel ran on the JS thread between the
670
+ // null-check above and this dispatch landing. Skip the
671
+ // ingest if we're no longer running — matches iOS' V12.1
672
+ // pattern (synchronous-stop + worker re-check).
673
+ if (!isRunning.get()) {
674
+ promise.resolve(Arguments.createMap().apply { putInt("outcome", -1) })
675
+ return@launch
676
+ }
677
+ try {
678
+ val telemetry: FrameTelemetry
679
+ val state: WritableMap?
680
+ val accepted: Int
681
+ if (firstwins != null) {
682
+ telemetry = firstwins.addFrameAtPath(
683
+ path = path,
684
+ qx = qx, qy = qy, qz = qz, qw = qw,
685
+ fx = fx, fy = fy, cx = cx, cy = cy,
686
+ imageWidth = imageWidth, imageHeight = imageHeight,
687
+ yaw = yaw, pitch = pitch,
688
+ fovHorizDegrees = fovH, fovVertDegrees = fovV,
689
+ trackingPoor = trackingPoor,
690
+ )
691
+ state = firstwins.snapshotIfDue(telemetry)
692
+ accepted = firstwins.acceptedCount
693
+ } else {
694
+ telemetry = hybrid!!.addFrameAtPath(
695
+ path = path,
696
+ qx = qx, qy = qy, qz = qz, qw = qw,
697
+ fx = fx, fy = fy, cx = cx, cy = cy,
698
+ imageWidth = imageWidth, imageHeight = imageHeight,
699
+ yaw = yaw, pitch = pitch,
700
+ fovHorizDegrees = fovH, fovVertDegrees = fovV,
701
+ trackingPoor = trackingPoor,
702
+ )
703
+ state = hybrid.snapshotIfDue(telemetry)
704
+ accepted = hybrid.acceptedCount
705
+ }
706
+ emitState(state)
707
+ val result = Arguments.createMap()
708
+ result.putInt("outcome", telemetry.outcome.ordinal)
709
+ result.putDouble("confidence", telemetry.confidence)
710
+ result.putDouble("overlapPercent", telemetry.overlapPercent)
711
+ result.putDouble("processingMs", telemetry.processingMs)
712
+ result.putInt("acceptedCount", accepted)
713
+ promise.resolve(result)
714
+ } catch (t: Throwable) {
715
+ promise.reject("incremental-process-failed", t.message, t)
716
+ }
717
+ }
718
+ }
719
+
720
+ @ReactMethod
721
+ fun finalize(options: ReadableMap, promise: Promise) {
722
+ val hybrid = this.engine
723
+ val firstwins = this.firstwinsEngine
724
+ if (hybrid == null && firstwins == null && !batchKeyframeMode) {
725
+ return promise.reject(
726
+ "incremental-not-running",
727
+ "No active capture — call start() first.",
728
+ )
729
+ }
730
+ val outputPathOpt = options.getString("outputPath") ?: ""
731
+ val outputPath = if (outputPathOpt.isEmpty()) {
732
+ File(reactContext.cacheDir, "RNImageStitcherIncremental-${System.nanoTime()}.jpg").absolutePath
733
+ } else {
734
+ outputPathOpt
735
+ }
736
+ val quality = options.getIntOrDefault("quality", 90)
737
+ // 2026-05-18 (iOS cross-orientation fix; symmetric on Android) —
738
+ // JS may pass a fresh deviceOrientation at finalize time; if
739
+ // so, override batchCaptureOrientation BEFORE we snapshot it
740
+ // for the stitcher. Empty/missing → keep legacy start-time
741
+ // value. Android cross-orientation was already working per
742
+ // user test (likely because users tested fewer rotation
743
+ // sequences here), but propagating the fresh value uniformly
744
+ // closes the same hole iOS had.
745
+ val freshOrientationOpt = options.getString("captureOrientation") ?: ""
746
+ if (freshOrientationOpt.isNotEmpty()) {
747
+ batchCaptureOrientation = freshOrientationOpt
748
+ }
749
+
750
+ // Disengage the ARCameraView ingestion path FIRST so no late
751
+ // frames slip into the engine while we serialize the canvas.
752
+ arCameraViewRef?.setIncrementalIngestionActive(false)
753
+ // Critic #4 fix: synchronously flip isRunning=false BEFORE
754
+ // dispatching the finalize body, so any in-flight
755
+ // processFrameAtPath workers that are about to launch will
756
+ // bail at the re-check (see processFrameAtPath above).
757
+ // Matches iOS V12.1 fix.
758
+ isRunning.set(false)
759
+
760
+ // V16 batch-keyframe finalize: snapshot the keyframe state
761
+ // synchronously under the same "stop ingestion before
762
+ // dispatching" pattern, then null out the live state.
763
+ val wasBatchKeyframe = batchKeyframeMode
764
+ val keyframePathsSnapshot = batchKeyframePaths.toList()
765
+ val captureOrientationSnapshot = batchCaptureOrientation
766
+ val warperTypeSnapshot = batchWarperType
767
+ val blenderTypeSnapshot = batchBlenderType
768
+ val seamFinderTypeSnapshot = batchSeamFinderType
769
+ val useInscribedRectCropSnapshot = batchUseInscribedRectCrop
770
+ // 2026-05-14 — resolve stitch-mode auto → concrete mode.
771
+ // 'auto' uses the pose deltas accumulated during capture to
772
+ // pick PANORAMA (rotation-heavy) vs SCANS (translation-heavy).
773
+ //
774
+ // Heuristic (matches design doc 2026-05-13-stitch-pipeline-mode-selection):
775
+ // translation_score = ||t_last − t_first|| (meters) / 0.10
776
+ // rotation_score = angle(fwd_last, fwd_first) (radians) / 1.00
777
+ // ratio = translation_score / (translation_score + rotation_score)
778
+ // ratio ≥ 0.55 → SCANS (translation-dominant)
779
+ // ratio < 0.55 → PANORAMA (rotation-dominant)
780
+ // missing poses → SCANS (safer default: bounded canvas).
781
+ //
782
+ // Why biased toward SCANS: PANORAMA on translation diverges
783
+ // catastrophically (3.2 GB compositing canvas observed
784
+ // 2026-05-14 → lmkd kill). SCANS on rotation degrades
785
+ // gracefully (slightly worse seams, never blows up).
786
+ val firstPose = batchFirstAcceptedPose
787
+ val lastPose = batchLastAcceptedPose
788
+ val stitchModeResolved: String = when (batchStitchMode) {
789
+ "panorama" -> "panorama"
790
+ "scans" -> "scans"
791
+ else -> resolveStitchModeAuto(firstPose, lastPose)
792
+ }
793
+ android.util.Log.i(
794
+ "IncrementalStitcher",
795
+ "finalize stitch-mode: configured=$batchStitchMode resolved=$stitchModeResolved " +
796
+ "firstPose=${firstPose != null} lastPose=${lastPose != null}",
797
+ )
798
+ batchKeyframeMode = false
799
+ batchKeyframePaths.clear()
800
+ batchKeyframeFrameCounter = 0
801
+ batchFirstAcceptedPose = null
802
+ batchLastAcceptedPose = null
803
+
804
+ // Null the bridge refs synchronously NOW so any worker that's
805
+ // about to run sees them as gone (V12.1 pattern). We keep
806
+ // local refs to do the actual finalize.
807
+ engine = null
808
+ firstwinsEngine = null
809
+
810
+ workScope.launch {
811
+ try {
812
+ val map = Arguments.createMap()
813
+ if (wasBatchKeyframe) {
814
+ // V16 batch-keyframe: hand keyframe paths to the
815
+ // JNI shim for one-shot cv::Stitcher processing.
816
+ if (keyframePathsSnapshot.size < 2) {
817
+ throw IllegalStateException(
818
+ "Batch-keyframe finalize: only " +
819
+ "${keyframePathsSnapshot.size} keyframe(s) " +
820
+ "captured — at least 2 required."
821
+ )
822
+ }
823
+ // Use the static `bridgeInstance` accessor on
824
+ // BatchStitcher rather than
825
+ // reactContext.getNativeModule — the latter
826
+ // returns null under bridgeless / new-architecture
827
+ // mode even for legacy-registered modules.
828
+ // Empirically: getNativeModule failed on Galaxy
829
+ // A35 with `BatchStitcher module not
830
+ // registered`, despite the module being present
831
+ // in RNImageStitcherPackage.createNativeModules.
832
+ // Same pattern that already works for
833
+ // IncrementalStitcher.bridgeInstance.
834
+ val stitcher = BatchStitcher.bridgeInstance
835
+ ?: throw IllegalStateException(
836
+ "BatchStitcher.bridgeInstance is null " +
837
+ "— module hasn't been instantiated yet. " +
838
+ "Check RNImageStitcherPackage registration."
839
+ )
840
+ val dims = stitcher.stitchSync(
841
+ keyframePathsSnapshot.toTypedArray(),
842
+ outputPath,
843
+ quality,
844
+ warperTypeSnapshot,
845
+ blenderTypeSnapshot,
846
+ seamFinderTypeSnapshot,
847
+ captureOrientationSnapshot,
848
+ useInscribedRectCropSnapshot,
849
+ stitchMode = stitchModeResolved,
850
+ )
851
+ // 2026-05-15 (D) — dims layout from native JNI:
852
+ // [0] width, [1] height, [2] framesRequested,
853
+ // [3] framesIncluded, [4] finalThresholdMilli
854
+ // The framesIncluded count is the post-
855
+ // leaveBiggestComponent retained subset. Any
856
+ // delta from acceptedCount = frames the stitcher
857
+ // dropped due to weak feature-matching confidence.
858
+ // Surfaced to JS so the capture-screen UX can
859
+ // show "Stitched N of M frames" when drops > 0.
860
+ val framesRequested =
861
+ if (dims.size > 2) dims[2] else keyframePathsSnapshot.size
862
+ val framesIncluded =
863
+ if (dims.size > 3) dims[3] else keyframePathsSnapshot.size
864
+ val finalConfidenceThresh =
865
+ if (dims.size > 4) dims[4].toDouble() / 1000.0 else -1.0
866
+ map.putString("panoramaPath", outputPath)
867
+ map.putInt("width", dims[0])
868
+ map.putInt("height", dims[1])
869
+ map.putInt("acceptedCount", keyframePathsSnapshot.size)
870
+ map.putInt("framesRequested", framesRequested)
871
+ map.putInt("framesIncluded", framesIncluded)
872
+ map.putInt("framesDropped", framesRequested - framesIncluded)
873
+ map.putDouble("finalConfidenceThresh", finalConfidenceThresh)
874
+ } else if (firstwins != null) {
875
+ val snap = firstwins.finalize(outputPath, quality)
876
+ ?: throw IllegalStateException("firstwins.finalize returned null")
877
+ map.putString("panoramaPath", snap.panoramaPath)
878
+ map.putInt("width", snap.width)
879
+ map.putInt("height", snap.height)
880
+ map.putInt("acceptedCount", snap.acceptedCount)
881
+ // Critic #22 fix: explicit native-buffer release.
882
+ firstwins.release()
883
+ } else {
884
+ val snap = hybrid!!.finalize(outputPath, quality)
885
+ map.putString("panoramaPath", snap.panoramaPath)
886
+ map.putInt("width", snap.width)
887
+ map.putInt("height", snap.height)
888
+ map.putInt("acceptedCount", snap.acceptedCount)
889
+ hybrid.release()
890
+ // 2026-05-16 — realtime+batch fusion (Option A
891
+ // "Replace on completion") hook. The live
892
+ // panorama has been written to outputPath; now
893
+ // fire-and-forget an async refinement on the
894
+ // engine's accepted keyframes via the shared C++
895
+ // stitcher.
896
+ //
897
+ // Today's `IncrementalEngine` (the hybrid live
898
+ // engine) does NOT retain per-frame JPEGs — it
899
+ // paints into a single persistent canvas Mat
900
+ // that's torn down by `release()` above. So the
901
+ // keyframe-paths list passed to runHybridAutoRefine
902
+ // is empty for the hybrid branch, which means
903
+ // the auto-trigger detects "< 2 keyframes on
904
+ // disk" and emits `isRefining=false` without
905
+ // running cv::Stitcher. Per the prompt's
906
+ // "no-op when no keyframes on disk" constraint.
907
+ //
908
+ // When a future change wires the hybrid engine
909
+ // to a keyframe collector (parallel to iOS'
910
+ // OpenCVKeyframeCollector), the same hook will
911
+ // light up automatically — just pass the
912
+ // populated list here.
913
+ val keyframePathsForHybrid: List<String> = emptyList()
914
+ val refinedOutputPath = refinedPathFromLive(outputPath)
915
+ runHybridAutoRefine(
916
+ framePaths = keyframePathsForHybrid,
917
+ refinedOutputPath = refinedOutputPath,
918
+ captureOrientation = captureOrientationSnapshot,
919
+ warperType = warperTypeSnapshot,
920
+ blenderType = blenderTypeSnapshot,
921
+ seamFinderType = seamFinderTypeSnapshot,
922
+ useInscribedRectCrop = useInscribedRectCropSnapshot,
923
+ )
924
+ }
925
+ map.putInt("droppedBackpressure", 0)
926
+ promise.resolve(map)
927
+ } catch (t: Throwable) {
928
+ firstwins?.release()
929
+ hybrid?.release()
930
+ promise.reject("incremental-finalize-failed", t.message, t)
931
+ }
932
+ }
933
+ }
934
+
935
+ @ReactMethod
936
+ fun cancel(promise: Promise) {
937
+ // Critic #4 fix: synchronously flip isRunning + null engine
938
+ // refs BEFORE releasing. Any in-flight worker bails at the
939
+ // re-check before touching the now-null engine. Matches
940
+ // iOS V12.1 cancel path.
941
+ arCameraViewRef?.setIncrementalIngestionActive(false)
942
+ isRunning.set(false)
943
+ val hybrid = engine
944
+ val firstwins = firstwinsEngine
945
+ engine = null
946
+ firstwinsEngine = null
947
+ // V16 Phase 2 (Android Fix-1) — clean up the per-session
948
+ // batch-keyframe subdir. iOS-parity: cancel removes the
949
+ // session's saved JPEGs because the operator explicitly
950
+ // aborted, so the keyframes aren't worth preserving for
951
+ // reprocessing. (Successful finalize keeps them — see the
952
+ // ivar declaration.)
953
+ val sessionDirToCleanup = captureSessionDir
954
+ captureSessionDir = null
955
+ batchKeyframeMode = false
956
+ batchKeyframePaths.clear()
957
+ batchKeyframeFrameCounter = 0
958
+ // Defer engine release + session-dir cleanup onto the work
959
+ // queue so we don't race with an ingest that already passed
960
+ // the null-check and is mid-execution on a captured local
961
+ // reference.
962
+ workScope.launch {
963
+ hybrid?.release()
964
+ firstwins?.reset()
965
+ sessionDirToCleanup?.deleteRecursively()
966
+ }
967
+ val map = Arguments.createMap()
968
+ map.putBoolean("ok", true)
969
+ promise.resolve(map)
970
+ }
971
+
972
+ /**
973
+ * Called by `RNSARCameraView` per ARCore frame when it has
974
+ * a fresh JPEG + pose to ingest. Synchronous-feeling from the
975
+ * caller's perspective but actually dispatched onto the engine's
976
+ * own queue so we don't stall the GL render thread. Drops the
977
+ * frame silently if no engine is running (race between view
978
+ * lifecycle and stitcher start/stop).
979
+ */
980
+ internal fun ingestFromARCameraView(
981
+ path: String,
982
+ tx: Double, ty: Double, tz: Double,
983
+ qx: Double, qy: Double, qz: Double, qw: Double,
984
+ fx: Double, fy: Double, cx: Double, cy: Double,
985
+ imageWidth: Int, imageHeight: Int,
986
+ yaw: Double,
987
+ pitch: Double,
988
+ fovHorizDegrees: Double,
989
+ fovVertDegrees: Double,
990
+ trackingPoor: Boolean,
991
+ ) {
992
+ // ── V16 batch-keyframe: AR-driven path ─────────────────────
993
+ //
994
+ // Batch-keyframe mode runs WITHOUT a live engine (engine ==
995
+ // firstwinsEngine == null) — frames accumulate as keyframe
996
+ // paths and the cv::Stitcher pipeline runs at finalize time.
997
+ //
998
+ // P3-F: this branch now calls into the shared-C++
999
+ // KeyframeGate (cpp/keyframe_gate.{hpp,cpp}, same algorithm
1000
+ // iOS uses). The placeholder frame-counter gate that lived
1001
+ // here previously (handleBatchKeyframeFrame) is GONE.
1002
+ //
1003
+ // The ARCameraView's JPEG-encode pipeline writes to a single
1004
+ // REUSED tmp file (rlis-arframe.jpg in cacheDir) — fine for
1005
+ // live engines (decoded into cv::Mat synchronously inside
1006
+ // addFrameAtPath, before next frame arrives), but FATAL for
1007
+ // batch-keyframe (all accepted keyframe paths would point to
1008
+ // the same overwritten file → finalize stitches 6 copies of
1009
+ // the same frame). So we must COPY to a unique path on
1010
+ // accept. We do the gate-evaluate BEFORE the copy so we
1011
+ // skip the ~5 ms JPEG copy on rejected frames.
1012
+ if (batchKeyframeMode) {
1013
+ // Build the POD pose for the gate. tx/ty/tz are passed
1014
+ // through from the AR camera view (camera.pose.tx() etc.);
1015
+ // they're required for the plane-overlap math. Falling
1016
+ // back to the angular path when no plane is latched is
1017
+ // handled internally by the gate (latchedPlane=null arg).
1018
+ val pose = RNSARFramePose(
1019
+ tx = tx, ty = ty, tz = tz,
1020
+ qx = qx, qy = qy, qz = qz, qw = qw,
1021
+ fx = fx, fy = fy, cx = cx, cy = cy,
1022
+ imageWidth = imageWidth, imageHeight = imageHeight,
1023
+ timestampMs = 0.0, // not used by the gate
1024
+ trackingState = RNSARSession.TRACKING_TRACKING,
1025
+ )
1026
+ // Fetch the latched plane (if any) from the AR session
1027
+ // and convert to a column-major 16-float matrix matching
1028
+ // the C++ PlaneTransform layout. ARCore's
1029
+ // Pose.toMatrix(out, offset) gives us exactly that layout
1030
+ // (same as iOS simd_float4x4).
1031
+ val planeMatrix: FloatArray? =
1032
+ RNSARSession.instance?.latchedPlaneTransform?.let { p ->
1033
+ FloatArray(16).also { p.toMatrix(it, 0) }
1034
+ }
1035
+
1036
+ val decision = keyframeGate.evaluate(pose, planeMatrix)
1037
+
1038
+ // ── P3-G diagnostic ──────────────────────────────────
1039
+ // Rate-limit at the same cadence as the plane evaluator
1040
+ // (every 30 frames ≈ 2 Hz at 60Hz frame rate) but ALWAYS
1041
+ // log accepts (rare, important signal).
1042
+ if (decision.accept || (frameIngestLogTick++ % 30 == 0)) {
1043
+ android.util.Log.i(
1044
+ "IncrementalStitcher",
1045
+ "ingestFromARCameraView batch: " +
1046
+ "accept=${decision.accept} reason=${decision.reason} " +
1047
+ "newContent=${"%.3f".format(decision.newContentFraction)} " +
1048
+ "gateCount=${decision.acceptedCount} " +
1049
+ "paths.size=${batchKeyframePaths.size} " +
1050
+ "planeAvailable=${planeMatrix != null}",
1051
+ )
1052
+ }
1053
+ if (!decision.accept) {
1054
+ // Frame rejected by the gate — could be overlap-too-
1055
+ // high (most common), max-reached, or projection-
1056
+ // degenerate. Drop silently; the next frame will be
1057
+ // evaluated. TODO(P1-followup): emit a state event
1058
+ // so the JS UI can surface the reason in the pill.
1059
+ return
1060
+ }
1061
+ // Accepted — copy the (reused) tmp JPEG to a persistent
1062
+ // per-keyframe path so subsequent frames overwriting the
1063
+ // source don't clobber it.
1064
+ val persistentPath = copyKeyframeToStore(path)
1065
+ if (persistentPath == null) {
1066
+ android.util.Log.w(
1067
+ "IncrementalStitcher",
1068
+ "ingestFromARCameraView batch: ACCEPTED but copy FAILED — frame dropped",
1069
+ )
1070
+ // Copy failed — drop the frame. Logged inside
1071
+ // copyKeyframeToStore. Counter was already incremented
1072
+ // inside the gate; that's fine — the next acceptable
1073
+ // frame will still be accepted on its own merits.
1074
+ return
1075
+ }
1076
+ batchKeyframePaths.add(persistentPath)
1077
+ // 2026-05-14 — capture pose at every accept for the
1078
+ // stitch-mode auto-decision at finalize(). First accept
1079
+ // anchors the "from" pose; every subsequent accept
1080
+ // updates "to". Cleared at start(). Order: tx, ty, tz,
1081
+ // qx, qy, qz, qw (same as the Pose struct in C++ KeyframeGate).
1082
+ val poseSnapshot = doubleArrayOf(tx, ty, tz, qx, qy, qz, qw)
1083
+ if (batchFirstAcceptedPose == null) batchFirstAcceptedPose = poseSnapshot
1084
+ batchLastAcceptedPose = poseSnapshot
1085
+ android.util.Log.i(
1086
+ "IncrementalStitcher",
1087
+ "ingestFromARCameraView batch: ACCEPTED keyframe #${batchKeyframePaths.size}" +
1088
+ " → $persistentPath",
1089
+ )
1090
+ // Emit a state event so the JS-side LiveFrameStrip
1091
+ // renders the thumbnail strip + the "Keyframes: n/max"
1092
+ // pill updates in real time. iOS counterpart:
1093
+ // emitBatchKeyframeAcceptedState in
1094
+ // IncrementalStitcher.swift — same field set
1095
+ // so the JS subscriber doesn't branch on platform.
1096
+ emitBatchKeyframeAcceptedState(
1097
+ thumbnailPath = persistentPath,
1098
+ keyframeIndex = batchKeyframePaths.size - 1,
1099
+ keyframeCount = batchKeyframePaths.size,
1100
+ keyframeMax = keyframeGate.maxCount,
1101
+ isLandscape = imageWidth >= imageHeight,
1102
+ )
1103
+ return
1104
+ }
1105
+
1106
+ val hybrid = this.engine
1107
+ val firstwins = this.firstwinsEngine
1108
+ if (hybrid == null && firstwins == null) return
1109
+ workScope.launch {
1110
+ val state: WritableMap? = if (firstwins != null) {
1111
+ val tele = firstwins.addFrameAtPath(
1112
+ path = path,
1113
+ qx = qx, qy = qy, qz = qz, qw = qw,
1114
+ fx = fx, fy = fy, cx = cx, cy = cy,
1115
+ imageWidth = imageWidth, imageHeight = imageHeight,
1116
+ yaw = yaw, pitch = pitch,
1117
+ fovHorizDegrees = fovHorizDegrees,
1118
+ fovVertDegrees = fovVertDegrees,
1119
+ trackingPoor = trackingPoor,
1120
+ )
1121
+ firstwins.snapshotIfDue(tele)
1122
+ } else {
1123
+ val tele = hybrid!!.addFrameAtPath(
1124
+ path = path,
1125
+ qx = qx, qy = qy, qz = qz, qw = qw,
1126
+ fx = fx, fy = fy, cx = cx, cy = cy,
1127
+ imageWidth = imageWidth, imageHeight = imageHeight,
1128
+ yaw = yaw, pitch = pitch,
1129
+ fovHorizDegrees = fovHorizDegrees,
1130
+ fovVertDegrees = fovVertDegrees,
1131
+ trackingPoor = trackingPoor,
1132
+ )
1133
+ hybrid.snapshotIfDue(tele)
1134
+ }
1135
+ emitState(state)
1136
+ }
1137
+ }
1138
+
1139
+ @ReactMethod
1140
+ fun getState(promise: Promise) {
1141
+ val state = firstwinsEngine?.lastState ?: engine?.lastState
1142
+ if (state == null) {
1143
+ promise.resolve(null)
1144
+ return
1145
+ }
1146
+ promise.resolve(state)
1147
+ }
1148
+
1149
+ // ── V15.0e — AR plane detection bridge (iOS-parity) ──────────────
1150
+ //
1151
+ // iOS exposes these on the IncrementalStitcherBridge (NOT on the
1152
+ // ARSession module) so the JS code calls
1153
+ // getIncrementalNativeModule().getARPlaneStatus()
1154
+ // (see retailens-capture-sdk/src/stitching/incremental.ts:535).
1155
+ // Both methods delegate to the AR session singleton — same pattern
1156
+ // as iOS' IncrementalStitcherBridge.swift, where the bridge holds
1157
+ // the RN @objc surface and the singleton holds the AR algorithm.
1158
+
1159
+ /**
1160
+ * Poll-friendly plane-status read. Called by JS at 2 Hz while
1161
+ * planeSource = 'ARKitDetected' (the default). When the AR session
1162
+ * native module isn't registered (e.g. plain stitching tests
1163
+ * without an active AR session), returns a stable "searching"
1164
+ * default so the JS gate never throws.
1165
+ */
1166
+ @ReactMethod
1167
+ fun getARPlaneStatus(promise: Promise) {
1168
+ val session = RNSARSession.instance
1169
+ if (session == null) {
1170
+ // Safe default: no AR session = no plane to lock onto.
1171
+ // Shape MUST match the iOS contract so JS doesn't branch.
1172
+ val map = Arguments.createMap()
1173
+ map.putString("status", "searching")
1174
+ map.putBoolean("hasPlane", false)
1175
+ map.putDouble("bestAlignment", -1.0)
1176
+ map.putDouble("threshold", 0.6)
1177
+ promise.resolve(map)
1178
+ return
1179
+ }
1180
+ promise.resolve(session.buildARPlaneStatusMap())
1181
+ }
1182
+
1183
+ /**
1184
+ * Force re-evaluation of plane detection. Used by the JS
1185
+ * hold-to-scan press handler in AuditCaptureScreen.tsx:529 (which
1186
+ * `.catch(()=>{})`s the result). Returns `latched=false`
1187
+ * synchronously; JS sees the new state on the next 2 Hz
1188
+ * getARPlaneStatus poll (~16 ms later, when the GL render thread
1189
+ * runs evaluatePlanesForFrame on the next ARCore frame). See
1190
+ * detailed semantic note in RNSARSession.buildARPlaneStatusMap.
1191
+ */
1192
+ @ReactMethod
1193
+ fun relatchARPlane(promise: Promise) {
1194
+ RNSARSession.instance?.clearPlaneLatch()
1195
+ val map = Arguments.createMap()
1196
+ map.putBoolean("latched", false)
1197
+ promise.resolve(map)
1198
+ }
1199
+
1200
+ /**
1201
+ * iOS-parity bridge method (was missing from Android — flagged
1202
+ * in the parity audit as Section C gap #1 / Section F gap #2).
1203
+ *
1204
+ * Arms the KeyframeGate to force-accept the NEXT frame regardless
1205
+ * of overlap. Used by the JS shutter-release path so we don't
1206
+ * truncate the trailing edge of the scan. iOS counterpart:
1207
+ * IncrementalStitcherBridge.swift markNextFrameAsLastKeyframe.
1208
+ *
1209
+ * Always resolves with `{ ok: true }`. No-op when the gate is
1210
+ * disabled (which is fine — the live engines don't need a
1211
+ * force-last; only batch-keyframe does).
1212
+ */
1213
+ @ReactMethod
1214
+ fun markNextFrameAsLastKeyframe(promise: Promise) {
1215
+ keyframeGate.forceAcceptNext = true
1216
+ val map = Arguments.createMap()
1217
+ map.putBoolean("ok", true)
1218
+ promise.resolve(map)
1219
+ }
1220
+
1221
+ /**
1222
+ * 2026-05-18 (Iss 3) — GC stale keyframe-session directories under
1223
+ * the SDK's cacheDir. Scans `cacheDir` for `rlis-capture-*`
1224
+ * subdirectories (created by start() above) and removes those whose
1225
+ * newest file mtime is older than `olderThanMs` (default 24h).
1226
+ *
1227
+ * iOS sibling: `IncrementalStitcher.swift::cleanupKeyframes`.
1228
+ *
1229
+ * Resolves with `{ sessionsDeleted, bytesFreed }`. Never rejects —
1230
+ * filesystem failures (missing dir, permission errors) resolve with
1231
+ * zero counts so the host can call this unconditionally on launch.
1232
+ *
1233
+ * Note: Android's OS already evicts cacheDir entries under storage
1234
+ * pressure, so this is a "be a good citizen and free space sooner"
1235
+ * helper rather than a hard requirement. Still useful so the user's
1236
+ * disk-usage report doesn't show 100s of MB of stale captures.
1237
+ */
1238
+ @ReactMethod
1239
+ fun cleanupKeyframes(options: ReadableMap?, promise: Promise) {
1240
+ val olderThanMs = options?.getDoubleOrDefault(
1241
+ "olderThanMs", 24.0 * 3600.0 * 1000.0,
1242
+ ) ?: (24.0 * 3600.0 * 1000.0)
1243
+ val cutoffMs = System.currentTimeMillis() - olderThanMs.toLong()
1244
+ var sessionsDeleted = 0
1245
+ var bytesFreed = 0L
1246
+ try {
1247
+ val cache = reactContext.cacheDir ?: throw IllegalStateException("no cacheDir")
1248
+ val sessions = cache.listFiles { f -> f.isDirectory && f.name.startsWith("rlis-capture-") }
1249
+ ?: emptyArray()
1250
+ for (sessionDir in sessions) {
1251
+ // Newest mtime across the session's files (flat tree today,
1252
+ // walked recursively for future-proofing).
1253
+ var newestMtime = 0L
1254
+ var bytes = 0L
1255
+ sessionDir.walkTopDown().forEach { f ->
1256
+ if (f.isFile) {
1257
+ if (f.lastModified() > newestMtime) newestMtime = f.lastModified()
1258
+ bytes += f.length()
1259
+ }
1260
+ }
1261
+ if (newestMtime == 0L) {
1262
+ // Empty session — fall back to the dir's own mtime.
1263
+ newestMtime = sessionDir.lastModified()
1264
+ }
1265
+ if (newestMtime in 1 until cutoffMs) {
1266
+ if (sessionDir.deleteRecursively()) {
1267
+ sessionsDeleted += 1
1268
+ bytesFreed += bytes
1269
+ }
1270
+ }
1271
+ }
1272
+ } catch (e: Exception) {
1273
+ android.util.Log.w(
1274
+ "IncrementalStitcher",
1275
+ "cleanupKeyframes: ${e.message}",
1276
+ )
1277
+ }
1278
+ val map = Arguments.createMap()
1279
+ map.putInt("sessionsDeleted", sessionsDeleted)
1280
+ map.putDouble("bytesFreed", bytesFreed.toDouble())
1281
+ promise.resolve(map)
1282
+ }
1283
+
1284
+ /**
1285
+ * 2026-05-18 (Iss 3) — return the current capture's keyframe
1286
+ * session directory. Empty string when no capture is in flight
1287
+ * (or not in batch-keyframe mode).
1288
+ *
1289
+ * iOS sibling: `IncrementalStitcher.swift::currentKeyframeDir`.
1290
+ */
1291
+ @ReactMethod
1292
+ fun getKeyframeDir(promise: Promise) {
1293
+ val path = if (batchKeyframeMode) {
1294
+ captureSessionDir?.absolutePath ?: ""
1295
+ } else {
1296
+ ""
1297
+ }
1298
+ val map = Arguments.createMap()
1299
+ map.putString("path", path)
1300
+ promise.resolve(map)
1301
+ }
1302
+
1303
+ /**
1304
+ * 2026-05-16 — realtime+batch fusion (Option A "Replace on
1305
+ * completion") entry point. Run the shared C++ `cv::Stitcher`
1306
+ * pipeline over a caller-supplied list of keyframe JPEGs and
1307
+ * write a refined panorama to `outputPath`.
1308
+ *
1309
+ * Pre-conditions:
1310
+ * - `framePaths.length >= 2`
1311
+ * - Each path must exist on disk
1312
+ *
1313
+ * Routing: delegates to `BatchStitcher.stitchSync(...)` —
1314
+ * the same shared-JNI shim the batch-keyframe finalize uses.
1315
+ * Quality defaults match the batch-keyframe finalize:
1316
+ * warperType = "spherical"
1317
+ * blenderType = "multiband"
1318
+ * seamFinderType = "graphcut"
1319
+ * captureOrientation = "portrait"
1320
+ * useInscribedRectCrop = false
1321
+ * stitchMode = "auto"
1322
+ * jpegQuality = 90
1323
+ *
1324
+ * Threading: dispatches onto `refineScope` so the JS promise
1325
+ * doesn't block the @ReactMethod thread for the 2-5 s the
1326
+ * stitcher takes. iOS-parity behaviour.
1327
+ *
1328
+ * iOS sibling: `IncrementalStitcher.swift::refinePanorama`.
1329
+ *
1330
+ * See: docs/site-content/design/2026-05-14-realtime-batch-fusion.md
1331
+ */
1332
+ @ReactMethod
1333
+ fun refinePanorama(options: ReadableMap, promise: Promise) {
1334
+ val framePathsArr = options.getArray("framePaths")
1335
+ if (framePathsArr == null || framePathsArr.size() < 2) {
1336
+ promise.reject(
1337
+ "incremental-refine-invalid-input",
1338
+ "refinePanorama requires at least 2 framePaths (got " +
1339
+ "${framePathsArr?.size() ?: 0}).",
1340
+ )
1341
+ return
1342
+ }
1343
+ val framePaths = Array(framePathsArr.size()) {
1344
+ stripFileScheme(framePathsArr.getString(it) ?: "")
1345
+ }
1346
+ val outputPathOpt = options.getString("outputPath")
1347
+ if (outputPathOpt.isNullOrEmpty()) {
1348
+ promise.reject(
1349
+ "incremental-refine-invalid-input",
1350
+ "refinePanorama requires a non-empty outputPath.",
1351
+ )
1352
+ return
1353
+ }
1354
+ val outputPath = stripFileScheme(outputPathOpt)
1355
+ val config: ReadableMap? =
1356
+ if (options.hasKey("config")) options.getMap("config") else null
1357
+ val warperType = config?.getString("warperType") ?: "spherical"
1358
+ val blenderType = config?.getString("blenderType") ?: "multiband"
1359
+ val seamFinderType = config?.getString("seamFinderType") ?: "graphcut"
1360
+ val captureOrientation = config?.getString("captureOrientation") ?: "portrait"
1361
+ val useInscribedRectCrop =
1362
+ config?.getBooleanOrDefault("useInscribedRectCrop", false) ?: false
1363
+ val stitchMode = (config?.getString("stitchMode") ?: "auto")
1364
+ .let { if (it in setOf("auto", "panorama", "scans")) it else "auto" }
1365
+ val jpegQuality = max(1, min(100,
1366
+ config?.getIntOrDefault("jpegQuality", 90) ?: 90))
1367
+
1368
+ // Pre-flight existence check — same defensive layer iOS has.
1369
+ for (p in framePaths) {
1370
+ if (!File(p).exists()) {
1371
+ promise.reject(
1372
+ "incremental-refine-missing-keyframe",
1373
+ "refinePanorama: keyframe missing on disk — $p",
1374
+ )
1375
+ return
1376
+ }
1377
+ }
1378
+
1379
+ refineScope.launch {
1380
+ try {
1381
+ val stitcher = BatchStitcher.bridgeInstance
1382
+ ?: throw IllegalStateException(
1383
+ "BatchStitcher.bridgeInstance is null — " +
1384
+ "module hasn't been instantiated yet.",
1385
+ )
1386
+ // "auto" mode is meaningful only when we have first/
1387
+ // last keyframe poses to consult; the explicit
1388
+ // refinePanorama entry point has no pose context, so
1389
+ // collapse 'auto' → 'scans' here (the safer fallback,
1390
+ // identical to resolveStitchModeAuto's null-pose
1391
+ // branch). Concrete modes pass through unchanged.
1392
+ val effectiveMode = if (stitchMode == "auto") "scans" else stitchMode
1393
+ val dims = stitcher.stitchSync(
1394
+ framePaths,
1395
+ outputPath,
1396
+ jpegQuality,
1397
+ warperType,
1398
+ blenderType,
1399
+ seamFinderType,
1400
+ captureOrientation,
1401
+ useInscribedRectCrop,
1402
+ stitchMode = effectiveMode,
1403
+ )
1404
+ val framesRequested =
1405
+ if (dims.size > 2) dims[2] else framePaths.size
1406
+ val framesIncluded =
1407
+ if (dims.size > 3) dims[3] else framePaths.size
1408
+ val finalConfidenceThresh =
1409
+ if (dims.size > 4) dims[4].toDouble() / 1000.0 else -1.0
1410
+ val map = Arguments.createMap().apply {
1411
+ putString("panoramaPath", outputPath)
1412
+ putInt("width", dims[0])
1413
+ putInt("height", dims[1])
1414
+ putInt("framesRequested", framesRequested)
1415
+ putInt("framesIncluded", framesIncluded)
1416
+ putInt("framesDropped", framesRequested - framesIncluded)
1417
+ putDouble("finalConfidenceThresh", finalConfidenceThresh)
1418
+ }
1419
+ promise.resolve(map)
1420
+ } catch (t: Throwable) {
1421
+ promise.reject("incremental-refine-failed", t.message, t)
1422
+ }
1423
+ }
1424
+ }
1425
+
1426
+ /**
1427
+ * 2026-05-16 — realtime+batch fusion auto-trigger called from
1428
+ * the hybrid-engine branch of `finalize()`. Fire-and-forget;
1429
+ * the finalize() promise has ALREADY resolved with the live
1430
+ * panorama before this is invoked.
1431
+ *
1432
+ * 1. Emits a state event with `isRefining = true` so the
1433
+ * host renders a "Refining…" pill.
1434
+ * 2. Runs `BatchStitcher.stitchSync(...)` on the supplied
1435
+ * keyframe paths.
1436
+ * 3. On success: emits a state event with `isRefining = false`
1437
+ * AND `refinedPanoramaPath = <path>`.
1438
+ * 4. On failure: emits a state event with `isRefining = false`
1439
+ * and no refined path. Host keeps showing the live
1440
+ * panorama; failure does not affect audit save.
1441
+ *
1442
+ * NO-OP when `framePaths.size < 2` or any path is missing on
1443
+ * disk — matches the design doc's "if keyframes are NOT on
1444
+ * disk, auto-trigger is a no-op" contract. Today's hybrid
1445
+ * engine retains no per-frame JPEGs so this is the
1446
+ * always-no-op path; the hook is wired in advance of a future
1447
+ * keyframe-collector enhancement.
1448
+ */
1449
+ internal fun runHybridAutoRefine(
1450
+ framePaths: List<String>,
1451
+ refinedOutputPath: String,
1452
+ captureOrientation: String,
1453
+ warperType: String,
1454
+ blenderType: String,
1455
+ seamFinderType: String,
1456
+ useInscribedRectCrop: Boolean,
1457
+ ) {
1458
+ if (framePaths.size < 2) {
1459
+ android.util.Log.i(
1460
+ "IncrementalStitcher",
1461
+ "[refine.auto] skipped: framePaths.size=${framePaths.size} " +
1462
+ "(hybrid engine retains no per-frame JPEGs)",
1463
+ )
1464
+ emitRefinementState(isRefining = false, refinedPanoramaPath = null)
1465
+ return
1466
+ }
1467
+ for (p in framePaths) {
1468
+ if (!File(p).exists()) {
1469
+ android.util.Log.i(
1470
+ "IncrementalStitcher",
1471
+ "[refine.auto] skipped: missing keyframe $p",
1472
+ )
1473
+ emitRefinementState(isRefining = false, refinedPanoramaPath = null)
1474
+ return
1475
+ }
1476
+ }
1477
+ emitRefinementState(isRefining = true, refinedPanoramaPath = null)
1478
+ refineScope.launch {
1479
+ try {
1480
+ val stitcher = BatchStitcher.bridgeInstance
1481
+ ?: throw IllegalStateException(
1482
+ "BatchStitcher.bridgeInstance is null at auto-refine time",
1483
+ )
1484
+ stitcher.stitchSync(
1485
+ framePaths.toTypedArray(),
1486
+ refinedOutputPath,
1487
+ 90,
1488
+ warperType,
1489
+ blenderType,
1490
+ seamFinderType,
1491
+ captureOrientation,
1492
+ useInscribedRectCrop,
1493
+ stitchMode = "scans",
1494
+ )
1495
+ android.util.Log.i(
1496
+ "IncrementalStitcher",
1497
+ "[refine.auto] success path=$refinedOutputPath",
1498
+ )
1499
+ emitRefinementState(
1500
+ isRefining = false,
1501
+ refinedPanoramaPath = refinedOutputPath,
1502
+ )
1503
+ } catch (t: Throwable) {
1504
+ android.util.Log.w(
1505
+ "IncrementalStitcher",
1506
+ "[refine.auto] refinement failed (live output kept): ${t.message}",
1507
+ )
1508
+ emitRefinementState(isRefining = false, refinedPanoramaPath = null)
1509
+ }
1510
+ }
1511
+ }
1512
+
1513
+ /**
1514
+ * 2026-05-16 — emit a refinement-related state event. Reuses
1515
+ * the same IncrementalStateUpdate channel the live
1516
+ * engines emit on; JS reads `isRefining` and `refinedPanoramaPath`
1517
+ * directly from the event payload (no schema change required on
1518
+ * the JS dispatch side).
1519
+ */
1520
+ private fun emitRefinementState(
1521
+ isRefining: Boolean,
1522
+ refinedPanoramaPath: String?,
1523
+ ) {
1524
+ val state = Arguments.createMap().apply {
1525
+ putNull("panoramaPath")
1526
+ putInt("width", 0)
1527
+ putInt("height", 0)
1528
+ putInt("acceptedCount", 0)
1529
+ putInt("outcome", 0) // AcceptedHigh
1530
+ putDouble("confidence", 1.0)
1531
+ putDouble("overlapPercent", -1.0)
1532
+ putInt("processingMs", 0)
1533
+ putBoolean("isLandscape", false)
1534
+ putInt("paintedExtent", 0)
1535
+ putInt("panExtent", 0)
1536
+ putInt("keyframeMax", 0)
1537
+ putBoolean("isRefining", isRefining)
1538
+ if (refinedPanoramaPath != null) {
1539
+ putString("refinedPanoramaPath", refinedPanoramaPath)
1540
+ }
1541
+ }
1542
+ emitState(state)
1543
+ }
1544
+
1545
+ /**
1546
+ * 2026-05-16 — given the live panorama path, derive a sibling
1547
+ * path for the refined output. Same algorithm iOS uses:
1548
+ * /…/<base>.jpg → /…/<base>-refined.jpg
1549
+ */
1550
+ private fun refinedPathFromLive(livePath: String): String {
1551
+ val cleaned = stripFileScheme(livePath)
1552
+ val file = File(cleaned)
1553
+ val parent = file.parentFile ?: File(reactContext.cacheDir, "panoramas")
1554
+ val name = file.name
1555
+ val dot = name.lastIndexOf('.')
1556
+ val refinedName = if (dot >= 0) {
1557
+ "${name.substring(0, dot)}-refined${name.substring(dot)}"
1558
+ } else {
1559
+ "$name-refined"
1560
+ }
1561
+ return File(parent, refinedName).absolutePath
1562
+ }
1563
+
1564
+ /**
1565
+ * Poll the process' memory footprint in MB. Android parity for
1566
+ * iOS' `getMemoryFootprintMB` (which polls Mach `phys_footprint`
1567
+ * via `task_info(TASK_VM_INFO)` — see
1568
+ * `IncrementalStitcherBridge.swift:231-259`).
1569
+ *
1570
+ * Returns the **total PSS** (proportional set size) of this
1571
+ * process in MB. PSS is the metric Android's Low-Memory-Killer
1572
+ * (`lmkd`) ranks against, so it's the right one-true-number for
1573
+ * the on-screen memory pill: it's "how close are we to being
1574
+ * killed by the system?".
1575
+ *
1576
+ * Total PSS = USS (private) + sum(shared / refcount). Read via
1577
+ * `ActivityManager.getProcessMemoryInfo()`, which is the same API
1578
+ * Android Studio's profiler uses. Granularity is 1 KB; we
1579
+ * divide by 1024 to MB so the JS side displays a number directly
1580
+ * comparable to the iOS phys_footprint value.
1581
+ *
1582
+ * Returns -1.0 on failure (very rare — `getProcessMemoryInfo()`
1583
+ * is generally infallible since Android 5.0 because PSS is read
1584
+ * from `/proc/self/smaps` synchronously on the calling thread).
1585
+ */
1586
+ @ReactMethod
1587
+ fun getMemoryFootprintMB(promise: Promise) {
1588
+ try {
1589
+ val am = reactContext.getSystemService(ActivityManager::class.java)
1590
+ if (am == null) {
1591
+ promise.resolve(-1.0)
1592
+ return
1593
+ }
1594
+ val pid = android.os.Process.myPid()
1595
+ val infos = am.getProcessMemoryInfo(intArrayOf(pid))
1596
+ if (infos == null || infos.isEmpty()) {
1597
+ promise.resolve(-1.0)
1598
+ return
1599
+ }
1600
+ // totalPss is in KB. Divide by 1024 → MB. Use Double so
1601
+ // the JS overlay can render fractional MB if it wants.
1602
+ val mb = infos[0].totalPss.toDouble() / 1024.0
1603
+ promise.resolve(mb)
1604
+ } catch (t: Throwable) {
1605
+ android.util.Log.w(
1606
+ "IncrementalStitcher",
1607
+ "getMemoryFootprintMB: failed: ${t.message}",
1608
+ )
1609
+ promise.resolve(-1.0)
1610
+ }
1611
+ }
1612
+
1613
+ /**
1614
+ * Release the C++ KeyframeGate heap allocation when RN tears
1615
+ * down the bridge module (e.g. on a JS reload). Without this,
1616
+ * each reload leaks ~100 bytes of native heap — small but
1617
+ * unbounded over a long dev session.
1618
+ */
1619
+ override fun onCatalystInstanceDestroy() {
1620
+ try {
1621
+ keyframeGate.close()
1622
+ } catch (t: Throwable) {
1623
+ android.util.Log.w(
1624
+ "IncrementalStitcher",
1625
+ "onCatalystInstanceDestroy: keyframeGate.close failed: ${t.message}",
1626
+ )
1627
+ }
1628
+ // V16 Phase 2 (Android Fix-1) — best-effort cleanup of the
1629
+ // current per-session subdir. Prevents leftover dirs
1630
+ // accumulating across dev-time RN reloads. OS cache cleanup
1631
+ // would eventually reclaim cacheDir entries anyway, but this
1632
+ // makes the dev loop tidy.
1633
+ try {
1634
+ captureSessionDir?.deleteRecursively()
1635
+ } catch (t: Throwable) {
1636
+ // Ignore — not critical at teardown.
1637
+ }
1638
+ captureSessionDir = null
1639
+ super.onCatalystInstanceDestroy()
1640
+ }
1641
+
1642
+ private fun emitState(state: WritableMap?) {
1643
+ if (state == null) return
1644
+ // Re-emit to JS via the standard DeviceEventEmitter pattern.
1645
+ // RN drops events when no listener is attached, so we don't
1646
+ // need our own gating.
1647
+ reactContext
1648
+ .getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
1649
+ .emit("IncrementalStateUpdate", state)
1650
+ }
1651
+
1652
+ /**
1653
+ * Emit a state event when a batch-keyframe is accepted. Carries
1654
+ * the on-disk thumbnail path so JS can render it in the
1655
+ * LiveFrameStrip + advance the "Keyframes: N/M" pill.
1656
+ *
1657
+ * iOS-parity field set — mirrors
1658
+ * IncrementalStitcher.swift::emitBatchKeyframeAcceptedState
1659
+ * exactly (same field names, types, order) so the JS subscriber
1660
+ * in incremental.ts doesn't need to branch on platform.
1661
+ */
1662
+ private fun emitBatchKeyframeAcceptedState(
1663
+ thumbnailPath: String,
1664
+ keyframeIndex: Int,
1665
+ keyframeCount: Int,
1666
+ keyframeMax: Int,
1667
+ isLandscape: Boolean,
1668
+ ) {
1669
+ val state = Arguments.createMap()
1670
+ state.putNull("panoramaPath")
1671
+ state.putInt("width", 0)
1672
+ state.putInt("height", 0)
1673
+ state.putInt("acceptedCount", keyframeCount)
1674
+ // Outcome 0 = AcceptedHigh — matches the FrameOutcome enum
1675
+ // ordinal that the live engines emit. Keeps the iOS
1676
+ // IncrementalOutcome contract: batch-keyframe
1677
+ // accepts all carry outcome=acceptedHigh.
1678
+ state.putInt("outcome", 0)
1679
+ state.putDouble("confidence", 1.0)
1680
+ state.putDouble("overlapPercent", -1.0)
1681
+ state.putInt("processingMs", 0)
1682
+ state.putBoolean("isLandscape", isLandscape)
1683
+ state.putInt("paintedExtent", 0) // batch-keyframe doesn't
1684
+ state.putInt("panExtent", 0) // paint a live canvas
1685
+ state.putInt("keyframeMax", keyframeMax)
1686
+ // Batch-keyframe extras the live-engine schema doesn't carry
1687
+ // — JS reads these directly from the event payload (matches
1688
+ // iOS' direct-userInfo-write at the bottom of the iOS
1689
+ // emitter).
1690
+ state.putString("batchKeyframeThumbnailPath", thumbnailPath)
1691
+ state.putInt("batchKeyframeIndex", keyframeIndex)
1692
+ emitState(state)
1693
+ }
1694
+
1695
+ // ── OpenCV bootstrap ────────────────────────────────────────────
1696
+
1697
+ /**
1698
+ * 2026-05-14 — stitch-mode auto-resolution.
1699
+ *
1700
+ * Inputs are the first and last accepted-keyframe poses captured
1701
+ * during this batch capture. Each pose is `[tx, ty, tz, qx, qy,
1702
+ * qz, qw]` in the AR-session world frame. When either pose is
1703
+ * null (e.g., < 2 keyframes were accepted, OR the capture used a
1704
+ * non-AR camera path where ARKit/ARCore poses aren't available)
1705
+ * we default to 'scans' — that's the safer of the two: SCANS on
1706
+ * pure rotation produces a slightly-less-sharp output, while
1707
+ * PANORAMA on translation produces an unbounded compositing
1708
+ * canvas and the lmkd kill we observed 2026-05-14.
1709
+ *
1710
+ * Heuristic (see design doc 2026-05-13-stitch-pipeline-mode-selection):
1711
+ * translation_score = ||t_last − t_first|| / 0.10 (10 cm → 1.0)
1712
+ * rotation_score = angle(fwd_last, fwd_first) / 1.00 (1 rad ≈ 57° → 1.0)
1713
+ * ratio = translation_score / (translation_score + rotation_score)
1714
+ * ratio ≥ 0.55 → SCANS (biased toward SCANS for safety)
1715
+ * ratio < 0.55 → PANORAMA
1716
+ *
1717
+ * Returns "panorama" or "scans" — never "auto".
1718
+ */
1719
+ private fun resolveStitchModeAuto(
1720
+ firstPose: DoubleArray?,
1721
+ lastPose: DoubleArray?,
1722
+ ): String {
1723
+ if (firstPose == null || lastPose == null) return "scans"
1724
+ if (firstPose.size != 7 || lastPose.size != 7) return "scans"
1725
+
1726
+ // Translation magnitude (Euclidean, in metres).
1727
+ val dtx = lastPose[0] - firstPose[0]
1728
+ val dty = lastPose[1] - firstPose[1]
1729
+ val dtz = lastPose[2] - firstPose[2]
1730
+ val tMeters = kotlin.math.sqrt(dtx * dtx + dty * dty + dtz * dtz)
1731
+
1732
+ // Rotation magnitude — angle between camera-forward vectors.
1733
+ // Camera-forward in body frame is (0, 0, -1) for ARKit/ARCore
1734
+ // conventions; rotated by the pose quaternion gives the world-
1735
+ // frame forward direction. Angle between the first and last
1736
+ // camera-forward vectors is the total rotation around any axis.
1737
+ val fwdFirst = qrotForward(firstPose[3], firstPose[4], firstPose[5], firstPose[6])
1738
+ val fwdLast = qrotForward(lastPose[3], lastPose[4], lastPose[5], lastPose[6])
1739
+ val dot = (fwdFirst[0] * fwdLast[0] + fwdFirst[1] * fwdLast[1] + fwdFirst[2] * fwdLast[2])
1740
+ .coerceIn(-1.0, 1.0)
1741
+ val rRadians = kotlin.math.acos(dot)
1742
+
1743
+ // Normalisation: 10 cm of translation ≈ 1 rad of rotation as
1744
+ // "equivalent magnitude" for the ratio. Empirically: shelf
1745
+ // scans cover ~30 cm of translation with ~10° (0.17 rad) of
1746
+ // rotation. ratio = (0.30/0.10) / (3.0 + 0.17) = 0.95 → SCANS.
1747
+ // Pure 90° rotation panorama: 0 translation, 1.57 rad rotation.
1748
+ // ratio = 0 / (0 + 1.57) = 0.0 → PANORAMA.
1749
+ val tScore = tMeters / 0.10
1750
+ val rScore = rRadians / 1.00
1751
+ val denom = tScore + rScore
1752
+ if (denom <= 1e-9) return "scans" // degenerate — no motion at all
1753
+ val ratio = tScore / denom
1754
+
1755
+ android.util.Log.i(
1756
+ "IncrementalStitcher",
1757
+ "stitch-mode auto: t=${"%.3f".format(tMeters)}m " +
1758
+ "r=${"%.3f".format(rRadians)}rad " +
1759
+ "ratio=${"%.3f".format(ratio)} " +
1760
+ "→ ${if (ratio >= 0.55) "scans" else "panorama"}",
1761
+ )
1762
+ return if (ratio >= 0.55) "scans" else "panorama"
1763
+ }
1764
+
1765
+ /**
1766
+ * Rotate the camera-forward unit vector (0, 0, -1) by a unit
1767
+ * quaternion (qx, qy, qz, qw). Closed-form expansion of
1768
+ * v' = q · v · q⁻¹. Same convention as `qrot` in
1769
+ * cpp/keyframe_gate.cpp.
1770
+ */
1771
+ private fun qrotForward(qx: Double, qy: Double, qz: Double, qw: Double): DoubleArray {
1772
+ // v = (0, 0, -1). q · v · q⁻¹ closed-form:
1773
+ // result = v + 2 * qw * (q_xyz × v) + 2 * q_xyz × (q_xyz × v)
1774
+ // Pre-computed for v=(0,0,-1):
1775
+ // q_xyz × v = (qy * -1 - qz * 0, qz * 0 - qx * -1, qx * 0 - qy * 0)
1776
+ // = (-qy, qx, 0)
1777
+ // q_xyz × (q_xyz × v):
1778
+ // = (qy*0 - qz*qx, qz*(-qy) - qx*0, qx*qx - qy*(-qy))
1779
+ // = (-qz*qx, -qz*qy, qx² + qy²)
1780
+ // result = (0 + 2*qw*(-qy) + 2*(-qz*qx),
1781
+ // 0 + 2*qw*qx + 2*(-qz*qy),
1782
+ // -1 + 2*qw*0 + 2*(qx² + qy²))
1783
+ return doubleArrayOf(
1784
+ -2.0 * (qw * qy + qz * qx),
1785
+ 2.0 * (qw * qx - qz * qy),
1786
+ -1.0 + 2.0 * (qx * qx + qy * qy),
1787
+ )
1788
+ }
1789
+
1790
+ private fun ensureOpenCv() {
1791
+ if (!opencvInitialised.get()) {
1792
+ try {
1793
+ System.loadLibrary("opencv_java4")
1794
+ opencvInitialised.set(true)
1795
+ } catch (e: UnsatisfiedLinkError) {
1796
+ throw IllegalStateException(
1797
+ "OpenCV native library 'opencv_java4' failed to load",
1798
+ e,
1799
+ )
1800
+ }
1801
+ }
1802
+ }
1803
+
1804
+ companion object {
1805
+ @JvmStatic
1806
+ private val opencvInitialised = AtomicBoolean(false)
1807
+
1808
+ /// Static back-pointer used by the camera view to reach the
1809
+ /// active bridge module instance without a DI dance. Set
1810
+ /// in `init {}` of the most recently constructed instance.
1811
+ @JvmStatic
1812
+ @Volatile
1813
+ var bridgeInstance: IncrementalStitcher? = null
1814
+ private set
1815
+ }
1816
+ }
1817
+
1818
+
1819
+ // ── ReadableMap helpers ─────────────────────────────────────────────
1820
+
1821
+ private fun ReadableMap.getIntOrDefault(key: String, default: Int): Int =
1822
+ if (hasKey(key) && !isNull(key)) getInt(key) else default
1823
+
1824
+ private fun ReadableMap.getDoubleOrDefault(key: String, default: Double): Double =
1825
+ if (hasKey(key) && !isNull(key)) getDouble(key) else default
1826
+
1827
+ private fun ReadableMap.getBooleanOrDefault(key: String, default: Boolean): Boolean =
1828
+ if (hasKey(key) && !isNull(key)) getBoolean(key) else default
1829
+
1830
+
1831
+ // ── Frame outcome — mirrors iOS RLISFrameOutcome ────────────────────
1832
+
1833
+ internal enum class FrameOutcome {
1834
+ AcceptedHigh,
1835
+ AcceptedMedium,
1836
+ SkippedTooClose,
1837
+ RejectedTooFar,
1838
+ RejectedSceneUniform,
1839
+ RejectedAlignmentLost,
1840
+ SkippedTrackingPoor,
1841
+ /** V12.11 Step D — operator panned BACKWARDS past the running
1842
+ * max along the pan axis. Engine has SKIPPED the paste; host
1843
+ * should auto-finalize. Rectilinear-only. */
1844
+ RejectedReverseDirection,
1845
+ }
1846
+
1847
+
1848
+ internal data class FrameTelemetry(
1849
+ val outcome: FrameOutcome,
1850
+ val overlapPercent: Double,
1851
+ val matchCount: Int,
1852
+ val inlierRatio: Double,
1853
+ val confidence: Double,
1854
+ val processingMs: Double,
1855
+ /** V12.12 — engine-detected orientation. Mirrors iOS'
1856
+ * `RLISFrameTelemetry.isLandscape`. TRUE for landscape capture
1857
+ * (vertical pan), FALSE for portrait (horizontal pan). Stays
1858
+ * at the FIRST-FRAME determination thereafter. */
1859
+ val isLandscape: Boolean = false,
1860
+ )
1861
+
1862
+
1863
+ internal data class StitcherSnapshot(
1864
+ val panoramaPath: String,
1865
+ val width: Int,
1866
+ val height: Int,
1867
+ val acceptedCount: Int,
1868
+ )
1869
+
1870
+
1871
+ /**
1872
+ * Pure-OpenCV implementation of the incremental algorithm. No RN
1873
+ * dependency — this class can be unit-tested with synthetic Mat
1874
+ * inputs. Exact algorithmic mirror of iOS' OpenCVIncrementalStitcher.mm.
1875
+ */
1876
+ internal class IncrementalEngine(
1877
+ val composeWidth: Int,
1878
+ val composeHeight: Int,
1879
+ val canvasWidth: Int,
1880
+ val canvasHeight: Int,
1881
+ val featherPx: Int,
1882
+ val snapshotJpegQuality: Int,
1883
+ val snapshotEveryNAccepts: Int,
1884
+ /// 0/90/180/270 — rotation applied to each ingested frame before
1885
+ /// any other processing. See iOS' equivalent for the full
1886
+ /// rationale. JS computes from device orientation.
1887
+ val frameRotationDegrees: Int,
1888
+ ) {
1889
+ private val canvas: Mat = Mat.zeros(canvasHeight, canvasWidth, CvType.CV_8UC3)
1890
+ private val canvasMask: Mat = Mat.zeros(canvasHeight, canvasWidth, CvType.CV_8UC1)
1891
+
1892
+ /// V7 pose-driven state — sensor-native compute path. Mirrors iOS.
1893
+ private var firstRotationArkit: Mat = Mat()
1894
+ private var kCompose: Mat = Mat()
1895
+ private var tCanvas: Mat = Mat.eye(3, 3, CvType.CV_64F)
1896
+ private val mArkitToCv: Mat = Mat(3, 3, CvType.CV_64F).apply {
1897
+ // diag(1, -1, -1) — ARKit/ARCore (Y-up, -Z forward) → OpenCV.
1898
+ setTo(Scalar(0.0))
1899
+ put(0, 0, 1.0); put(1, 1, -1.0); put(2, 2, -1.0)
1900
+ }
1901
+
1902
+ private var lastAcceptedYaw: Double = 0.0
1903
+ private var lastAcceptedPitch: Double = 0.0
1904
+ private var hasFirstFrame: Boolean = false
1905
+ private var acceptsSinceSnapshot: Int = 0
1906
+ var acceptedCount: Int = 0
1907
+ private set
1908
+ private var snapshotSeq: Int = 0
1909
+ var lastState: WritableMap? = null
1910
+ private set
1911
+
1912
+ /**
1913
+ * Read the JPEG at `path`, downscale to compose-resolution, run
1914
+ * the same algorithm as the iOS engine.
1915
+ */
1916
+ fun addFrameAtPath(
1917
+ path: String,
1918
+ qx: Double,
1919
+ qy: Double,
1920
+ qz: Double,
1921
+ qw: Double,
1922
+ fx: Double,
1923
+ fy: Double,
1924
+ cx: Double,
1925
+ cy: Double,
1926
+ imageWidth: Int,
1927
+ imageHeight: Int,
1928
+ yaw: Double,
1929
+ pitch: Double,
1930
+ fovHorizDegrees: Double,
1931
+ fovVertDegrees: Double,
1932
+ trackingPoor: Boolean,
1933
+ ): FrameTelemetry {
1934
+ val t0 = System.nanoTime()
1935
+ if (trackingPoor) {
1936
+ return FrameTelemetry(
1937
+ FrameOutcome.SkippedTrackingPoor, -1.0, 0, 0.0, 0.0,
1938
+ msSince(t0),
1939
+ )
1940
+ }
1941
+
1942
+ val cleaned = stripFileScheme(path)
1943
+ val srcRaw = Imgcodecs.imread(cleaned, Imgcodecs.IMREAD_COLOR)
1944
+ if (srcRaw.empty()) {
1945
+ return FrameTelemetry(
1946
+ FrameOutcome.SkippedTrackingPoor, -1.0, 0, 0.0, 0.0,
1947
+ msSince(t0),
1948
+ )
1949
+ }
1950
+ // V7: NO input rotation. ARCore (and the JS gyro fallback)
1951
+ // deliver sensor-native landscape frames; we keep them in
1952
+ // that frame through the entire compute pipeline. Output
1953
+ // rotation for display happens at snapshot/finalize time.
1954
+ // See iOS' equivalent fix for the architectural rationale.
1955
+ val frame = downsampleToCompose(srcRaw)
1956
+ if (frame !== srcRaw) srcRaw.release()
1957
+
1958
+ // Build R_new from quaternion.
1959
+ val rNew = quaternionToRotationMat(qx, qy, qz, qw)
1960
+
1961
+ if (!hasFirstFrame) {
1962
+ firstRotationArkit = rNew.clone()
1963
+ // V7: K is in COMPOSE pixel coordinates. Sensor intrinsics
1964
+ // get scaled by the same uniform factor we used to downsample
1965
+ // the frame, so K · ray → pixel produces the right pixel
1966
+ // in compose space directly. No rotation chain needed.
1967
+ val sx = frame.cols().toDouble() / maxOf(1, imageWidth)
1968
+ val sy = frame.rows().toDouble() / maxOf(1, imageHeight)
1969
+ val s = 0.5 * (sx + sy)
1970
+ kCompose = Mat(3, 3, CvType.CV_64F).apply {
1971
+ setTo(Scalar(0.0))
1972
+ put(0, 0, fx * s); put(0, 2, cx * s)
1973
+ put(1, 1, fy * s); put(1, 2, cy * s)
1974
+ put(2, 2, 1.0)
1975
+ }
1976
+
1977
+ // Place first frame at canvas centre.
1978
+ val ox = (canvas.cols() - frame.cols()) / 2
1979
+ val oy = (canvas.rows() - frame.rows()) / 2
1980
+ val roi = Rect(ox, oy, frame.cols(), frame.rows())
1981
+ frame.copyTo(canvas.submat(roi))
1982
+ canvasMask.submat(roi).setTo(Scalar(255.0))
1983
+ tCanvas = Mat.eye(3, 3, CvType.CV_64F)
1984
+ tCanvas.put(0, 2, ox.toDouble())
1985
+ tCanvas.put(1, 2, oy.toDouble())
1986
+
1987
+ lastAcceptedYaw = yaw
1988
+ lastAcceptedPitch = pitch
1989
+ hasFirstFrame = true
1990
+ acceptedCount = 1
1991
+ frame.release()
1992
+ return FrameTelemetry(
1993
+ FrameOutcome.AcceptedHigh, 0.0, 0, 0.0, 1.0, msSince(t0),
1994
+ )
1995
+ }
1996
+
1997
+ val overlap = computeOverlapPct(
1998
+ yaw - lastAcceptedYaw, pitch - lastAcceptedPitch,
1999
+ fovHorizDegrees, fovVertDegrees,
2000
+ )
2001
+ if (overlap > MAX_OVERLAP_PCT) {
2002
+ frame.release()
2003
+ return FrameTelemetry(
2004
+ FrameOutcome.SkippedTooClose, overlap, 0, 0.0, 0.0, msSince(t0),
2005
+ )
2006
+ }
2007
+ if (overlap < MIN_OVERLAP_PCT) {
2008
+ frame.release()
2009
+ return FrameTelemetry(
2010
+ FrameOutcome.RejectedTooFar, overlap, 0, 0.0, 0.0, msSince(t0),
2011
+ )
2012
+ }
2013
+
2014
+ // V7 pose-driven homography (sensor-native compose space):
2015
+ // R_rel_cv = M · R_first⁻¹ · R_new · M
2016
+ // H_compose = K_compose · R_rel_cv · K_compose⁻¹
2017
+ // H_canvas = T_canvas · H_compose
2018
+ // No R2S/S chain — the v6 bug was applying input rotation
2019
+ // and undoing it via the chain; v7 keeps everything in
2020
+ // sensor-native compose space and rotates only at output.
2021
+ val firstInv = Mat()
2022
+ Core.transpose(firstRotationArkit, firstInv)
2023
+ val tmp1 = Mat(); Core.gemm(mArkitToCv, firstInv, 1.0, Mat(), 0.0, tmp1)
2024
+ val tmp2 = Mat(); Core.gemm(tmp1, rNew, 1.0, Mat(), 0.0, tmp2)
2025
+ val rRelCv = Mat(); Core.gemm(tmp2, mArkitToCv, 1.0, Mat(), 0.0, rRelCv)
2026
+ firstInv.release(); tmp1.release(); tmp2.release()
2027
+
2028
+ val kInv = kCompose.inv()
2029
+ val hcTmp = Mat(); Core.gemm(kCompose, rRelCv, 1.0, Mat(), 0.0, hcTmp)
2030
+ val hCompose = Mat(); Core.gemm(hcTmp, kInv, 1.0, Mat(), 0.0, hCompose)
2031
+ kInv.release(); hcTmp.release(); rRelCv.release(); rNew.release()
2032
+
2033
+ val hCanvas = Mat(); Core.gemm(tCanvas, hCompose, 1.0, Mat(), 0.0, hCanvas)
2034
+ hCompose.release()
2035
+
2036
+ warpAndBlend(frame, hCanvas)
2037
+ hCanvas.release()
2038
+ frame.release()
2039
+
2040
+ lastAcceptedYaw = yaw
2041
+ lastAcceptedPitch = pitch
2042
+ acceptedCount++
2043
+
2044
+ // Confidence as in iOS — function of how centred the overlap
2045
+ // is in the [10, 75]% acceptance window.
2046
+ val midOverlap = 0.5 * (MIN_OVERLAP_PCT + MAX_OVERLAP_PCT)
2047
+ val overlapDistance = kotlin.math.abs(overlap - midOverlap) /
2048
+ (MAX_OVERLAP_PCT - midOverlap)
2049
+ val confidence = maxOf(0.0, 1.0 - overlapDistance)
2050
+ val outcome = if (confidence >= 0.6) FrameOutcome.AcceptedHigh
2051
+ else FrameOutcome.AcceptedMedium
2052
+
2053
+ return FrameTelemetry(
2054
+ outcome, overlap, -1, -1.0, confidence, msSince(t0),
2055
+ )
2056
+ }
2057
+
2058
+ /** Write a JPEG snapshot if accept-counter has hit the configured cadence. */
2059
+ fun snapshotIfDue(tele: FrameTelemetry): WritableMap? {
2060
+ val isAccept = tele.outcome == FrameOutcome.AcceptedHigh
2061
+ || tele.outcome == FrameOutcome.AcceptedMedium
2062
+ var snapshotPath: String? = null
2063
+ var snapW = 0
2064
+ var snapH = 0
2065
+ if (isAccept) {
2066
+ acceptsSinceSnapshot++
2067
+ if (acceptsSinceSnapshot >= snapshotEveryNAccepts) {
2068
+ acceptsSinceSnapshot = 0
2069
+ snapshotSeq++
2070
+ val slot = snapshotSeq % 4
2071
+ val tmpPath = "${System.getProperty("java.io.tmpdir") ?: "/data/local/tmp"}" +
2072
+ "/rlis-live-$slot.jpg"
2073
+ // tightCrop = true for live snapshots: the canvas is
2074
+ // 4800x2200, but most of it is empty until the pan
2075
+ // covers it. Without a tight crop, every snapshot
2076
+ // was a ~24 MB JPEG that RN's <Image> couldn't keep
2077
+ // up with. Tight-cropped snapshots are 50–500 KB.
2078
+ val snap = writeJpeg(tmpPath, snapshotJpegQuality, tightCrop = true)
2079
+ if (snap != null) {
2080
+ snapshotPath = snap.panoramaPath
2081
+ snapW = snap.width
2082
+ snapH = snap.height
2083
+ }
2084
+ }
2085
+ }
2086
+
2087
+ val map = Arguments.createMap().apply {
2088
+ putInt("width", snapW)
2089
+ putInt("height", snapH)
2090
+ putInt("acceptedCount", acceptedCount)
2091
+ putInt("outcome", tele.outcome.ordinal)
2092
+ putDouble("confidence", tele.confidence)
2093
+ putDouble("overlapPercent", tele.overlapPercent)
2094
+ putDouble("processingMs", tele.processingMs)
2095
+ if (snapshotPath != null) putString("panoramaPath", snapshotPath)
2096
+ }
2097
+ lastState = map
2098
+ return map
2099
+ }
2100
+
2101
+ fun finalize(outputPath: String, quality: Int): StitcherSnapshot {
2102
+ val cleaned = stripFileScheme(outputPath)
2103
+ val snap = writeJpeg(cleaned, quality, tightCrop = true)
2104
+ ?: throw IllegalStateException(
2105
+ "No frames have been accepted yet, or write failed: $cleaned",
2106
+ )
2107
+ release()
2108
+ return snap
2109
+ }
2110
+
2111
+ fun release() {
2112
+ canvas.release()
2113
+ canvasMask.release()
2114
+ firstRotationArkit.release()
2115
+ kCompose.release()
2116
+ tCanvas.release()
2117
+ mArkitToCv.release()
2118
+ }
2119
+
2120
+ // ── internal helpers ────────────────────────────────────────────
2121
+
2122
+ private fun downsampleToCompose(src: Mat): Mat {
2123
+ // Uniform scale that fits inside the compose-dim budget — the
2124
+ // smaller of the two ratios wins so neither axis distorts.
2125
+ val sw = src.cols().toDouble()
2126
+ val sh = src.rows().toDouble()
2127
+ var scale = minOf(composeWidth.toDouble() / sw, composeHeight.toDouble() / sh)
2128
+ if (scale > 1.0) scale = 1.0 // never upscale
2129
+ val outW = maxOf(1, (sw * scale).toInt())
2130
+ val outH = maxOf(1, (sh * scale).toInt())
2131
+ if (src.cols() == outW && src.rows() == outH) return src
2132
+ val out = Mat()
2133
+ Imgproc.resize(src, out, Size(outW.toDouble(), outH.toDouble()), 0.0, 0.0, Imgproc.INTER_AREA)
2134
+ return out
2135
+ }
2136
+
2137
+ // `placeFirstFrame` was dropped in v6 — the first-frame logic is
2138
+ // now inlined in `addFrameAtPath` so the engine can capture the
2139
+ // reference pose + intrinsics in the same place it positions the
2140
+ // frame on the canvas.
2141
+
2142
+ private fun warpAndBlend(frame: Mat, worldH: Mat) {
2143
+ val canvasSize = Size(canvasWidth.toDouble(), canvasHeight.toDouble())
2144
+
2145
+ val warped = Mat()
2146
+ Imgproc.warpPerspective(
2147
+ frame, warped, worldH, canvasSize,
2148
+ Imgproc.INTER_LINEAR, Core.BORDER_CONSTANT, Scalar(0.0, 0.0, 0.0),
2149
+ )
2150
+
2151
+ val frameOnesMask = Mat(frame.rows(), frame.cols(), CvType.CV_8UC1, Scalar(255.0))
2152
+ val warpedMask = Mat()
2153
+ Imgproc.warpPerspective(
2154
+ frameOnesMask, warpedMask, worldH, canvasSize,
2155
+ Imgproc.INTER_NEAREST, Core.BORDER_CONSTANT, Scalar(0.0),
2156
+ )
2157
+ frameOnesMask.release()
2158
+
2159
+ // Hard midline seam (replaces v4 ratio-feather). Same fix
2160
+ // as iOS v5: each output pixel comes from exactly one frame,
2161
+ // so misalignment between frames can't produce ghosts. The
2162
+ // seam is placed where each pixel is equidistant from both
2163
+ // frames' outer edges (the "middle" of the overlap), then
2164
+ // softened with a small Gaussian to hide the pixel-perfect
2165
+ // cut.
2166
+ val distNew = Mat()
2167
+ Imgproc.distanceTransform(warpedMask, distNew, Imgproc.DIST_L2, 3)
2168
+ val distCanvas = Mat()
2169
+ Imgproc.distanceTransform(canvasMask, distCanvas, Imgproc.DIST_L2, 3)
2170
+
2171
+ // alpha8: 255 where new is deeper, 0 where canvas is deeper.
2172
+ val alpha8 = Mat()
2173
+ Core.compare(distNew, distCanvas, alpha8, Core.CMP_GE)
2174
+
2175
+ // First-touch regions need new frame to write unconditionally.
2176
+ val noPriorMask = Mat()
2177
+ Core.compare(canvasMask, Scalar(0.0), noPriorMask, Core.CMP_EQ)
2178
+ alpha8.setTo(Scalar(255.0), noPriorMask)
2179
+ noPriorMask.release()
2180
+
2181
+ val alpha = Mat()
2182
+ alpha8.convertTo(alpha, CvType.CV_32F, 1.0 / 255.0)
2183
+ alpha8.release()
2184
+ Imgproc.GaussianBlur(alpha, alpha, Size(7.0, 7.0), 0.0)
2185
+ distNew.release(); distCanvas.release()
2186
+
2187
+ val alphaChannels = mutableListOf(alpha, alpha, alpha)
2188
+ val alpha3 = Mat()
2189
+ Core.merge(alphaChannels, alpha3)
2190
+ val invAlpha3 = Mat()
2191
+ Core.subtract(Mat.ones(alpha3.size(), alpha3.type()).apply {
2192
+ setTo(Scalar(1.0, 1.0, 1.0))
2193
+ }, alpha3, invAlpha3)
2194
+
2195
+ val warpedF = Mat(); warped.convertTo(warpedF, CvType.CV_32FC3)
2196
+ val canvasF = Mat(); canvas.convertTo(canvasF, CvType.CV_32FC3)
2197
+ val blendedF = Mat()
2198
+ Core.multiply(warpedF, alpha3, warpedF)
2199
+ Core.multiply(canvasF, invAlpha3, canvasF)
2200
+ Core.add(warpedF, canvasF, blendedF)
2201
+ warpedF.release(); canvasF.release()
2202
+ alpha.release(); alpha3.release(); invAlpha3.release()
2203
+
2204
+ val blended8 = Mat()
2205
+ blendedF.convertTo(blended8, CvType.CV_8UC3)
2206
+ blendedF.release()
2207
+ // Only write where warpedMask is set; rest of canvas is unchanged.
2208
+ blended8.copyTo(canvas, warpedMask)
2209
+ blended8.release()
2210
+
2211
+ Core.bitwise_or(canvasMask, warpedMask, canvasMask)
2212
+ warpedMask.release()
2213
+ warped.release()
2214
+ }
2215
+
2216
+ private fun writeJpeg(
2217
+ outputPath: String,
2218
+ quality: Int,
2219
+ tightCrop: Boolean,
2220
+ ): StitcherSnapshot? {
2221
+ if (acceptedCount == 0) return null
2222
+ var crop = Rect(0, 0, canvas.cols(), canvas.rows())
2223
+ if (tightCrop) {
2224
+ val nonZero = MatOfPoint2f()
2225
+ // boundingRect on the mask matrix gives us the tight crop;
2226
+ // OpenCV Java's API takes a Mat of points, but for an
2227
+ // image mask we use Imgproc.boundingRect on a contour.
2228
+ // Cheaper path: walk the mask once.
2229
+ val contoured = Imgproc.boundingRect(MaskNonZeroContour(canvasMask))
2230
+ if (contoured.width > 0 && contoured.height > 0) {
2231
+ crop = contoured
2232
+ }
2233
+ nonZero.release()
2234
+ }
2235
+ val cropped = Mat(canvas, crop)
2236
+ // V7.1 GRAVITY-DERIVED OUTPUT ROTATION. Mirrors iOS — see
2237
+ // OpenCVIncrementalStitcher.mm for the full derivation. The
2238
+ // rotation comes from the AR pose (which knows gravity) so
2239
+ // we don't need a device-orientation hook (which was the
2240
+ // source of the v7 "sideways for landscape" bug).
2241
+ var rotationDeg = 0
2242
+ if (hasFirstFrame && !firstRotationArkit.empty()) {
2243
+ val gravWorld = Mat(3, 1, CvType.CV_64F).apply {
2244
+ put(0, 0, 0.0); put(1, 0, -1.0); put(2, 0, 0.0)
2245
+ }
2246
+ val firstT = Mat()
2247
+ Core.transpose(firstRotationArkit, firstT)
2248
+ val gravArkit = Mat(); Core.gemm(firstT, gravWorld, 1.0, Mat(), 0.0, gravArkit)
2249
+ val gravCv = Mat(); Core.gemm(mArkitToCv, gravArkit, 1.0, Mat(), 0.0, gravCv)
2250
+ val gx = gravCv.get(0, 0)[0]
2251
+ val gy = gravCv.get(1, 0)[0]
2252
+ val angle = kotlin.math.atan2(gx, gy) * 180.0 / Math.PI
2253
+ rotationDeg = (kotlin.math.round(angle / 90.0).toInt()) * 90
2254
+ rotationDeg = ((rotationDeg % 360) + 360) % 360
2255
+ gravWorld.release(); firstT.release(); gravArkit.release(); gravCv.release()
2256
+ }
2257
+ val out = when (rotationDeg) {
2258
+ 90 -> Mat().also { Core.rotate(cropped, it, Core.ROTATE_90_CLOCKWISE) }
2259
+ 180 -> Mat().also { Core.rotate(cropped, it, Core.ROTATE_180) }
2260
+ 270 -> Mat().also { Core.rotate(cropped, it, Core.ROTATE_90_COUNTERCLOCKWISE) }
2261
+ else -> cropped
2262
+ }
2263
+ val outW = out.cols()
2264
+ val outH = out.rows()
2265
+ val params = MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, quality)
2266
+ val ok = Imgcodecs.imwrite(outputPath, out, params)
2267
+ if (out !== cropped) out.release()
2268
+ cropped.release()
2269
+ if (!ok) return null
2270
+ return StitcherSnapshot(
2271
+ panoramaPath = outputPath,
2272
+ width = outW,
2273
+ height = outH,
2274
+ acceptedCount = acceptedCount,
2275
+ )
2276
+ }
2277
+
2278
+ private fun msSince(t0Nanos: Long): Double =
2279
+ (System.nanoTime() - t0Nanos) / 1_000_000.0
2280
+
2281
+ companion object {
2282
+ // v3 thresholds — relaxed match-count + inlier minimums for
2283
+ // light-texture shelf scenes; tighter det range because the
2284
+ // affine fit produces a much narrower legitimate scale band.
2285
+ private const val MIN_OVERLAP_PCT = 10.0
2286
+ private const val MAX_OVERLAP_PCT = 75.0
2287
+ private const val MIN_MATCHES_ACCEPT = 10
2288
+ private const val MIN_INLIER_RATIO_ACCEPT = 0.18
2289
+ private const val HIGH_CONF_MATCHES = 60
2290
+ private const val HIGH_CONF_INLIER_RATIO = 0.55
2291
+ private const val ORB_MAX_FEATURES = 1000
2292
+ private const val ORB_SCALE_FACTOR = 1.2f
2293
+ private const val ORB_LEVELS = 8
2294
+ private const val ORB_EDGE_THRESHOLD = 31
2295
+ private const val LOWE_RATIO = 0.75f
2296
+ private const val RANSAC_REPROJ_THRESH = 5.0
2297
+ private const val HOM_DET_MIN = 0.7
2298
+ private const val HOM_DET_MAX = 1.4
2299
+ }
2300
+ }
2301
+
2302
+
2303
+ /// Helper for OpenCV's odd boundingRect signature on a binary mask.
2304
+ /// Wraps the mask's non-zero pixel coords as a MatOfPoint that
2305
+ /// `Imgproc.boundingRect` will accept.
2306
+ private fun MaskNonZeroContour(mask: Mat): org.opencv.core.MatOfPoint {
2307
+ val locations = Mat()
2308
+ Core.findNonZero(mask, locations)
2309
+ if (locations.empty()) {
2310
+ locations.release()
2311
+ return org.opencv.core.MatOfPoint()
2312
+ }
2313
+ val pts = mutableListOf<Point>()
2314
+ // findNonZero returns CV_32SC2 with N rows, 1 col. Each entry is
2315
+ // a (col, row) pair; build a point list for boundingRect.
2316
+ val n = locations.rows()
2317
+ val buf = IntArray(2)
2318
+ for (i in 0 until n) {
2319
+ locations.get(i, 0, buf)
2320
+ pts.add(Point(buf[0].toDouble(), buf[1].toDouble()))
2321
+ }
2322
+ locations.release()
2323
+ val out = org.opencv.core.MatOfPoint()
2324
+ out.fromList(pts)
2325
+ return out
2326
+ }
2327
+
2328
+
2329
+ // computeOverlapPct, stripFileScheme — same code as iOS's static helpers,
2330
+ // transcribed to Kotlin.
2331
+
2332
+ internal fun computeOverlapPct(
2333
+ deltaYaw: Double,
2334
+ deltaPitch: Double,
2335
+ fovHorizDegrees: Double,
2336
+ fovVertDegrees: Double,
2337
+ ): Double {
2338
+ val absYaw = kotlin.math.abs(deltaYaw)
2339
+ val absPitch = kotlin.math.abs(deltaPitch)
2340
+ var fovH = fovHorizDegrees * Math.PI / 180.0
2341
+ var fovV = fovVertDegrees * Math.PI / 180.0
2342
+ if (fovH <= 1e-6) fovH = 65.0 * Math.PI / 180.0
2343
+ if (fovV <= 1e-6) fovV = 50.0 * Math.PI / 180.0
2344
+ val overlap = if (absYaw >= absPitch) {
2345
+ 1.0 - absYaw / fovH
2346
+ } else {
2347
+ 1.0 - absPitch / fovV
2348
+ }
2349
+ return overlap.coerceIn(0.0, 1.0) * 100.0
2350
+ }
2351
+
2352
+
2353
+ internal fun stripFileScheme(path: String): String =
2354
+ if (path.startsWith("file://")) path.removePrefix("file://") else path
2355
+
2356
+
2357
+ /// Quaternion → 3x3 rotation matrix, mirroring iOS `quaternionToRotationMat`.
2358
+ internal fun quaternionToRotationMat(qx0: Double, qy0: Double, qz0: Double, qw0: Double): Mat {
2359
+ var qx = qx0; var qy = qy0; var qz = qz0; var qw = qw0
2360
+ val n = kotlin.math.sqrt(qx*qx + qy*qy + qz*qz + qw*qw)
2361
+ if (n > 1e-9) { qx /= n; qy /= n; qz /= n; qw /= n }
2362
+ val r = Mat(3, 3, CvType.CV_64F)
2363
+ r.put(0, 0, 1 - 2*(qy*qy + qz*qz)); r.put(0, 1, 2*(qx*qy - qw*qz)); r.put(0, 2, 2*(qx*qz + qw*qy))
2364
+ r.put(1, 0, 2*(qx*qy + qw*qz)); r.put(1, 1, 1 - 2*(qx*qx + qz*qz)); r.put(1, 2, 2*(qy*qz - qw*qx))
2365
+ r.put(2, 0, 2*(qx*qz - qw*qy)); r.put(2, 1, 2*(qy*qz + qw*qx)); r.put(2, 2, 1 - 2*(qx*qx + qy*qy))
2366
+ return r
2367
+ }
2368
+
2369
+
2370
+ // `sensorRotationMatrix` was removed in V7 — the rotation chain it
2371
+ // powered is no longer in the homography path. See iOS' equivalent.