react-native-image-stitcher 0.14.2 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +164 -0
  2. package/README.md +35 -0
  3. package/RNImageStitcher.podspec +8 -7
  4. package/android/build.gradle +0 -16
  5. package/android/src/main/cpp/CMakeLists.txt +2 -63
  6. package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
  7. package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
  8. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
  9. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
  10. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
  11. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
  12. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +129 -71
  13. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +49 -0
  14. package/cpp/keyframe_gate.cpp +82 -23
  15. package/cpp/keyframe_gate.hpp +31 -2
  16. package/cpp/stitcher.cpp +208 -28
  17. package/cpp/tests/CMakeLists.txt +18 -12
  18. package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
  19. package/cpp/tests/warp_guard_test.cpp +48 -0
  20. package/cpp/warp_guard.hpp +41 -0
  21. package/dist/camera/Camera.d.ts +31 -16
  22. package/dist/camera/Camera.js +11 -3
  23. package/dist/camera/CameraView.js +93 -3
  24. package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
  25. package/dist/camera/CaptureStitchStatsToast.js +27 -7
  26. package/dist/camera/PanoramaSettings.d.ts +10 -223
  27. package/dist/camera/PanoramaSettings.js +6 -28
  28. package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
  29. package/dist/camera/PanoramaSettingsBridge.js +3 -102
  30. package/dist/camera/PanoramaSettingsModal.js +7 -1
  31. package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
  32. package/dist/camera/buildPanoramaInitialSettings.js +4 -0
  33. package/dist/camera/cameraErrorMessages.d.ts +32 -0
  34. package/dist/camera/cameraErrorMessages.js +53 -0
  35. package/dist/camera/selectCaptureDevice.d.ts +5 -1
  36. package/dist/camera/selectCaptureDevice.js +22 -2
  37. package/dist/camera/useCapture.js +38 -0
  38. package/dist/index.d.ts +5 -8
  39. package/dist/index.js +11 -34
  40. package/dist/stitching/incremental.d.ts +1 -117
  41. package/dist/stitching/stitchVideo.d.ts +0 -35
  42. package/dist/types.d.ts +0 -87
  43. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
  44. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
  45. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
  46. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
  47. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
  48. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
  49. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
  50. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
  51. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
  52. package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +82 -7
  53. package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
  54. package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
  55. package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
  56. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
  57. package/package.json +3 -2
  58. package/src/camera/Camera.tsx +44 -23
  59. package/src/camera/CameraView.tsx +113 -4
  60. package/src/camera/CaptureStitchStatsToast.tsx +58 -14
  61. package/src/camera/PanoramaSettings.ts +16 -289
  62. package/src/camera/PanoramaSettingsBridge.ts +3 -114
  63. package/src/camera/PanoramaSettingsModal.tsx +14 -1
  64. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
  65. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
  66. package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
  67. package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
  68. package/src/camera/buildPanoramaInitialSettings.ts +17 -0
  69. package/src/camera/cameraErrorMessages.ts +84 -0
  70. package/src/camera/selectCaptureDevice.ts +28 -3
  71. package/src/camera/useCapture.ts +44 -1
  72. package/src/index.ts +11 -40
  73. package/src/stitching/incremental.ts +3 -140
  74. package/src/stitching/stitchVideo.ts +0 -26
  75. package/src/types.ts +0 -95
  76. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
  77. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
  78. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
  79. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
  80. package/cpp/stitcher_frame_jsi.cpp +0 -214
  81. package/cpp/stitcher_frame_jsi.hpp +0 -108
  82. package/cpp/stitcher_proxy_jsi.cpp +0 -109
  83. package/cpp/stitcher_proxy_jsi.hpp +0 -46
  84. package/cpp/stitcher_worklet_dispatch.cpp +0 -103
  85. package/cpp/stitcher_worklet_dispatch.hpp +0 -71
  86. package/cpp/stitcher_worklet_registry.cpp +0 -91
  87. package/cpp/stitcher_worklet_registry.hpp +0 -146
  88. package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
  89. package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
  90. package/dist/stitching/IncrementalStitcherView.js +0 -157
  91. package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
  92. package/dist/stitching/StitcherWorkletRegistry.js +0 -78
  93. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
  94. package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
  95. package/dist/stitching/useFrameProcessor.d.ts +0 -119
  96. package/dist/stitching/useFrameProcessor.js +0 -196
  97. package/dist/stitching/useFrameStream.d.ts +0 -34
  98. package/dist/stitching/useFrameStream.js +0 -234
  99. package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
  100. package/dist/stitching/useThrottledFrameProcessor.js +0 -132
  101. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
  102. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
  104. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
  105. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
  106. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
  107. package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
  108. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  109. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
  110. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
  111. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
  112. package/src/stitching/IncrementalStitcherView.tsx +0 -198
  113. package/src/stitching/StitcherWorkletRegistry.ts +0 -156
  114. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
  115. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
  116. package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
  117. package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
  118. package/src/stitching/useFrameProcessor.ts +0 -226
  119. package/src/stitching/useFrameStream.ts +0 -271
  120. package/src/stitching/useThrottledFrameProcessor.ts +0 -145
@@ -1,1081 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- package io.imagestitcher.rn
3
-
4
- import android.util.Log
5
- import com.facebook.react.bridge.WritableMap
6
- // V13.0a — removed unused imports (Calib3d, DMatch, KeyPoint,
7
- // MatOfByte, MatOfDMatch, MatOfKeyPoint, MatOfPoint2f, BFMatcher,
8
- // ORB) after the homography revert. CvType + MatOfInt kept because
9
- // they're still used elsewhere in the engine.
10
- import org.opencv.core.Core
11
- import org.opencv.core.CvType
12
- import org.opencv.core.Mat
13
- import org.opencv.core.MatOfInt
14
- import org.opencv.core.Point
15
- import org.opencv.core.Rect
16
- import org.opencv.core.Scalar
17
- import org.opencv.core.Size
18
- import org.opencv.imgcodecs.Imgcodecs
19
- import org.opencv.imgproc.Imgproc
20
- import java.io.File
21
- import java.util.Locale
22
- import java.util.concurrent.atomic.AtomicInteger
23
- import kotlin.math.atan2
24
- import kotlin.math.cos
25
- import kotlin.math.floor
26
- import kotlin.math.max
27
- import kotlin.math.min
28
- import kotlin.math.round
29
- import kotlin.math.sin
30
- import kotlin.math.sqrt
31
-
32
- /**
33
- * Android port of iOS' OpenCVFirstWinsCylindricalStitcher with the
34
- * full V12.x stack baked in:
35
- *
36
- * V12.2 — cylindrical projection (mirror-fixed: theta = atan2(-wx, wz))
37
- * V12.3 — orientation-aware cylinder axis (transverse for landscape)
38
- * V12.4 — central 70% (pan) × 85% (perpendicular) post-warp crop
39
- * V12.6 — orientation detection from R_panToCam at first frame
40
- * (NOT from JS-passed frameRotationDegrees, which is wrong
41
- * under portrait-locked hosts — pose-derived detection
42
- * works regardless of host orientation config, so it's
43
- * the single source of truth)
44
- * V12.7 — rectilinear path: skip cylindrical warp entirely. First
45
- * frame pasted raw onto canvas; subsequent frames contribute
46
- * a narrow central strip placed by pose-delta around the
47
- * dominant pan axis. First-painted-wins masks the strip.
48
- *
49
- * Differences from the iOS engine:
50
- * - Frames arrive as JPEG paths (vision-camera + gyro driver writes
51
- * them to disk on each accept), not as raw CVPixelBuffers. We
52
- * read with Imgcodecs and downsample to compose dims here.
53
- * - OpenCV Java bindings: cv::Mat is org.opencv.core.Mat with
54
- * element accessors that return double[] — slower per-element
55
- * than the C++ at<double>() but only used in setup paths, not
56
- * the per-pixel inverse-map loop (that's done with put/get
57
- * bulk float arrays for the remap).
58
- *
59
- * What this DOESN'T do (yet, intentional scope):
60
- * - No KLT optical-flow refinement (iOS hybrid only; firstwins
61
- * doesn't use it either)
62
- * - No per-pair gain compensation
63
- * - No CLAHE finalize (kept simpler to match firstwins minimalism)
64
- */
65
- internal class IncrementalFirstwinsEngine(
66
- val composeWidth: Int,
67
- val composeHeight: Int,
68
- val canvasWidth: Int,
69
- val canvasHeight: Int,
70
- val snapshotJpegQuality: Int,
71
- val snapshotEveryNAccepts: Int,
72
- /// 0/90/180/270 — output rotation applied at finalize for display.
73
- /// Compute pipeline works in sensor-native landscape compose space.
74
- /// V12.6: orientation detection no longer uses this; ARKit/ARCore
75
- /// pose at first frame is ground truth.
76
- val frameRotationDegrees: Int,
77
- /// V12.7 Variant B: when true, skip cylindrical warp. See class doc.
78
- val useRectilinear: Boolean,
79
- /// Critic #27 fix: cache-dir from the bridge. Snapshot JPEGs are
80
- /// written here. System.getProperty("java.io.tmpdir") on Android
81
- /// resolves to /data/local/tmp which is NOT writable by ordinary
82
- /// apps, so the previous version silently dropped every snapshot.
83
- val snapshotCacheDir: String,
84
- ) {
85
- // V12.12 — canvas allocation:
86
- // • RECTILINEAR engine: deferred to first-frame branch so the
87
- // canvas can be sized from the pose-detected orientation.
88
- // Empty Mats here; the first-frame branch checks
89
- // `canvas.empty()` and allocates.
90
- // • CYLINDRICAL engines (hybrid, firstwins): use the
91
- // constructor's square 5000×5000 canvas as before — they
92
- // compute paste positions in cylinder coords and don't
93
- // benefit from orientation-aware sizing.
94
- private var canvas: Mat = if (useRectilinear) {
95
- Mat()
96
- } else {
97
- Mat.zeros(canvasHeight, canvasWidth, CvType.CV_8UC3)
98
- }
99
- private var canvasMask: Mat = if (useRectilinear) {
100
- Mat()
101
- } else {
102
- Mat.zeros(canvasHeight, canvasWidth, CvType.CV_8UC1)
103
- }
104
-
105
- // V12.x state — mirrors iOS layout.
106
- private var firstRotationArkit: Mat = Mat()
107
- private var rPanToWorld: Mat = Mat()
108
- private var kCompose: Mat = Mat()
109
- private var focalCompose: Double = 0.0
110
- private val mArkitToCv: Mat = Mat(3, 3, CvType.CV_64F).apply {
111
- // diag(1, -1, -1) — ARKit/ARCore world (Y-up, -Z forward) → OpenCV.
112
- setTo(Scalar(0.0))
113
- put(0, 0, 1.0); put(1, 1, -1.0); put(2, 2, -1.0)
114
- }
115
- private var canvasOriginCylX: Int = 0
116
- private var canvasOriginCylY: Int = 0
117
- /// V12.7 first-frame anchor for rectilinear placement (canvas-pixel).
118
- private var firstFrameDstX: Int = 0
119
- private var firstFrameDstY: Int = 0
120
- /// V12.11 Step D — running max position along the pan axis.
121
- /// Mirrors iOS' _maxDstX/_maxDstY. Reverse-direction stop
122
- /// fires when the homography-corrected dst regresses below
123
- /// max - kReverseStopPx.
124
- private var maxDstX: Int = 0
125
- private var maxDstY: Int = 0
126
- /// V12.6 detected at first frame from R_panToCam.
127
- private var isLandscape: Boolean = false
128
-
129
- private var hasFirstFrame: Boolean = false
130
- private var acceptsSinceSnapshot: Int = 0
131
- /// Critic #19 fix: AtomicInteger so JS-thread reads from
132
- /// `getState`/promise resolves see a consistent value.
133
- private val acceptedCountAtomic = AtomicInteger(0)
134
- val acceptedCount: Int get() = acceptedCountAtomic.get()
135
- private var snapshotSeq: Int = 0
136
- /// Critic #30 fix: cache the painted-region bbox so we don't
137
- /// re-scan a 25M-px mask N times per accept (writeOut + width
138
- /// + height + buildState all called boundingRect separately).
139
- private var cachedBoundingRect: Rect? = null
140
- var lastState: WritableMap? = null
141
- private set
142
-
143
- /// V12.4 slit-scan + long-side clip fractions. Same values as iOS.
144
- private val kPanStripFraction: Double = 0.70
145
- private val kLongSideFraction: Double = 0.85
146
- /// V12.12 — fraction of the PAN AXIS (not the long side) the
147
- /// rectilinear engine retains per frame. Apple-pano slit-scan:
148
- /// each frame contributes a narrower-than-frame slit perpendicular
149
- /// to motion. See iOS `kPanAxisFractionRect` for the full
150
- /// rationale. Both platforms MUST stay in sync.
151
- private val kPanAxisFractionRect: Double = 0.70
152
-
153
- /// V12.12 — pan-axis canvas extent, sized at first-frame
154
- /// allocation alongside the orientation-detected perpendicular
155
- /// dim. Default 5000 (max of the constructor canvas dims) —
156
- /// kept as max-of-args for backwards compatibility with hosts
157
- /// that pass canvasWidth/Height hints.
158
- private val canvasPanExtent: Int = max(canvasWidth, canvasHeight)
159
-
160
- /**
161
- * Same shape as the V7 IncrementalEngine.addFrameAtPath() so the
162
- * RN bridge can route frames here without changing the JS contract.
163
- */
164
- fun addFrameAtPath(
165
- path: String,
166
- qx: Double,
167
- qy: Double,
168
- qz: Double,
169
- qw: Double,
170
- fx: Double,
171
- fy: Double,
172
- cx: Double,
173
- cy: Double,
174
- imageWidth: Int,
175
- imageHeight: Int,
176
- yaw: Double,
177
- pitch: Double,
178
- fovHorizDegrees: Double,
179
- fovVertDegrees: Double,
180
- trackingPoor: Boolean,
181
- ): FrameTelemetry {
182
- val t0 = System.nanoTime()
183
- if (trackingPoor) {
184
- return FrameTelemetry(
185
- FrameOutcome.SkippedTrackingPoor, -1.0, 0, yaw, pitch,
186
- msSince(t0),
187
- isLandscape = isLandscape,
188
- )
189
- }
190
- val cleaned = path.removePrefix("file://")
191
- val srcRaw = Imgcodecs.imread(cleaned, Imgcodecs.IMREAD_COLOR)
192
- if (srcRaw.empty()) {
193
- return FrameTelemetry(
194
- FrameOutcome.RejectedAlignmentLost, -1.0, 0, yaw, pitch,
195
- msSince(t0),
196
- isLandscape = isLandscape,
197
- )
198
- }
199
- val frameBGR = downsampleToCompose(srcRaw)
200
- if (frameBGR !== srcRaw) srcRaw.release()
201
- return addFrameMat(
202
- frameBGR,
203
- qx, qy, qz, qw,
204
- fx, fy, cx, cy,
205
- imageWidth, imageHeight,
206
- yaw, pitch,
207
- fovHorizDegrees, fovVertDegrees,
208
- t0,
209
- )
210
- }
211
-
212
- /**
213
- * F8.6 — pixel-data twin of [addFrameAtPath]. Accepts the
214
- * camera frame as an NV21 byte buffer instead of a JPEG file
215
- * path; skips the JPEG decode round-trip (~30–50 ms per
216
- * accepted frame on a mid-tier device).
217
- *
218
- * Use this from the vision-camera Frame Processor path (where
219
- * we already have direct producer-thread access to YUV bytes)
220
- * and from the ARCore path (where the previous
221
- * JPEG-encode-on-every-frame in `RNSARCameraView` was a
222
- * measurable hot-spot).
223
- *
224
- * The body matches [addFrameAtPath] one-for-one except for the
225
- * Mat construction: an `Mat(h*3/2, w, CV_8UC1)` wraps the
226
- * NV21 bytes, then `Imgproc.cvtColor` produces the BGR Mat the
227
- * engine pipeline already expects. Everything downstream is
228
- * the shared [addFrameMat] helper.
229
- *
230
- * `nv21Width`/`nv21Height` describe the buffer's actual
231
- * dimensions. `imageWidth`/`imageHeight` describe the
232
- * camera's reported sensor dims used for intrinsics scaling —
233
- * these can differ when the camera is downsampling for Frame
234
- * Processor output.
235
- */
236
- fun addFramePixelData(
237
- nv21: ByteArray,
238
- nv21Width: Int,
239
- nv21Height: Int,
240
- qx: Double, qy: Double, qz: Double, qw: Double,
241
- fx: Double, fy: Double, cx: Double, cy: Double,
242
- imageWidth: Int, imageHeight: Int,
243
- yaw: Double, pitch: Double,
244
- fovHorizDegrees: Double, fovVertDegrees: Double,
245
- trackingPoor: Boolean,
246
- ): FrameTelemetry {
247
- val t0 = System.nanoTime()
248
- if (trackingPoor) {
249
- return FrameTelemetry(
250
- FrameOutcome.SkippedTrackingPoor, -1.0, 0, yaw, pitch,
251
- msSince(t0),
252
- isLandscape = isLandscape,
253
- )
254
- }
255
- // NV21 layout: Y plane (w*h bytes) + interleaved VU
256
- // (w*h/2 bytes). A single CV_8UC1 Mat of height h*3/2
257
- // packs the whole thing; cvtColor with COLOR_YUV2BGR_NV21
258
- // does the planar-aware decode in one call.
259
- //
260
- // F8.6 IS-1 — length guard. If the caller supplied a
261
- // short buffer, `yuv.put(0,0,nv21)` would copy only
262
- // `nv21.size` bytes and leave the rest zero-init; cvtColor
263
- // would then read stale/zero UV and produce silently
264
- // corrupt colour. Fail fast instead.
265
- val expectedBytes = nv21Width * nv21Height * 3 / 2
266
- require(nv21.size >= expectedBytes) {
267
- "addFramePixelData: nv21 buffer too small " +
268
- "(${nv21.size} bytes < $expectedBytes for " +
269
- "${nv21Width}x${nv21Height})"
270
- }
271
- val yuv = Mat(nv21Height + nv21Height / 2, nv21Width, CvType.CV_8UC1)
272
- yuv.put(0, 0, nv21)
273
- val srcRaw = Mat()
274
- Imgproc.cvtColor(yuv, srcRaw, Imgproc.COLOR_YUV2BGR_NV21)
275
- yuv.release()
276
- if (srcRaw.empty()) {
277
- return FrameTelemetry(
278
- FrameOutcome.RejectedAlignmentLost, -1.0, 0, yaw, pitch,
279
- msSince(t0),
280
- isLandscape = isLandscape,
281
- )
282
- }
283
- val frameBGR = downsampleToCompose(srcRaw)
284
- if (frameBGR !== srcRaw) srcRaw.release()
285
- return addFrameMat(
286
- frameBGR,
287
- qx, qy, qz, qw,
288
- fx, fy, cx, cy,
289
- imageWidth, imageHeight,
290
- yaw, pitch,
291
- fovHorizDegrees, fovVertDegrees,
292
- t0,
293
- )
294
- }
295
-
296
- /**
297
- * F8.6 — the body extracted from [addFrameAtPath]. Takes a
298
- * BGR `Mat` (downsampled to compose dims) and runs the full
299
- * engine pipeline: first-frame init or subsequent-frame paste
300
- * via either rectilinear or cylindrical warp.
301
- *
302
- * Behaviour is identical to the pre-F8.6 `addFrameAtPath`
303
- * (the body is a verbatim move). Both `addFrameAtPath` and
304
- * `addFramePixelData` delegate here after their respective
305
- * Mat constructions.
306
- */
307
- private fun addFrameMat(
308
- frameBGR: Mat,
309
- qx: Double, qy: Double, qz: Double, qw: Double,
310
- fx: Double, fy: Double, cx: Double, cy: Double,
311
- imageWidth: Int, imageHeight: Int,
312
- yaw: Double, pitch: Double,
313
- // FOV params kept for symmetry with `IncrementalEngine.addFrameMat`
314
- // (the hybrid engine, which uses them in `computeOverlapPct`).
315
- // The firstwins/slit-scan engine here doesn't consume them —
316
- // its paste decision is driven by pose-projected pixel
317
- // displacement, not FoV-overlap percent.
318
- @Suppress("UNUSED_PARAMETER") fovHorizDegrees: Double,
319
- @Suppress("UNUSED_PARAMETER") fovVertDegrees: Double,
320
- t0: Long,
321
- ): FrameTelemetry {
322
- val rNew = quaternionToRotationMat(qx, qy, qz, qw)
323
-
324
- if (!hasFirstFrame) {
325
- // V11/V12 first-frame setup: build panorama frame, detect
326
- // orientation, paste/warp the first frame.
327
- firstRotationArkit = rNew.clone()
328
- val sx = frameBGR.cols().toDouble() / max(1, imageWidth)
329
- val sy = frameBGR.rows().toDouble() / max(1, imageHeight)
330
- kCompose = Mat(3, 3, CvType.CV_64F).apply {
331
- setTo(Scalar(0.0))
332
- put(0, 0, fx * sx); put(0, 2, cx * sx)
333
- put(1, 1, fy * sy); put(1, 2, cy * sy)
334
- put(2, 2, 1.0)
335
- }
336
- focalCompose = sqrt(fx * sx * fy * sy)
337
-
338
- // Build R_panToWorld from horizontal projection of camera-forward.
339
- val fwdArkitCam = Mat(3, 1, CvType.CV_64F).apply {
340
- put(0, 0, 0.0); put(1, 0, 0.0); put(2, 0, -1.0)
341
- }
342
- val fwdWorld = Mat()
343
- Core.gemm(firstRotationArkit, fwdArkitCam, 1.0, Mat(), 0.0, fwdWorld)
344
- val fwx = fwdWorld[0, 0][0]
345
- val fwz = fwdWorld[2, 0][0]
346
- val horiz = sqrt(fwx * fwx + fwz * fwz)
347
- if (horiz < 0.1) {
348
- // V11 Gap #3 — refuse first-frame init while looking near vertical.
349
- frameBGR.release()
350
- return FrameTelemetry(
351
- FrameOutcome.RejectedAlignmentLost, -1.0, 0, yaw, pitch,
352
- msSince(t0),
353
- isLandscape = isLandscape,
354
- )
355
- }
356
- val pzx = fwx / horiz
357
- val pzz = fwz / horiz
358
- rPanToWorld = Mat(3, 3, CvType.CV_64F).apply {
359
- put(0, 0, pzz); put(0, 1, 0.0); put(0, 2, pzx)
360
- put(1, 0, 0.0); put(1, 1, 1.0); put(1, 2, 0.0)
361
- put(2, 0, -pzx); put(2, 1, 0.0); put(2, 2, pzz)
362
- }
363
-
364
- // V12.6 orientation detection from R_panToCam.
365
- val rPanToCamFirst = computeRPanToCam(firstRotationArkit)
366
- val absR01 = Math.abs(rPanToCamFirst[0, 1][0])
367
- val absR11 = Math.abs(rPanToCamFirst[1, 1][0])
368
- isLandscape = absR11 > absR01
369
- Log.i(
370
- "V12.6-orient",
371
- "engine=android-firstwins detected isLandscape=$isLandscape " +
372
- "|R[0,1]|=${"%.4f".format(absR01)} " +
373
- "|R[1,1]|=${"%.4f".format(absR11)}",
374
- )
375
-
376
- if (useRectilinear) {
377
- // V12.12 — orientation-aware clip + engine-internal
378
- // canvas allocation. Pan axis = canvas Y for
379
- // landscape (vertical pan), canvas X for portrait
380
- // (horizontal pan). Clip is ALONG the pan axis.
381
- // Mirrors iOS' OpenCVSlitScanStitcher.mm exactly.
382
- val clipW: Int
383
- val clipH: Int
384
- val srcClipX: Int
385
- val srcClipY: Int
386
- if (isLandscape) {
387
- clipW = frameBGR.cols()
388
- clipH = max(1, (frameBGR.rows() * kPanAxisFractionRect).toInt())
389
- srcClipX = 0
390
- srcClipY = (frameBGR.rows() - clipH) / 2
391
- } else {
392
- clipW = max(1, (frameBGR.cols() * kPanAxisFractionRect).toInt())
393
- clipH = frameBGR.rows()
394
- srcClipX = (frameBGR.cols() - clipW) / 2
395
- srcClipY = 0
396
- }
397
- val frameClipped = Mat(frameBGR, Rect(srcClipX, srcClipY, clipW, clipH))
398
-
399
- // V12.12 — engine-internal canvas allocation, sized
400
- // from detected orientation + actual frame dims.
401
- if (canvas.empty()) {
402
- val newCanvasCols: Int
403
- val newCanvasRows: Int
404
- if (isLandscape) {
405
- newCanvasCols = frameBGR.cols() // perp full
406
- newCanvasRows = canvasPanExtent // pan extent
407
- } else {
408
- newCanvasCols = canvasPanExtent // pan extent
409
- newCanvasRows = frameBGR.rows() // perp full
410
- }
411
- canvas = Mat.zeros(newCanvasRows, newCanvasCols, CvType.CV_8UC3)
412
- canvasMask = Mat.zeros(newCanvasRows, newCanvasCols, CvType.CV_8UC1)
413
- Log.i(
414
- "V12.12-canvas",
415
- "allocated ${newCanvasCols}x${newCanvasRows} (cols x rows) for " +
416
- "isLandscape=$isLandscape (pan extent $canvasPanExtent, " +
417
- "frame=${frameBGR.cols()}x${frameBGR.rows()})",
418
- )
419
- }
420
-
421
- // V12.12 — first-frame placement at canvas ORIGIN
422
- // (0,0). Canvas perpendicular dim now matches
423
- // clipped frame perpendicular dim, so no centring
424
- // offset. As user pans, dstX (portrait) or dstY
425
- // (landscape) advances from 0.
426
- val dstX = 0
427
- val dstY = 0
428
- val roi = Rect(dstX, dstY, clipW, clipH).intersection(
429
- Rect(0, 0, canvas.cols(), canvas.rows())
430
- )
431
- val srcR = Rect(0, 0, roi.width, roi.height)
432
- Mat(frameClipped, srcR).copyTo(Mat(canvas, roi))
433
- Imgproc.rectangle(
434
- canvasMask,
435
- Point(roi.x.toDouble(), roi.y.toDouble()),
436
- Point((roi.x + roi.width).toDouble(), (roi.y + roi.height).toDouble()),
437
- Scalar(255.0), -1
438
- )
439
- firstFrameDstX = dstX
440
- firstFrameDstY = dstY
441
- // V12.11 Step D — initialise running-max trackers
442
- // to first-frame position.
443
- maxDstX = dstX
444
- maxDstY = dstY
445
- hasFirstFrame = true
446
- acceptedCountAtomic.set(1); cachedBoundingRect = null
447
- Log.i(
448
- "V12.12-rect",
449
- "first frame placed at ($dstX,$dstY) clipped=${clipW}x${clipH} " +
450
- "(srcClip=$srcClipX,$srcClipY) along-pan-axis " +
451
- "isLandscape=$isLandscape focal=${"%.2f".format(focalCompose)} " +
452
- "canvas=${canvas.cols()}x${canvas.rows()}",
453
- )
454
- frameClipped.release()
455
- frameBGR.release()
456
- return FrameTelemetry(
457
- FrameOutcome.AcceptedHigh, 1.0, 0, yaw, pitch, msSince(t0),
458
- isLandscape = isLandscape,
459
- )
460
- }
461
-
462
- // V12.2 cylindrical first frame: warp + place at canvas centre.
463
- val warped = Mat()
464
- val warpedMask = Mat()
465
- val firstCornerCyl = cylindricalWarp(frameBGR, rNew, warped, warpedMask)
466
- if (warped.empty()) {
467
- frameBGR.release()
468
- return FrameTelemetry(
469
- FrameOutcome.RejectedAlignmentLost, -1.0, 0, yaw, pitch,
470
- msSince(t0),
471
- isLandscape = isLandscape,
472
- )
473
- }
474
- val dstX = (canvasWidth - warped.cols()) / 2
475
- val dstY = (canvasHeight - warped.rows()) / 2
476
- val roi = Rect(dstX, dstY, warped.cols(), warped.rows()).intersection(
477
- Rect(0, 0, canvasWidth, canvasHeight)
478
- )
479
- val srcR = Rect(0, 0, roi.width, roi.height)
480
- Mat(warped, srcR).copyTo(Mat(canvas, roi), Mat(warpedMask, srcR))
481
- Mat(warpedMask, srcR).copyTo(Mat(canvasMask, roi), Mat(warpedMask, srcR))
482
- canvasOriginCylX = firstCornerCyl.x.toInt() - dstX
483
- canvasOriginCylY = firstCornerCyl.y.toInt() - dstY
484
-
485
- warped.release(); warpedMask.release()
486
- hasFirstFrame = true
487
- acceptedCountAtomic.set(1); cachedBoundingRect = null
488
- frameBGR.release()
489
- return FrameTelemetry(
490
- FrameOutcome.AcceptedHigh, 1.0, 0, yaw, pitch, msSince(t0),
491
- isLandscape = isLandscape,
492
- )
493
- }
494
-
495
- // ─── Subsequent frame ───────────────────────────────────────
496
- if (useRectilinear) {
497
- // V12.8 Variant B: paste the SAME long-side-clipped portion
498
- // as the first frame at canvas offset = pan_angle * focal.
499
- // First-painted-wins masking ensures only the leading-edge
500
- // sliver (the part outside the previously-painted region)
501
- // gets painted — smooth incremental growth from frame 2,
502
- // no V12.7 dead-zone.
503
- val rRel = Mat()
504
- val firstT = firstRotationArkit.t()
505
- try {
506
- Core.gemm(firstT, rNew, 1.0, Mat(), 0.0, rRel)
507
- } finally {
508
- firstT.release()
509
- }
510
-
511
- // V12.12 — orientation-aware clip, MUST match the
512
- // first-frame branch above so subsequent frames have the
513
- // same shape as the first frame (otherwise paste positions
514
- // get inconsistent).
515
- val clipW: Int
516
- val clipH: Int
517
- val srcClipX: Int
518
- val srcClipY: Int
519
- if (isLandscape) {
520
- clipW = frameBGR.cols()
521
- clipH = max(1, (frameBGR.rows() * kPanAxisFractionRect).toInt())
522
- srcClipX = 0
523
- srcClipY = (frameBGR.rows() - clipH) / 2
524
- } else {
525
- clipW = max(1, (frameBGR.cols() * kPanAxisFractionRect).toInt())
526
- clipH = frameBGR.rows()
527
- srcClipX = (frameBGR.cols() - clipW) / 2
528
- srcClipY = 0
529
- }
530
- val frameClipped = Mat(frameBGR, Rect(srcClipX, srcClipY, clipW, clipH))
531
-
532
- var dstX: Int
533
- var dstY: Int
534
- val alpha: Double
535
- if (isLandscape) {
536
- // Vertical pan around cam +X: alpha = atan2(R_rel[2,1], R_rel[1,1]).
537
- alpha = atan2(rRel[2, 1][0], rRel[1, 1][0])
538
- dstX = firstFrameDstX
539
- // alpha > 0 (look up) → content shifts UP in canvas.
540
- dstY = firstFrameDstY - round(alpha * focalCompose).toInt()
541
- } else {
542
- // Horizontal pan around cam +Y: alpha = atan2(R_rel[0,2], R_rel[0,0]).
543
- alpha = atan2(rRel[0, 2][0], rRel[0, 0][0])
544
- // alpha > 0 (look right) → content shifts RIGHT in canvas.
545
- dstX = firstFrameDstX + round(alpha * focalCompose).toInt()
546
- dstY = firstFrameDstY
547
- }
548
- rRel.release()
549
-
550
- // V13.0a — REVERTED V12.11 Step 4 + V12.11.1 Item E + V12.14
551
- // homography refinement (mirrors iOS revert). See
552
- // OpenCVSlitScanStitcher.mm for the rationale. Restored
553
- // pose-only paste; perpendicular drift correction will
554
- // come back in V13.0b as 1D column-edge NCC correlation.
555
-
556
- // V12.11 Step D — reverse-direction detection. Mirrors
557
- // iOS' running-max check. See OpenCVSlitScanStitcher.mm
558
- // for the full rationale. 150 px ≈ 4° of pan at the
559
- // typical iPhone focal — comfortably above wobble.
560
- val kReverseStopPx = 150
561
- if (isLandscape) {
562
- if (dstY > maxDstY) {
563
- maxDstY = dstY
564
- } else if (dstY < maxDstY - kReverseStopPx) {
565
- Log.i(
566
- "V12.11-reverse",
567
- "landscape stop: dstY=$dstY max=$maxDstY (regressed ${maxDstY - dstY} px)",
568
- )
569
- frameClipped.release()
570
- frameBGR.release()
571
- return FrameTelemetry(
572
- FrameOutcome.RejectedReverseDirection, -1.0, 0, yaw, pitch,
573
- msSince(t0),
574
- isLandscape = isLandscape,
575
- )
576
- }
577
- } else {
578
- if (dstX > maxDstX) {
579
- maxDstX = dstX
580
- } else if (dstX < maxDstX - kReverseStopPx) {
581
- Log.i(
582
- "V12.11-reverse",
583
- "portrait stop: dstX=$dstX max=$maxDstX (regressed ${maxDstX - dstX} px)",
584
- )
585
- frameClipped.release()
586
- frameBGR.release()
587
- return FrameTelemetry(
588
- FrameOutcome.RejectedReverseDirection, -1.0, 0, yaw, pitch,
589
- msSince(t0),
590
- isLandscape = isLandscape,
591
- )
592
- }
593
- }
594
-
595
- // V13.0a — pose-only paste (the ONLY paste path now).
596
- // V12.11.1's full warpPerspective paste was removed; this
597
- // pose-projected (dstX, dstY) is the final paste position.
598
- val dstRoi = Rect(dstX, dstY, clipW, clipH).intersection(
599
- Rect(0, 0, canvas.cols(), canvas.rows())
600
- )
601
- if (dstRoi.width <= 0 || dstRoi.height <= 0) {
602
- frameClipped.release()
603
- frameBGR.release()
604
- return FrameTelemetry(
605
- FrameOutcome.RejectedAlignmentLost, -1.0, 0, yaw, pitch,
606
- msSince(t0),
607
- isLandscape = isLandscape,
608
- )
609
- }
610
- val srcRoi = Rect(
611
- dstRoi.x - dstX,
612
- dstRoi.y - dstY,
613
- dstRoi.width, dstRoi.height,
614
- )
615
- val srcRegion = Mat(frameClipped, srcRoi)
616
- val canvasRoi = Mat(canvas, dstRoi)
617
- val maskRoi = Mat(canvasMask, dstRoi)
618
- val emptyMask = Mat()
619
- Core.compare(maskRoi, Scalar(0.0), emptyMask, Core.CMP_EQ)
620
- val newPixels = Core.countNonZero(emptyMask)
621
- if (newPixels > 0) {
622
- srcRegion.copyTo(canvasRoi, emptyMask)
623
- maskRoi.setTo(Scalar(255.0), emptyMask)
624
- acceptedCountAtomic.incrementAndGet(); cachedBoundingRect = null
625
- srcRegion.release(); canvasRoi.release(); maskRoi.release()
626
- emptyMask.release(); frameClipped.release()
627
- frameBGR.release()
628
- return FrameTelemetry(
629
- FrameOutcome.AcceptedHigh, 1.0, 0, yaw, pitch, msSince(t0),
630
- isLandscape = isLandscape,
631
- )
632
- }
633
- srcRegion.release(); canvasRoi.release(); maskRoi.release()
634
- emptyMask.release(); frameClipped.release()
635
- frameBGR.release()
636
- return FrameTelemetry(
637
- FrameOutcome.SkippedTooClose, 0.0, 0, yaw, pitch, msSince(t0),
638
- isLandscape = isLandscape,
639
- )
640
- }
641
-
642
- // V12.4 firstwins (cylindrical + central crop + first-painted-wins).
643
- val warped = Mat()
644
- val warpedMask = Mat()
645
- val newCornerCyl = cylindricalWarp(frameBGR, rNew, warped, warpedMask)
646
- if (warped.empty()) {
647
- frameBGR.release()
648
- return FrameTelemetry(
649
- FrameOutcome.RejectedAlignmentLost, -1.0, 0, yaw, pitch, msSince(t0),
650
- isLandscape = isLandscape,
651
- )
652
- }
653
- val newCornerCanvas = Point(
654
- (newCornerCyl.x - canvasOriginCylX).toDouble(),
655
- (newCornerCyl.y - canvasOriginCylY).toDouble(),
656
- )
657
- val dstRoi = Rect(
658
- newCornerCanvas.x.toInt(), newCornerCanvas.y.toInt(),
659
- warped.cols(), warped.rows(),
660
- ).intersection(Rect(0, 0, canvas.cols(), canvas.rows()))
661
- if (dstRoi.width <= 0 || dstRoi.height <= 0) {
662
- warped.release(); warpedMask.release(); frameBGR.release()
663
- return FrameTelemetry(
664
- FrameOutcome.RejectedAlignmentLost, -1.0, 0, yaw, pitch, msSince(t0),
665
- isLandscape = isLandscape,
666
- )
667
- }
668
- val srcRoi = Rect(
669
- dstRoi.x - newCornerCanvas.x.toInt(),
670
- dstRoi.y - newCornerCanvas.y.toInt(),
671
- dstRoi.width, dstRoi.height,
672
- )
673
- val warpedClipped = Mat(warped, srcRoi)
674
- val warpedMaskClipped = Mat(warpedMask, srcRoi)
675
- val canvasRoi = Mat(canvas, dstRoi)
676
- val canvasMaskRoi = Mat(canvasMask, dstRoi)
677
-
678
- // First-painted-wins: paint where canvasMask == 0 AND warped mask == 255.
679
- val noPrior = Mat()
680
- Core.compare(canvasMaskRoi, Scalar(0.0), noPrior, Core.CMP_EQ)
681
- val paintMask = Mat()
682
- Core.bitwise_and(noPrior, warpedMaskClipped, paintMask)
683
- val newPixels = Core.countNonZero(paintMask)
684
- if (newPixels > 0) {
685
- warpedClipped.copyTo(canvasRoi, paintMask)
686
- paintMask.copyTo(canvasMaskRoi, paintMask)
687
- acceptedCountAtomic.incrementAndGet(); cachedBoundingRect = null
688
- }
689
- noPrior.release(); paintMask.release()
690
- warped.release(); warpedMask.release(); frameBGR.release()
691
- return FrameTelemetry(
692
- if (newPixels > 0) FrameOutcome.AcceptedHigh else FrameOutcome.SkippedTooClose,
693
- if (newPixels > 0) 1.0 else 0.0, 0, yaw, pitch, msSince(t0),
694
- isLandscape = isLandscape,
695
- )
696
- }
697
-
698
- /**
699
- * Live-snapshot path — same JPEG-cycle pattern as iOS. Cycles
700
- * through 4 filenames so RN's <Image> cache sees a fresh URI
701
- * each accept.
702
- *
703
- * Critic #8 / #9 fix: ALWAYS build state and pass `telemetry` so
704
- * JS sees outcome / confidence / overlapPercent / processingMs
705
- * even on rejected/skipped frames. Matches the JS
706
- * IncrementalState contract in src/stitching/incremental.ts.
707
- */
708
- fun snapshotIfDue(telemetry: FrameTelemetry): WritableMap? {
709
- val isAccept = telemetry.outcome == FrameOutcome.AcceptedHigh ||
710
- telemetry.outcome == FrameOutcome.AcceptedMedium
711
- var snapshotPath: String? = null
712
- if (isAccept) {
713
- acceptsSinceSnapshot += 1
714
- if (acceptsSinceSnapshot >= snapshotEveryNAccepts) {
715
- acceptsSinceSnapshot = 0
716
- val path = currentSnapshotPath()
717
- if (writeOut(path, snapshotJpegQuality, applyExposureComp = false)) {
718
- snapshotPath = path
719
- }
720
- }
721
- }
722
- lastState = buildState(snapshotPath = snapshotPath, telemetry = telemetry)
723
- return lastState
724
- }
725
-
726
- fun finalize(outputPath: String, quality: Int): StitcherSnapshot? {
727
- val cleaned = outputPath.removePrefix("file://")
728
- val ok = writeOut(cleaned, quality, applyExposureComp = true)
729
- if (!ok) return null
730
- val bbox = cachedBoundingRect ?: Imgproc.boundingRect(canvasMask).also { cachedBoundingRect = it }
731
- val snap = StitcherSnapshot(
732
- cleaned,
733
- if (bbox.width > 0) bbox.width else canvasWidth,
734
- if (bbox.height > 0) bbox.height else canvasHeight,
735
- acceptedCount,
736
- )
737
- reset()
738
- return snap
739
- }
740
-
741
- /**
742
- * Critic #22 fix: explicit native-buffer release (75 MB canvas
743
- * + 25 MB mask + smaller transient Mats). Call from the bridge
744
- * when the engine is being thrown away (finalize/cancel paths).
745
- * After this, the engine is unusable.
746
- */
747
- fun release() {
748
- canvas.release()
749
- canvasMask.release()
750
- firstRotationArkit.release()
751
- rPanToWorld.release()
752
- kCompose.release()
753
- mArkitToCv.release()
754
- cachedBoundingRect = null
755
- }
756
-
757
- fun reset() {
758
- canvas.setTo(Scalar(0.0, 0.0, 0.0))
759
- canvasMask.setTo(Scalar(0.0))
760
- firstRotationArkit = Mat()
761
- rPanToWorld = Mat()
762
- kCompose = Mat()
763
- focalCompose = 0.0
764
- canvasOriginCylX = 0
765
- canvasOriginCylY = 0
766
- firstFrameDstX = 0
767
- firstFrameDstY = 0
768
- // V12.11 Step D — clear running-max trackers; reinitialised
769
- // to first-frame position on next accept.
770
- maxDstX = 0
771
- maxDstY = 0
772
- isLandscape = false
773
- hasFirstFrame = false
774
- acceptsSinceSnapshot = 0
775
- acceptedCountAtomic.set(0)
776
- snapshotSeq = 0
777
- lastState = null
778
- }
779
-
780
- // ─── Internals ─────────────────────────────────────────────────────
781
-
782
- private fun computeRPanToCam(rArkit: Mat): Mat {
783
- // R_panToCam = M · R_arkit^T · R_panToWorld
784
- val tmp1 = Mat()
785
- Core.gemm(rArkit.t(), rPanToWorld, 1.0, Mat(), 0.0, tmp1)
786
- val out = Mat()
787
- Core.gemm(mArkitToCv, tmp1, 1.0, Mat(), 0.0, out)
788
- tmp1.release()
789
- return out
790
- }
791
-
792
- /**
793
- * V12.6 cylindrical warp — full annotation in iOS' OpenCVSlitScanStitcher.mm.
794
- * Returns the bbox top-left in cylindrical-pixel coords. Writes the
795
- * warped frame into outImage and the corresponding mask into outMask.
796
- */
797
- private fun cylindricalWarp(src: Mat, rArkit: Mat, outImage: Mat, outMask: Mat): Point {
798
- if (rPanToWorld.empty() || focalCompose <= 0) {
799
- outImage.release(); outMask.release()
800
- return Point(0.0, 0.0)
801
- }
802
- val rPanToCam = computeRPanToCam(rArkit)
803
- val rCamToPan = rPanToCam.t()
804
-
805
- val fx = kCompose[0, 0][0]
806
- val fy = kCompose[1, 1][0]
807
- val cx = kCompose[0, 2][0]
808
- val cy = kCompose[1, 2][0]
809
- val f = focalCompose
810
-
811
- // Forward-project the 4 source corners.
812
- val r00 = rPanToCam[0, 0][0]; val r01 = rPanToCam[0, 1][0]; val r02 = rPanToCam[0, 2][0]
813
- val r10 = rPanToCam[1, 0][0]; val r11 = rPanToCam[1, 1][0]; val r12 = rPanToCam[1, 2][0]
814
- val r20 = rPanToCam[2, 0][0]; val r21 = rPanToCam[2, 1][0]; val r22 = rPanToCam[2, 2][0]
815
- val cTP00 = rCamToPan[0, 0][0]; val cTP01 = rCamToPan[0, 1][0]; val cTP02 = rCamToPan[0, 2][0]
816
- val cTP10 = rCamToPan[1, 0][0]; val cTP11 = rCamToPan[1, 1][0]; val cTP12 = rCamToPan[1, 2][0]
817
- val cTP20 = rCamToPan[2, 0][0]; val cTP21 = rCamToPan[2, 1][0]; val cTP22 = rCamToPan[2, 2][0]
818
-
819
- fun project(u: Double, v: Double): DoubleArray {
820
- val rx = (u - cx) / fx
821
- val ry = (v - cy) / fy
822
- val rz = 1.0
823
- val wx = cTP00 * rx + cTP01 * ry + cTP02 * rz
824
- val wy = cTP10 * rx + cTP11 * ry + cTP12 * rz
825
- val wz = cTP20 * rx + cTP21 * ry + cTP22 * rz
826
- return if (isLandscape) {
827
- // Transverse cylinder (axis = pan_X)
828
- val denom = sqrt(wy * wy + wz * wz)
829
- val s = if (denom > 1e-9) (-wx / denom) else 0.0
830
- val theta = atan2(wy, wz)
831
- doubleArrayOf(f * s, -f * theta)
832
- } else {
833
- // Vertical cylinder (axis = pan_Y), V12 mirror fix.
834
- val theta = atan2(-wx, wz)
835
- val denom = sqrt(wx * wx + wz * wz)
836
- val h = if (denom > 1e-9) (wy / denom) else 0.0
837
- doubleArrayOf(f * theta, -f * h)
838
- }
839
- }
840
-
841
- val c00 = project(0.0, 0.0)
842
- val c10 = project((src.cols() - 1).toDouble(), 0.0)
843
- val c01 = project(0.0, (src.rows() - 1).toDouble())
844
- val c11 = project((src.cols() - 1).toDouble(), (src.rows() - 1).toDouble())
845
- val xs = doubleArrayOf(c00[0], c10[0], c01[0], c11[0])
846
- val ys = doubleArrayOf(c00[1], c10[1], c01[1], c11[1])
847
- val minX = xs.min(); val maxX = xs.max()
848
- val minY = ys.min(); val maxY = ys.max()
849
-
850
- var bboxX = floor(minX).toInt()
851
- var bboxY = floor(minY).toInt()
852
- var bboxW = (Math.ceil(maxX - minX).toInt()) + 1
853
- var bboxH = (Math.ceil(maxY - minY).toInt()) + 1
854
- if (bboxW <= 0 || bboxH <= 0 ||
855
- bboxW > canvasWidth * 2 || bboxH > canvasHeight * 2) {
856
- outImage.release(); outMask.release()
857
- rPanToCam.release(); rCamToPan.release()
858
- return Point(0.0, 0.0)
859
- }
860
-
861
- // V12.4 slit-scan + long-side clip.
862
- run {
863
- val newW: Int
864
- val newH: Int
865
- if (isLandscape) {
866
- newW = max(1, (bboxW * kLongSideFraction).toInt())
867
- newH = max(1, (bboxH * kPanStripFraction).toInt())
868
- } else {
869
- newW = max(1, (bboxW * kPanStripFraction).toInt())
870
- newH = max(1, (bboxH * kLongSideFraction).toInt())
871
- }
872
- bboxX += (bboxW - newW) / 2
873
- bboxY += (bboxH - newH) / 2
874
- bboxW = newW
875
- bboxH = newH
876
- }
877
-
878
- // Inverse-map: build mapX/mapY for cv::remap, then warp.
879
- val mapX = Mat(bboxH, bboxW, CvType.CV_32FC1)
880
- val mapY = Mat(bboxH, bboxW, CvType.CV_32FC1)
881
-
882
- val rowMx = FloatArray(bboxW)
883
- val rowMy = FloatArray(bboxW)
884
- val srcCols = src.cols(); val srcRows = src.rows()
885
-
886
- if (isLandscape) {
887
- for (y in 0 until bboxH) {
888
- val sphereY = (bboxY + y).toDouble()
889
- val theta = -sphereY / f
890
- val sinT = sin(theta); val cosT = cos(theta)
891
- for (x in 0 until bboxW) {
892
- val sphereX = (bboxX + x).toDouble()
893
- val s = sphereX / f
894
- val wx = -s
895
- val wy = sinT
896
- val wz = cosT
897
- val rx = r00 * wx + r01 * wy + r02 * wz
898
- val ry = r10 * wx + r11 * wy + r12 * wz
899
- val rz = r20 * wx + r21 * wy + r22 * wz
900
- if (rz <= 1e-6) {
901
- rowMx[x] = -1.0f; rowMy[x] = -1.0f
902
- } else {
903
- val u = fx * rx / rz + cx
904
- val v = fy * ry / rz + cy
905
- if (u < 0 || u >= srcCols || v < 0 || v >= srcRows) {
906
- rowMx[x] = -1.0f; rowMy[x] = -1.0f
907
- } else {
908
- rowMx[x] = u.toFloat(); rowMy[x] = v.toFloat()
909
- }
910
- }
911
- }
912
- mapX.put(y, 0, rowMx)
913
- mapY.put(y, 0, rowMy)
914
- }
915
- } else {
916
- for (y in 0 until bboxH) {
917
- val cylY = (bboxY + y).toDouble()
918
- val h = -cylY / f
919
- for (x in 0 until bboxW) {
920
- val cylX = (bboxX + x).toDouble()
921
- val theta = cylX / f
922
- val sinT = sin(theta); val cosT = cos(theta)
923
- val wx = -sinT; val wy = h; val wz = cosT
924
- val rx = r00 * wx + r01 * wy + r02 * wz
925
- val ry = r10 * wx + r11 * wy + r12 * wz
926
- val rz = r20 * wx + r21 * wy + r22 * wz
927
- if (rz <= 1e-6) {
928
- rowMx[x] = -1.0f; rowMy[x] = -1.0f
929
- } else {
930
- val u = fx * rx / rz + cx
931
- val v = fy * ry / rz + cy
932
- if (u < 0 || u >= srcCols || v < 0 || v >= srcRows) {
933
- rowMx[x] = -1.0f; rowMy[x] = -1.0f
934
- } else {
935
- rowMx[x] = u.toFloat(); rowMy[x] = v.toFloat()
936
- }
937
- }
938
- }
939
- mapX.put(y, 0, rowMx)
940
- mapY.put(y, 0, rowMy)
941
- }
942
- }
943
-
944
- outImage.create(bboxH, bboxW, src.type())
945
- Imgproc.remap(
946
- src, outImage, mapX, mapY,
947
- Imgproc.INTER_LINEAR, Core.BORDER_CONSTANT, Scalar(0.0, 0.0, 0.0),
948
- )
949
-
950
- // Build the mask in a single pass by re-reading mapX rows.
951
- outMask.create(bboxH, bboxW, CvType.CV_8UC1)
952
- outMask.setTo(Scalar(0.0))
953
- val maskRow = ByteArray(bboxW)
954
- for (y in 0 until bboxH) {
955
- mapX.get(y, 0, rowMx)
956
- for (x in 0 until bboxW) {
957
- maskRow[x] = if (rowMx[x] >= 0.0f) 255.toByte() else 0.toByte()
958
- }
959
- outMask.put(y, 0, maskRow)
960
- }
961
-
962
- mapX.release(); mapY.release()
963
- rPanToCam.release(); rCamToPan.release()
964
- return Point(bboxX.toDouble(), bboxY.toDouble())
965
- }
966
-
967
- private fun downsampleToCompose(src: Mat): Mat {
968
- val scale = min(
969
- composeWidth.toDouble() / src.cols(),
970
- composeHeight.toDouble() / src.rows(),
971
- )
972
- if (scale >= 1.0) return src
973
- val outW = max(1, round(src.cols() * scale).toInt())
974
- val outH = max(1, round(src.rows() * scale).toInt())
975
- val out = Mat()
976
- Imgproc.resize(src, out, Size(outW.toDouble(), outH.toDouble()), 0.0, 0.0, Imgproc.INTER_AREA)
977
- return out
978
- }
979
-
980
- private fun quaternionToRotationMat(qx: Double, qy: Double, qz: Double, qw: Double): Mat {
981
- val n = sqrt(qx * qx + qy * qy + qz * qz + qw * qw)
982
- val x = if (n > 1e-9) qx / n else qx
983
- val y = if (n > 1e-9) qy / n else qy
984
- val z = if (n > 1e-9) qz / n else qz
985
- val w = if (n > 1e-9) qw / n else qw
986
- return Mat(3, 3, CvType.CV_64F).apply {
987
- put(0, 0, 1 - 2 * (y * y + z * z)); put(0, 1, 2 * (x * y - w * z)); put(0, 2, 2 * (x * z + w * y))
988
- put(1, 0, 2 * (x * y + w * z)); put(1, 1, 1 - 2 * (x * x + z * z)); put(1, 2, 2 * (y * z - w * x))
989
- put(2, 0, 2 * (x * z - w * y)); put(2, 1, 2 * (y * z + w * x)); put(2, 2, 1 - 2 * (x * x + y * y))
990
- }
991
- }
992
-
993
- private fun currentSnapshotPath(): String {
994
- snapshotSeq += 1
995
- val slot = snapshotSeq % 4
996
- // Critic #27 fix: use the bridge-provided cache dir
997
- // (reactContext.cacheDir.absolutePath), NOT java.io.tmpdir
998
- // which on Android is /data/local/tmp (rooted-only).
999
- return "$snapshotCacheDir/rlis-live-$slot.jpg"
1000
- }
1001
-
1002
- private fun writeOut(path: String, quality: Int, applyExposureComp: Boolean): Boolean {
1003
- // V12 — bbox crop only, no inscribed-rect search (dropped per V12 plan).
1004
- val bbox = Imgproc.boundingRect(canvasMask)
1005
- val cropRect = if (bbox.width > 0 && bbox.height > 0) bbox
1006
- else Rect(0, 0, canvasWidth, canvasHeight)
1007
- val cropped = Mat(canvas, cropRect).clone()
1008
- val out = if (applyExposureComp) applyClahe(cropped) else cropped
1009
- val params = MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, max(1, min(100, quality)))
1010
- val ok = Imgcodecs.imwrite(path, out, params)
1011
- if (cropped !== out) cropped.release()
1012
- out.release()
1013
- return ok
1014
- }
1015
-
1016
- private fun applyClahe(src: Mat): Mat {
1017
- val lab = Mat()
1018
- Imgproc.cvtColor(src, lab, Imgproc.COLOR_BGR2Lab)
1019
- val channels = mutableListOf<Mat>()
1020
- Core.split(lab, channels)
1021
- val clahe = Imgproc.createCLAHE(2.0, Size(8.0, 8.0))
1022
- clahe.apply(channels[0], channels[0])
1023
- Core.merge(channels, lab)
1024
- val out = Mat()
1025
- Imgproc.cvtColor(lab, out, Imgproc.COLOR_Lab2BGR)
1026
- for (c in channels) c.release()
1027
- lab.release()
1028
- return out
1029
- }
1030
-
1031
- /**
1032
- * Critic #8/#9/#23 fix: always include the full state event shape
1033
- * the JS IncrementalState interface expects (outcome, confidence,
1034
- * overlapPercent, processingMs). Matches the hybrid engine's
1035
- * shape so JS subscribers don't break when the engine variant
1036
- * is toggled at runtime.
1037
- *
1038
- * Critic #30 fix: use cached bounding rect; refresh once per
1039
- * accept inside the inner loops, NOT here on every state event.
1040
- */
1041
- private fun buildState(snapshotPath: String?, telemetry: FrameTelemetry): WritableMap {
1042
- val map = com.facebook.react.bridge.Arguments.createMap()
1043
- map.putInt("acceptedCount", acceptedCount)
1044
- if (snapshotPath != null) {
1045
- map.putString("panoramaPath", snapshotPath)
1046
- } else {
1047
- map.putNull("panoramaPath")
1048
- }
1049
- val r = cachedBoundingRect ?: Imgproc.boundingRect(canvasMask).also { cachedBoundingRect = it }
1050
- map.putInt("width", if (r.width > 0) r.width else 0)
1051
- map.putInt("height", if (r.height > 0) r.height else 0)
1052
- map.putInt("outcome", telemetry.outcome.ordinal)
1053
- map.putDouble("confidence", telemetry.confidence)
1054
- map.putDouble("overlapPercent", telemetry.overlapPercent)
1055
- map.putDouble("processingMs", telemetry.processingMs)
1056
- // V12.12 — engine-detected orientation, plumbed through to JS
1057
- // for the band overlay + dim bar UI.
1058
- map.putBoolean("isLandscape", telemetry.isLandscape)
1059
- return map
1060
- }
1061
-
1062
- private fun msSince(t0Nanos: Long): Double =
1063
- (System.nanoTime() - t0Nanos) / 1_000_000.0
1064
- }
1065
-
1066
- // Helper: Rect intersection (Android OpenCV doesn't expose it cleanly).
1067
- private fun Rect.intersection(other: Rect): Rect {
1068
- val x = max(this.x, other.x)
1069
- val y = max(this.y, other.y)
1070
- val r = min(this.x + this.width, other.x + other.width)
1071
- val b = min(this.y + this.height, other.y + other.height)
1072
- return Rect(x, y, max(0, r - x), max(0, b - y))
1073
- }
1074
-
1075
- // V13.0a — homographyOffset(), HomographyResult, and the kHomogTier*
1076
- // constants were removed in the revert from V12.11.1 + V12.14
1077
- // (ORB+RANSAC homography correction with 3-tier confidence ladder)
1078
- // back to pose-only paste. See iOS OpenCVSlitScanStitcher.mm for
1079
- // the matching revert + V13.0b plan (1D column-edge NCC correlation
1080
- // for sub-pixel perpendicular drift correction).
1081
-