react-native-image-stitcher 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +316 -1
- package/README.md +1 -1
- package/android/src/main/cpp/keyframe_gate_jni.cpp +138 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +412 -40
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +128 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +87 -45
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +46 -4
- package/cpp/stitcher.cpp +101 -1
- package/cpp/stitcher.hpp +8 -0
- package/dist/camera/Camera.d.ts +9 -0
- package/dist/camera/Camera.js +118 -8
- package/dist/camera/CaptureDebugOverlay.d.ts +45 -0
- package/dist/camera/CaptureDebugOverlay.js +146 -0
- package/dist/camera/CaptureKeyframePill.d.ts +28 -0
- package/dist/camera/CaptureKeyframePill.js +60 -0
- package/dist/camera/CaptureMemoryPill.d.ts +28 -0
- package/dist/camera/CaptureMemoryPill.js +109 -0
- package/dist/camera/CaptureOrientationPill.d.ts +22 -0
- package/dist/camera/CaptureOrientationPill.js +44 -0
- package/dist/camera/CaptureStitchStatsToast.d.ts +45 -0
- package/dist/camera/CaptureStitchStatsToast.js +133 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +6 -5
- package/dist/index.d.ts +10 -0
- package/dist/index.js +15 -1
- package/dist/sensors/useIMUTranslationGate.d.ts +26 -0
- package/dist/sensors/useIMUTranslationGate.js +83 -1
- package/dist/stitching/incremental.d.ts +25 -0
- package/dist/stitching/useIncrementalStitcher.d.ts +12 -1
- package/dist/stitching/useIncrementalStitcher.js +7 -1
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +321 -7
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +8 -0
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +12 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +13 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +15 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +1 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +17 -4
- package/ios/Sources/RNImageStitcher/Stitcher.swift +6 -1
- package/package.json +1 -1
- package/src/camera/Camera.tsx +165 -7
- package/src/camera/CaptureDebugOverlay.tsx +180 -0
- package/src/camera/CaptureKeyframePill.tsx +77 -0
- package/src/camera/CaptureMemoryPill.tsx +96 -0
- package/src/camera/CaptureOrientationPill.tsx +57 -0
- package/src/camera/CaptureStitchStatsToast.tsx +155 -0
- package/src/camera/PanoramaSettingsModal.tsx +6 -5
- package/src/index.ts +19 -0
- package/src/sensors/useIMUTranslationGate.ts +112 -1
- package/src/stitching/incremental.ts +25 -0
- package/src/stitching/useIncrementalStitcher.ts +18 -0
|
@@ -62,6 +62,24 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
62
62
|
get() = nativeIsEnabled(nativeHandle)
|
|
63
63
|
set(value) = nativeSetEnabled(nativeHandle, value)
|
|
64
64
|
|
|
65
|
+
/// 2026-05-22 (audit F6) — Gate strategy. Matches the C++ enum
|
|
66
|
+
/// retailens::GateStrategy (0 = Pose, 1 = Flow). Pose strategy
|
|
67
|
+
/// uses plane-projection / angular novelty; Flow strategy uses
|
|
68
|
+
/// sparse optical-flow KLT. iOS parity: Swift facade's
|
|
69
|
+
/// `keyframeGate.strategy = .flow / .pose`. Default `Pose`
|
|
70
|
+
/// (matches C++ default). Write-only; the C++ side has a getter
|
|
71
|
+
/// but the Kotlin facade caches locally to avoid JNI round-trip.
|
|
72
|
+
enum class Strategy(val nativeValue: Int) {
|
|
73
|
+
Pose(0),
|
|
74
|
+
Flow(1);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
var strategy: Strategy = Strategy.Pose
|
|
78
|
+
set(value) {
|
|
79
|
+
field = value
|
|
80
|
+
nativeSetStrategy(nativeHandle, value.nativeValue)
|
|
81
|
+
}
|
|
82
|
+
|
|
65
83
|
/// Required new-content fraction (0…1). Default 0.4. No getter
|
|
66
84
|
/// — the C++ side has no read accessor (Swift side never needed
|
|
67
85
|
/// to read this back either). Stored locally for diagnostic
|
|
@@ -122,6 +140,47 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
122
140
|
nativeSetFlowMaxTranslationM(nativeHandle, value)
|
|
123
141
|
}
|
|
124
142
|
|
|
143
|
+
/// 2026-05-22 (audit F5) — Flow strategy: Shi-Tomasi max corners
|
|
144
|
+
/// to track per frame. Same knob iOS exposes via setFlowMaxCorners.
|
|
145
|
+
/// C++ clamps to ≥ 30. Higher = more sensitive to fine detail but
|
|
146
|
+
/// CPU-quadratic in the KLT step. Default 150 (matches iOS).
|
|
147
|
+
var flowMaxCorners: Int = 150
|
|
148
|
+
set(value) {
|
|
149
|
+
field = value
|
|
150
|
+
nativeSetFlowMaxCorners(nativeHandle, value)
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/// 2026-05-22 (audit F5) — Flow strategy: Shi-Tomasi minimum
|
|
154
|
+
/// eigenvalue threshold (0, 1]. C++ default 0.01. Lower lets
|
|
155
|
+
/// weaker corners in (more candidate points, more KLT noise);
|
|
156
|
+
/// higher demands stronger corners (fewer points, more robust).
|
|
157
|
+
var flowQualityLevel: Double = 0.01
|
|
158
|
+
set(value) {
|
|
159
|
+
field = value
|
|
160
|
+
nativeSetFlowQualityLevel(nativeHandle, value)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/// 2026-05-22 (audit F5) — Flow strategy: Shi-Tomasi minimum
|
|
164
|
+
/// distance between accepted corners, in working-resolution
|
|
165
|
+
/// pixels. C++ clamps to ≥ 1.0. Default 10.0 (matches iOS).
|
|
166
|
+
var flowMinDistance: Double = 10.0
|
|
167
|
+
set(value) {
|
|
168
|
+
field = value
|
|
169
|
+
nativeSetFlowMinDistance(nativeHandle, value)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/// 2026-05-22 (audit F5) — Eval cadence: caller-side throttle so
|
|
173
|
+
/// the Flow strategy runs every Nth frame instead of every frame.
|
|
174
|
+
/// iOS parity with `IncrementalStitcher.swift:2459-2471` —
|
|
175
|
+
/// the GATE doesn't enforce the throttle itself; it just stores
|
|
176
|
+
/// the value here. The caller (`IncrementalStitcher.kt`) reads
|
|
177
|
+
/// this and decides per-frame whether to evaluate. Default 1
|
|
178
|
+
/// (no throttle). Caller is responsible for clamping to [1, 10].
|
|
179
|
+
var flowEvalEveryNFrames: Int = 1
|
|
180
|
+
set(value) {
|
|
181
|
+
field = value.coerceAtLeast(1)
|
|
182
|
+
}
|
|
183
|
+
|
|
125
184
|
// ── Read-only state ─────────────────────────────────────────
|
|
126
185
|
|
|
127
186
|
val acceptedCount: Int get() = nativeGetAcceptedCount(nativeHandle)
|
|
@@ -175,6 +234,56 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
175
234
|
)
|
|
176
235
|
}
|
|
177
236
|
|
|
237
|
+
/**
|
|
238
|
+
* Pixel-aware evaluate. Hands the gate the frame's grayscale
|
|
239
|
+
* plane so the C++ Flow strategy (sparse optical-flow novelty)
|
|
240
|
+
* actually runs — without grayData, the gate silently falls back
|
|
241
|
+
* to the Pose strategy (angular-delta). iOS parity: see
|
|
242
|
+
* `KeyframeGateBridge.mm::evaluateWithPixelBuffer:...`.
|
|
243
|
+
*
|
|
244
|
+
* 2026-05-21 (v0.3) added. Two call-site categories:
|
|
245
|
+
*
|
|
246
|
+
* - AR mode (`RNSARCameraView.forwardToIncremental`): extracts
|
|
247
|
+
* the Y plane from the ARCore camera image (YUV_420_888) and
|
|
248
|
+
* hands it through. Zero-copy on the way in (the byte[] is
|
|
249
|
+
* pinned via GetPrimitiveArrayCritical in the JNI).
|
|
250
|
+
* - Non-AR mode (`IncrementalStitcher.processFrameAtPath`): the
|
|
251
|
+
* JS-driver path supplies a JPEG path; the caller decodes the
|
|
252
|
+
* JPEG to grayscale before calling this method.
|
|
253
|
+
*
|
|
254
|
+
* @param grayData The grayscale plane bytes. Length must be
|
|
255
|
+
* at least `grayStride * grayHeight`.
|
|
256
|
+
* @param grayWidth Image width in pixels (≤ grayStride).
|
|
257
|
+
* @param grayHeight Image height in pixels.
|
|
258
|
+
* @param grayStride Bytes per row. May exceed `grayWidth` when
|
|
259
|
+
* the plane has padding (ARCore can pad).
|
|
260
|
+
*/
|
|
261
|
+
fun evaluateWithFrame(
|
|
262
|
+
pose: RNSARFramePose,
|
|
263
|
+
latchedPlaneMatrix: FloatArray?,
|
|
264
|
+
grayData: ByteArray,
|
|
265
|
+
grayWidth: Int,
|
|
266
|
+
grayHeight: Int,
|
|
267
|
+
grayStride: Int,
|
|
268
|
+
): KeyframeGateDecision {
|
|
269
|
+
val result = nativeEvaluateWithFrame(
|
|
270
|
+
nativeHandle,
|
|
271
|
+
pose.tx.toFloat(), pose.ty.toFloat(), pose.tz.toFloat(),
|
|
272
|
+
pose.qx.toFloat(), pose.qy.toFloat(), pose.qz.toFloat(), pose.qw.toFloat(),
|
|
273
|
+
pose.fx.toFloat(), pose.fy.toFloat(), pose.cx.toFloat(), pose.cy.toFloat(),
|
|
274
|
+
pose.imageWidth, pose.imageHeight,
|
|
275
|
+
latchedPlaneMatrix,
|
|
276
|
+
grayData, grayWidth, grayHeight, grayStride,
|
|
277
|
+
)
|
|
278
|
+
return KeyframeGateDecision(
|
|
279
|
+
accept = result[0] >= 0.5,
|
|
280
|
+
reason = reasonFromCode(result[1].toInt()),
|
|
281
|
+
newContentFraction = result[2],
|
|
282
|
+
acceptedCount = result[3].toInt(),
|
|
283
|
+
maxCount = result[4].toInt(),
|
|
284
|
+
)
|
|
285
|
+
}
|
|
286
|
+
|
|
178
287
|
// ── JNI thunks ──────────────────────────────────────────────
|
|
179
288
|
|
|
180
289
|
private external fun nativeCreate(): Long
|
|
@@ -193,6 +302,15 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
193
302
|
private external fun nativeSetDisableAngularFallback(handle: Long, disabled: Boolean)
|
|
194
303
|
private external fun nativeSetFlowNoveltyPercentile(handle: Long, percentile: Double)
|
|
195
304
|
private external fun nativeSetFlowMaxTranslationM(handle: Long, metres: Double)
|
|
305
|
+
// 2026-05-22 (audit F5) — flow-strategy tunables that were
|
|
306
|
+
// previously iOS-only. Add Android JNI parity so the Settings UI
|
|
307
|
+
// sliders work on both platforms.
|
|
308
|
+
private external fun nativeSetFlowMaxCorners(handle: Long, maxCorners: Int)
|
|
309
|
+
private external fun nativeSetFlowQualityLevel(handle: Long, quality: Double)
|
|
310
|
+
private external fun nativeSetFlowMinDistance(handle: Long, minDistance: Double)
|
|
311
|
+
// 2026-05-22 (audit F6) — gate-strategy selector. Maps to C++
|
|
312
|
+
// retailens::GateStrategy (Pose=0, Flow=1).
|
|
313
|
+
private external fun nativeSetStrategy(handle: Long, strategy: Int)
|
|
196
314
|
private external fun nativeEvaluate(
|
|
197
315
|
handle: Long,
|
|
198
316
|
tx: Float, ty: Float, tz: Float,
|
|
@@ -201,6 +319,16 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
201
319
|
imageWidth: Int, imageHeight: Int,
|
|
202
320
|
plane16: FloatArray?,
|
|
203
321
|
): DoubleArray
|
|
322
|
+
private external fun nativeEvaluateWithFrame(
|
|
323
|
+
handle: Long,
|
|
324
|
+
tx: Float, ty: Float, tz: Float,
|
|
325
|
+
qx: Float, qy: Float, qz: Float, qw: Float,
|
|
326
|
+
fx: Float, fy: Float, cx: Float, cy: Float,
|
|
327
|
+
imageWidth: Int, imageHeight: Int,
|
|
328
|
+
plane16: FloatArray?,
|
|
329
|
+
grayData: ByteArray,
|
|
330
|
+
grayWidth: Int, grayHeight: Int, grayStride: Int,
|
|
331
|
+
): DoubleArray
|
|
204
332
|
|
|
205
333
|
companion object {
|
|
206
334
|
init {
|
|
@@ -196,6 +196,15 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
196
196
|
// to work for hosts that prefer explicit control — the
|
|
197
197
|
// refs/state are shared.
|
|
198
198
|
RNSARSession.instance?.stopForView()
|
|
199
|
+
// 2026-05-23 (crash fix) — drop our local Session reference
|
|
200
|
+
// too. stopForView() above pause+close'd the session and
|
|
201
|
+
// nulled the singleton's ref, but our own sessionRef still
|
|
202
|
+
// pointed at the closed Session. If the view ever got
|
|
203
|
+
// re-used (re-attach without recreating), the next
|
|
204
|
+
// session.resume() / forwardToIncremental call would
|
|
205
|
+
// dereference a closed Session → SEGV in libarcore_c.so's
|
|
206
|
+
// internal cleanup, exactly the tombstone we saw.
|
|
207
|
+
sessionRef.set(null)
|
|
199
208
|
}
|
|
200
209
|
|
|
201
210
|
/// Called by IncrementalStitcher.start/stop. When true,
|
|
@@ -420,22 +429,38 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
420
429
|
return
|
|
421
430
|
}
|
|
422
431
|
try {
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
432
|
+
// 2026-05-21 (v0.3) — pixel-data path. Pre-0.3 this code
|
|
433
|
+
// unconditionally encoded the YUV camera image to JPEG and
|
|
434
|
+
// wrote it to disk for EVERY ARCore frame at ~60 Hz (~25 ms
|
|
435
|
+
// per frame of JPEG encode + disk I/O on the GL render
|
|
436
|
+
// thread), regardless of whether the C++ KeyframeGate would
|
|
437
|
+
// accept it. Now we extract the Y plane bytes (cheap
|
|
438
|
+
// memcpy from a DirectByteBuffer), feed them to the gate
|
|
439
|
+
// for proper Flow-strategy evaluation, and defer the JPEG
|
|
440
|
+
// encode + disk write to the `onAccept` lambda so it only
|
|
441
|
+
// runs on the rare frames the gate actually keeps
|
|
442
|
+
// (typically ~6 per capture).
|
|
443
|
+
//
|
|
444
|
+
// Y-plane extraction for ARCore's YUV_420_888 format:
|
|
445
|
+
// plane[0] is the luminance channel at full resolution,
|
|
446
|
+
// pixelStride=1, rowStride may equal width OR be padded.
|
|
447
|
+
// We pass rowStride as the C++ side's `stride` so the gate
|
|
448
|
+
// skips padding correctly.
|
|
449
|
+
val yPlane = image.planes[0]
|
|
450
|
+
val yBuffer = yPlane.buffer
|
|
451
|
+
val yStride = yPlane.rowStride
|
|
452
|
+
val yWidth = image.width
|
|
453
|
+
val yHeight = image.height
|
|
454
|
+
// Copy Y bytes into a JVM-side ByteArray. Using
|
|
455
|
+
// duplicate() so we don't mutate the original buffer's
|
|
456
|
+
// position state (ARCore may have other readers).
|
|
457
|
+
// For 1920×1080 Y plane that's ~2 MB; on Galaxy A35 the
|
|
458
|
+
// memcpy itself is < 1 ms. JNI side pins via
|
|
459
|
+
// GetPrimitiveArrayCritical so the byte[] stays a single
|
|
460
|
+
// copy through the entire frame's lifecycle.
|
|
461
|
+
val ySize = yStride * yHeight
|
|
462
|
+
val yBytes = ByteArray(ySize)
|
|
463
|
+
yBuffer.duplicate().apply { rewind() }.get(yBytes, 0, ySize)
|
|
439
464
|
|
|
440
465
|
// Compute yaw + pitch from the ARCore quaternion using
|
|
441
466
|
// the same convention the iOS Swift side uses (camera-
|
|
@@ -469,8 +494,32 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
469
494
|
val tArr = camera.pose.translation
|
|
470
495
|
|
|
471
496
|
val trackingPoor = camera.trackingState != TrackingState.TRACKING
|
|
472
|
-
|
|
473
|
-
|
|
497
|
+
val module = IncrementalStitcher.bridgeInstance ?: return
|
|
498
|
+
// 2026-05-15 (B3) — pass current display rotation so the
|
|
499
|
+
// encoded JPEG gets an EXIF orientation tag. Captured into
|
|
500
|
+
// a local val so the lambda below closes over a primitive
|
|
501
|
+
// (avoids re-reading lastDisplayRotation if it shifts
|
|
502
|
+
// between gate-evaluate and lambda invocation).
|
|
503
|
+
val rotationForEncode = if (lastDisplayRotation >= 0)
|
|
504
|
+
lastDisplayRotation else android.view.Surface.ROTATION_0
|
|
505
|
+
// 2026-05-21 (v0.3) — eager JPEG encode is only needed when
|
|
506
|
+
// the engine is in the legacy hybrid/firstwins live-engine
|
|
507
|
+
// mode (which feeds JPEG paths into addFrameAtPath every
|
|
508
|
+
// frame). In batch-keyframe mode (the production Camera
|
|
509
|
+
// component's path), the JPEG is encoded LAZILY inside
|
|
510
|
+
// the onAccept lambda below — only on the ~6 frames per
|
|
511
|
+
// capture that the C++ KeyframeGate actually keeps.
|
|
512
|
+
val legacyJpegPath: String? = if (module.isBatchKeyframeMode) {
|
|
513
|
+
null
|
|
514
|
+
} else {
|
|
515
|
+
YuvImageConverter.encodeToJpeg(
|
|
516
|
+
image,
|
|
517
|
+
tmpJpegFile.absolutePath,
|
|
518
|
+
jpegQuality = 70,
|
|
519
|
+
displayRotation = rotationForEncode,
|
|
520
|
+
)
|
|
521
|
+
}
|
|
522
|
+
module.ingestFromARCameraView(
|
|
474
523
|
tx = tArr[0].toDouble(),
|
|
475
524
|
ty = tArr[1].toDouble(),
|
|
476
525
|
tz = tArr[2].toDouble(),
|
|
@@ -482,39 +531,32 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
482
531
|
yaw = yaw, pitch = pitch,
|
|
483
532
|
fovHorizDegrees = fovHDeg, fovVertDegrees = fovVDeg,
|
|
484
533
|
trackingPoor = trackingPoor,
|
|
534
|
+
grayData = yBytes,
|
|
535
|
+
grayWidth = yWidth,
|
|
536
|
+
grayHeight = yHeight,
|
|
537
|
+
grayStride = yStride,
|
|
538
|
+
legacyJpegPath = legacyJpegPath,
|
|
539
|
+
onAccept = { targetPath ->
|
|
540
|
+
// Lazy JPEG encode. Runs ONLY if the C++ KeyframeGate
|
|
541
|
+
// accepted the frame. The ARCore Image is still open
|
|
542
|
+
// at this point (we haven't reached `image.close()`
|
|
543
|
+
// in the surrounding `finally` block yet), so the
|
|
544
|
+
// encode reads raw camera pixels directly into a
|
|
545
|
+
// JPEG at the final persistent path — no tmp file,
|
|
546
|
+
// no second copy.
|
|
547
|
+
YuvImageConverter.encodeToJpeg(
|
|
548
|
+
image,
|
|
549
|
+
targetPath,
|
|
550
|
+
jpegQuality = 70,
|
|
551
|
+
displayRotation = rotationForEncode,
|
|
552
|
+
) != null
|
|
553
|
+
},
|
|
485
554
|
)
|
|
486
555
|
} finally {
|
|
487
556
|
image.close()
|
|
488
557
|
}
|
|
489
558
|
}
|
|
490
559
|
|
|
491
|
-
private fun postFrameToEngine(
|
|
492
|
-
path: String,
|
|
493
|
-
tx: Double, ty: Double, tz: Double,
|
|
494
|
-
qx: Double, qy: Double, qz: Double, qw: Double,
|
|
495
|
-
fx: Double, fy: Double, cx: Double, cy: Double,
|
|
496
|
-
imageWidth: Int, imageHeight: Int,
|
|
497
|
-
yaw: Double,
|
|
498
|
-
pitch: Double,
|
|
499
|
-
fovHorizDegrees: Double,
|
|
500
|
-
fovVertDegrees: Double,
|
|
501
|
-
trackingPoor: Boolean,
|
|
502
|
-
) {
|
|
503
|
-
val module = IncrementalStitcher.bridgeInstance ?: return
|
|
504
|
-
module.ingestFromARCameraView(
|
|
505
|
-
path = path,
|
|
506
|
-
tx = tx, ty = ty, tz = tz,
|
|
507
|
-
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
508
|
-
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
509
|
-
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
510
|
-
yaw = yaw,
|
|
511
|
-
pitch = pitch,
|
|
512
|
-
fovHorizDegrees = fovHorizDegrees,
|
|
513
|
-
fovVertDegrees = fovVertDegrees,
|
|
514
|
-
trackingPoor = trackingPoor,
|
|
515
|
-
)
|
|
516
|
-
}
|
|
517
|
-
|
|
518
560
|
private fun applyDisplayGeometry() {
|
|
519
561
|
val session = sessionRef.get() ?: return
|
|
520
562
|
val rotation = (context.getSystemService(Context.WINDOW_SERVICE) as? WindowManager)
|
|
@@ -148,7 +148,34 @@ class RNSARSession(reactContext: ReactApplicationContext)
|
|
|
148
148
|
@ReactMethod
|
|
149
149
|
fun stop(promise: Promise) {
|
|
150
150
|
try {
|
|
151
|
-
|
|
151
|
+
// 2026-05-23 (crash fix) — Session.pause() stops frame
|
|
152
|
+
// production but keeps the native session ALIVE: its
|
|
153
|
+
// internal worker threads (tango_pool_lp4, etc.) keep
|
|
154
|
+
// running. Once the session reference is nulled here,
|
|
155
|
+
// those threads become orphaned — still alive, but with
|
|
156
|
+
// no owner to clean them up. Under later memory
|
|
157
|
+
// pressure scudo unmaps freed pages and an in-flight
|
|
158
|
+
// tango_pool_lp4 memcpy SEGVs on an unmapped destination
|
|
159
|
+
// (the crash we diagnosed from tombstone_03, with
|
|
160
|
+
// libarcore_c.so internal `ImageBlockData` frames).
|
|
161
|
+
//
|
|
162
|
+
// Session.close() shuts down those threads AND releases
|
|
163
|
+
// native resources, which is what we actually want for
|
|
164
|
+
// an explicit "AR off" toggle. Pause+close together is
|
|
165
|
+
// ARCore's documented full-teardown sequence. The next
|
|
166
|
+
// start() recreates the Session from scratch (see
|
|
167
|
+
// line 105's `sessionRef.get() ?: Session(...)` path).
|
|
168
|
+
val prev = sessionRef.getAndSet(null)
|
|
169
|
+
try {
|
|
170
|
+
prev?.pause()
|
|
171
|
+
} catch (t: Throwable) {
|
|
172
|
+
Log.w(TAG, "stop: pause failed (ignoring): ${t.message}")
|
|
173
|
+
}
|
|
174
|
+
try {
|
|
175
|
+
prev?.close()
|
|
176
|
+
} catch (t: Throwable) {
|
|
177
|
+
Log.w(TAG, "stop: close failed (ignoring): ${t.message}")
|
|
178
|
+
}
|
|
152
179
|
trackingStateRef.set(TRACKING_NOT_AVAILABLE)
|
|
153
180
|
clearPoseLogInternal()
|
|
154
181
|
promise.resolve(null)
|
|
@@ -291,11 +318,26 @@ class RNSARSession(reactContext: ReactApplicationContext)
|
|
|
291
318
|
// Common on first-attach failures (no Activity, etc.).
|
|
292
319
|
return
|
|
293
320
|
}
|
|
294
|
-
|
|
321
|
+
// 2026-05-23 (crash fix) — pause + close, not just pause.
|
|
322
|
+
// See the matching fix in `stop()` above for the full
|
|
323
|
+
// rationale. Short version: pause() leaves ARCore's
|
|
324
|
+
// internal worker threads alive but orphaned; close()
|
|
325
|
+
// tears them down. Required for the AR-off toggle path
|
|
326
|
+
// (ARCameraView unmount → onDetachedFromWindow → here).
|
|
327
|
+
try {
|
|
328
|
+
prev.pause()
|
|
329
|
+
} catch (t: Throwable) {
|
|
330
|
+
Log.w(TAG, "stopForView: pause failed (ignoring): ${t.message}")
|
|
331
|
+
}
|
|
332
|
+
try {
|
|
333
|
+
prev.close()
|
|
334
|
+
} catch (t: Throwable) {
|
|
335
|
+
Log.w(TAG, "stopForView: close failed (ignoring): ${t.message}")
|
|
336
|
+
}
|
|
295
337
|
trackingStateRef.set(TRACKING_NOT_AVAILABLE)
|
|
296
|
-
Log.i(TAG, "stopForView: AR session paused")
|
|
338
|
+
Log.i(TAG, "stopForView: AR session paused + closed")
|
|
297
339
|
} catch (t: Throwable) {
|
|
298
|
-
Log.w(TAG, "stopForView:
|
|
340
|
+
Log.w(TAG, "stopForView: teardown failed: ${t.message}", t)
|
|
299
341
|
}
|
|
300
342
|
}
|
|
301
343
|
|
package/cpp/stitcher.cpp
CHANGED
|
@@ -244,11 +244,97 @@ cv::Rect maxInscribedRectFromMask(const cv::Mat& mask) {
|
|
|
244
244
|
} // namespace
|
|
245
245
|
|
|
246
246
|
|
|
247
|
+
// Forward declaration — body is the renamed inner entry point further
|
|
248
|
+
// down. The public `stitchFramePaths` wraps this with the
|
|
249
|
+
// mode-fallback retry logic added in the 2026-05-22 audit.
|
|
250
|
+
static StitchResult stitchFramePathsImpl_(
|
|
251
|
+
const std::vector<std::string>& framePaths,
|
|
252
|
+
const std::string& outputPath,
|
|
253
|
+
const StitchConfig& config,
|
|
254
|
+
LogFn logFn);
|
|
255
|
+
|
|
256
|
+
|
|
247
257
|
StitchResult stitchFramePaths(
|
|
248
258
|
const std::vector<std::string>& framePaths,
|
|
249
259
|
const std::string& outputPath,
|
|
250
260
|
const StitchConfig& config,
|
|
251
261
|
LogFn logFn)
|
|
262
|
+
{
|
|
263
|
+
// 2026-05-22 (audit follow-up) — mode-fallback retry. When the
|
|
264
|
+
// configured stitchMode produces degenerate camera params (the
|
|
265
|
+
// "warpRoi too large" crash users hit on translation-heavy
|
|
266
|
+
// captures stitched as PANORAMA, or low-texture inputs stitched
|
|
267
|
+
// as SCANS), automatically retry once with the OPPOSITE mode
|
|
268
|
+
// before giving up. Symmetric: PANORAMA-then-SCANS or
|
|
269
|
+
// SCANS-then-PANORAMA depending on configured mode.
|
|
270
|
+
//
|
|
271
|
+
// Why this is safe to enable unconditionally:
|
|
272
|
+
// - The retry only fires on a failed attempt (no perf hit on
|
|
273
|
+
// happy paths).
|
|
274
|
+
// - Both modes share the load-images and write-output stages,
|
|
275
|
+
// so the per-frame I/O cost isn't duplicated — only the
|
|
276
|
+
// estimator/BA/warp middle is re-run.
|
|
277
|
+
// - Result reflects whichever mode succeeded (returned via
|
|
278
|
+
// StitchResult.stitchModeUsed, populated below).
|
|
279
|
+
auto runOnce = [&](StitchMode modeOverride) -> StitchResult {
|
|
280
|
+
StitchConfig cfg = config;
|
|
281
|
+
cfg.stitchMode = modeOverride;
|
|
282
|
+
return stitchFramePathsImpl_(framePaths, outputPath, cfg, logFn);
|
|
283
|
+
};
|
|
284
|
+
StitchResult firstAttempt = runOnce(config.stitchMode);
|
|
285
|
+
if (firstAttempt.errorCode == StitchErrorCode::Ok) {
|
|
286
|
+
firstAttempt.stitchModeUsed = config.stitchMode;
|
|
287
|
+
return firstAttempt;
|
|
288
|
+
}
|
|
289
|
+
// First attempt failed. Try the opposite mode unless the error
|
|
290
|
+
// is something the opposite mode wouldn't fix (e.g. invalid
|
|
291
|
+
// argument count, file-read failure, OOM).
|
|
292
|
+
bool worthRetrying =
|
|
293
|
+
firstAttempt.errorCode == StitchErrorCode::UnknownCvException
|
|
294
|
+
|| firstAttempt.errorCode == StitchErrorCode::HomographyEstimationFailed
|
|
295
|
+
|| firstAttempt.errorCode == StitchErrorCode::CameraParamsAdjustFailed
|
|
296
|
+
|| firstAttempt.errorCode == StitchErrorCode::WarpFailed
|
|
297
|
+
|| firstAttempt.errorCode == StitchErrorCode::EmptyPanorama;
|
|
298
|
+
if (!worthRetrying) {
|
|
299
|
+
firstAttempt.stitchModeUsed = config.stitchMode;
|
|
300
|
+
return firstAttempt;
|
|
301
|
+
}
|
|
302
|
+
StitchMode fallbackMode =
|
|
303
|
+
(config.stitchMode == StitchMode::Panorama) ? StitchMode::Scans
|
|
304
|
+
: StitchMode::Panorama;
|
|
305
|
+
log_info(logFn, "[stitch-fallback]",
|
|
306
|
+
"primary mode (%s) failed with code=%d msg=%s — retrying with %s",
|
|
307
|
+
config.stitchMode == StitchMode::Scans ? "scans" : "panorama",
|
|
308
|
+
static_cast<int>(firstAttempt.errorCode),
|
|
309
|
+
firstAttempt.errorMessage.c_str(),
|
|
310
|
+
fallbackMode == StitchMode::Scans ? "scans" : "panorama");
|
|
311
|
+
StitchResult secondAttempt = runOnce(fallbackMode);
|
|
312
|
+
if (secondAttempt.errorCode == StitchErrorCode::Ok) {
|
|
313
|
+
secondAttempt.stitchModeUsed = fallbackMode;
|
|
314
|
+
log_info(logFn, "[stitch-fallback]",
|
|
315
|
+
"fallback mode (%s) succeeded",
|
|
316
|
+
fallbackMode == StitchMode::Scans ? "scans" : "panorama");
|
|
317
|
+
return secondAttempt;
|
|
318
|
+
}
|
|
319
|
+
// Both attempts failed. Return the FIRST attempt's error (it's
|
|
320
|
+
// what the operator's chosen mode produced — more useful for
|
|
321
|
+
// diagnosis than the fallback's failure).
|
|
322
|
+
log_info(logFn, "[stitch-fallback]",
|
|
323
|
+
"fallback mode (%s) also failed with code=%d — returning primary error",
|
|
324
|
+
fallbackMode == StitchMode::Scans ? "scans" : "panorama",
|
|
325
|
+
static_cast<int>(secondAttempt.errorCode));
|
|
326
|
+
firstAttempt.stitchModeUsed = config.stitchMode;
|
|
327
|
+
return firstAttempt;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// 2026-05-22 (audit follow-up) — renamed inner entry point so the
|
|
331
|
+
// public `stitchFramePaths` wrapper above can layer the mode-fallback
|
|
332
|
+
// retry on top. This used to be the public function.
|
|
333
|
+
static StitchResult stitchFramePathsImpl_(
|
|
334
|
+
const std::vector<std::string>& framePaths,
|
|
335
|
+
const std::string& outputPath,
|
|
336
|
+
const StitchConfig& config,
|
|
337
|
+
LogFn logFn)
|
|
252
338
|
{
|
|
253
339
|
// V2 routing — when caller opts in, hand off to the manual
|
|
254
340
|
// cv::detail::* pipeline. See stitcher.hpp::StitchConfig::
|
|
@@ -1572,13 +1658,27 @@ StitchResult stitchFramePathsManual(
|
|
|
1572
1658
|
i, roi.width, roi.height,
|
|
1573
1659
|
(long long)roiPixels,
|
|
1574
1660
|
(long long)kMaxWarpPixels);
|
|
1661
|
+
// 2026-05-22 (audit follow-up) — include
|
|
1662
|
+
// stitchMode + frame index in the error message
|
|
1663
|
+
// so the JS host can correlate the failure with
|
|
1664
|
+
// operator behaviour. Pre-fix the error said
|
|
1665
|
+
// nothing about which pipeline diverged. The
|
|
1666
|
+
// value tells you: PANORAMA usually fails on
|
|
1667
|
+
// translation-heavy input (homography + BA-Ray
|
|
1668
|
+
// assume pure rotation); SCANS usually fails on
|
|
1669
|
+
// low-texture or low-overlap input (affine needs
|
|
1670
|
+
// enough matches).
|
|
1671
|
+
const char* modeStr =
|
|
1672
|
+
(config.stitchMode == StitchMode::Scans) ? "scans" : "panorama";
|
|
1575
1673
|
throw cv::Exception(
|
|
1576
1674
|
cv::Error::StsOutOfRange,
|
|
1577
1675
|
std::string("warpRoi too large (")
|
|
1578
1676
|
+ std::to_string(roi.width) + "x"
|
|
1579
1677
|
+ std::to_string(roi.height)
|
|
1580
1678
|
+ ") — estimator produced degenerate "
|
|
1581
|
-
+ "camera params on this frame"
|
|
1679
|
+
+ "camera params on this frame (stitchMode="
|
|
1680
|
+
+ modeStr + ", frameIdx="
|
|
1681
|
+
+ std::to_string(i) + ")",
|
|
1582
1682
|
"stitchFramePathsManual",
|
|
1583
1683
|
__FILE__, __LINE__);
|
|
1584
1684
|
}
|
package/cpp/stitcher.hpp
CHANGED
|
@@ -217,6 +217,14 @@ struct StitchResult {
|
|
|
217
217
|
double finalConfidenceThresh = -1.0; // The threshold value that succeeded; -1 if not relevant.
|
|
218
218
|
|
|
219
219
|
int64_t durationMs = 0;
|
|
220
|
+
|
|
221
|
+
// 2026-05-22 (audit follow-up) — the stitchMode that actually
|
|
222
|
+
// produced the output, after the auto-fallback in `stitchFramePaths`
|
|
223
|
+
// (which retries with the opposite mode when the configured one
|
|
224
|
+
// fails with degenerate camera params). May differ from
|
|
225
|
+
// StitchConfig::stitchMode iff the fallback ran. Defaults to
|
|
226
|
+
// Panorama for back-compat in code paths that don't set it.
|
|
227
|
+
StitchMode stitchModeUsed = StitchMode::Panorama;
|
|
220
228
|
};
|
|
221
229
|
|
|
222
230
|
|
package/dist/camera/Camera.d.ts
CHANGED
|
@@ -74,6 +74,15 @@ export type CameraCaptureResult = {
|
|
|
74
74
|
framesDropped: number;
|
|
75
75
|
finalConfidenceThresh: number;
|
|
76
76
|
durationMs: number;
|
|
77
|
+
/**
|
|
78
|
+
* 2026-05-22 (audit F2g) — which cv::Stitcher pipeline the
|
|
79
|
+
* batch finalize ran (after auto-resolution if applicable).
|
|
80
|
+
* Useful for displaying a "Stitched as: scans" pill on the
|
|
81
|
+
* output preview. Undefined when the engine wasn't
|
|
82
|
+
* batch-keyframe (hybrid / slit-scan don't go through
|
|
83
|
+
* cv::Stitcher at finalize).
|
|
84
|
+
*/
|
|
85
|
+
stitchModeResolved?: 'panorama' | 'scans';
|
|
77
86
|
};
|
|
78
87
|
/**
|
|
79
88
|
* Errors surfaced via `onError`. Classified codes so consumers can
|