react-native-image-stitcher 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -1
- package/android/src/main/java/io/imagestitcher/rn/CvFlowGateFrameProcessor.kt +2 -2
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +2 -30
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +55 -368
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +6 -3
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +17 -30
- package/dist/camera/Camera.d.ts +11 -27
- package/dist/camera/Camera.js +46 -78
- package/dist/index.d.ts +0 -2
- package/dist/index.js +4 -6
- package/dist/stitching/incremental.d.ts +10 -11
- package/dist/stitching/useFrameProcessorDriver.d.ts +7 -6
- package/dist/stitching/useFrameProcessorDriver.js +12 -11
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +25 -206
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +0 -8
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +6 -126
- package/ios/Sources/RNImageStitcher/KeyframeGateFrameProcessor.mm +6 -6
- package/package.json +1 -1
- package/src/camera/Camera.tsx +57 -106
- package/src/index.ts +3 -8
- package/src/stitching/incremental.ts +10 -11
- package/src/stitching/useFrameProcessorDriver.ts +12 -11
- package/dist/stitching/useIncrementalJSDriver.d.ts +0 -74
- package/dist/stitching/useIncrementalJSDriver.js +0 -220
- package/src/stitching/useIncrementalJSDriver.ts +0 -297
|
@@ -58,20 +58,20 @@ import java.util.concurrent.atomic.AtomicBoolean
|
|
|
58
58
|
*
|
|
59
59
|
* What the bridge exposes to JS:
|
|
60
60
|
* - start(options) — spin up the engine
|
|
61
|
-
* - processFrameAtPath() — feed a JPEG path + pose; engine returns
|
|
62
|
-
* the same outcome enum iOS emits as events
|
|
63
61
|
* - finalize(options) — write the final panorama and reset
|
|
64
62
|
* - cancel() — abort without producing output
|
|
65
63
|
* - getState() — pull the latest state on demand
|
|
66
|
-
* -
|
|
67
|
-
*
|
|
64
|
+
* - refinePanorama() — re-run the C++ stitcher over saved keyframes
|
|
65
|
+
* - cleanupKeyframes() — GC stale per-capture keyframe directories
|
|
66
|
+
* - Event "IncrementalStateUpdate" emitted on every accepted frame
|
|
68
67
|
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
68
|
+
* How frames reach the engine (no JS-driven path post-v0.6):
|
|
69
|
+
* - AR mode: `RNSARCameraView` calls `ingestFromARCameraView(...)`
|
|
70
|
+
* once per ARCore Frame from its scene-update listener.
|
|
71
|
+
* - Non-AR mode: the vision-camera Frame Processor plugin
|
|
72
|
+
* (`CvFlowGateFrameProcessor`) calls `consumeFrameFromPlugin(...)`
|
|
73
|
+
* on the producer thread, gated by `frameProcessorIngestEnabled`.
|
|
74
|
+
* The pre-v0.6 `processFrameAtPath` JS-driver entry point is gone.
|
|
75
75
|
*/
|
|
76
76
|
class IncrementalStitcher(
|
|
77
77
|
private val reactContext: ReactApplicationContext,
|
|
@@ -115,9 +115,9 @@ class IncrementalStitcher(
|
|
|
115
115
|
// The MVP gate is frame-count-based ("accept every Nth frame
|
|
116
116
|
// until cap"). iOS uses a pose-based gate (overlap < threshold)
|
|
117
117
|
// — adding that here is a follow-up that needs ARCore-pose
|
|
118
|
-
// accumulation across
|
|
119
|
-
// N-th frame is good enough to validate end-to-end
|
|
120
|
-
// parity.
|
|
118
|
+
// accumulation across `ingestFromARCameraView` calls. For now,
|
|
119
|
+
// every N-th frame is good enough to validate end-to-end
|
|
120
|
+
// stitching parity.
|
|
121
121
|
private var batchKeyframeMode: Boolean = false
|
|
122
122
|
private val batchKeyframePaths: MutableList<String> = mutableListOf()
|
|
123
123
|
private var batchKeyframeFrameCounter: Int = 0
|
|
@@ -217,9 +217,10 @@ class IncrementalStitcher(
|
|
|
217
217
|
|
|
218
218
|
/// 2026-05-22 (audit F5) — Frame counter for the flowEvalEveryNFrames
|
|
219
219
|
/// throttle. Incremented on every per-frame entry point
|
|
220
|
-
/// (ingestFromARCameraView for AR,
|
|
221
|
-
/// gate evaluation only runs when
|
|
222
|
-
/// so frame #1 always evaluates
|
|
220
|
+
/// (`ingestFromARCameraView` for AR mode, `consumeFrameFromPlugin`
|
|
221
|
+
/// for non-AR Frame Processor mode); gate evaluation only runs when
|
|
222
|
+
/// (counter - 1) % evalCadence == 0 so frame #1 always evaluates
|
|
223
|
+
/// regardless of cadence. iOS parity:
|
|
223
224
|
/// IncrementalStitcher.swift:2459-2471 (`consumeFrameCounter`).
|
|
224
225
|
/// Reset to 0 on each `start()` call.
|
|
225
226
|
private var consumeFrameCounter: Long = 0L
|
|
@@ -229,27 +230,23 @@ class IncrementalStitcher(
|
|
|
229
230
|
/// F8.4 — gate for `consumeFrameFromPlugin` (the vision-camera
|
|
230
231
|
/// Frame Processor producer-thread entry point on Android).
|
|
231
232
|
/// TRUE only when the current capture was started with
|
|
232
|
-
/// `frameSourceMode == "frameProcessor"`. In
|
|
233
|
-
/// (
|
|
234
|
-
/// `
|
|
235
|
-
///
|
|
236
|
-
/// the
|
|
237
|
-
///
|
|
233
|
+
/// `frameSourceMode == "frameProcessor"`. In AR mode
|
|
234
|
+
/// (`frameSourceMode == "arSession"`) the plugin would double-feed
|
|
235
|
+
/// the engine alongside `ingestFromARCameraView` — bytes from the
|
|
236
|
+
/// producer thread + bytes from the ARCore Frame listener, racing
|
|
237
|
+
/// on the same workScope serial dispatcher — so we drop the
|
|
238
|
+
/// producer-thread call.
|
|
238
239
|
///
|
|
239
240
|
/// AtomicBoolean: producer thread reads lock-free, JS thread
|
|
240
241
|
/// (start/cancel/finalize) writes via `set()`/`compareAndSet()`.
|
|
241
242
|
/// Mirror of iOS' `frameProcessorIngestEnabled` ivar.
|
|
242
243
|
private val frameProcessorIngestEnabled = AtomicBoolean(false)
|
|
243
|
-
|
|
244
|
-
///
|
|
245
|
-
///
|
|
246
|
-
///
|
|
247
|
-
|
|
248
|
-
///
|
|
249
|
-
/// processFrameAtPath() calls can't race on the engine's canvas.
|
|
250
|
-
/// `limitedParallelism(1)` guarantees one-at-a-time execution
|
|
251
|
-
/// while still backing onto the Default pool — matches iOS'
|
|
252
|
-
/// `workQueue` (DispatchQueue.serial).
|
|
244
|
+
/// Critic #5 fix: serial dispatcher so concurrent per-frame
|
|
245
|
+
/// ingest calls (today: `ingestFromARCameraView` in AR mode,
|
|
246
|
+
/// `consumeFrameFromPlugin` in frame-processor mode) can't race
|
|
247
|
+
/// on the engine's canvas. `limitedParallelism(1)` guarantees
|
|
248
|
+
/// one-at-a-time execution while still backing onto the Default
|
|
249
|
+
/// pool — matches iOS' `workQueue` (DispatchQueue.serial).
|
|
253
250
|
@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class)
|
|
254
251
|
private val workScope = CoroutineScope(Dispatchers.Default.limitedParallelism(1))
|
|
255
252
|
|
|
@@ -280,17 +277,6 @@ class IncrementalStitcher(
|
|
|
280
277
|
/// on start/stop so the view feeds frames only during a capture.
|
|
281
278
|
@Volatile private var arCameraViewRef: RNSARCameraView? = null
|
|
282
279
|
|
|
283
|
-
/// 2026-05-21 (v0.3) — public-to-the-view-file getter so
|
|
284
|
-
/// `RNSARCameraView.forwardToIncremental` can ask whether the
|
|
285
|
-
/// currently-running engine is the batch-keyframe path (v0.3
|
|
286
|
-
/// Y-plane pixel-data flow) or the legacy hybrid/firstwins live
|
|
287
|
-
/// engine (which still needs a per-frame JPEG path). Used to
|
|
288
|
-
/// elide the eager JPEG encode for batchKeyframe mode (the
|
|
289
|
-
/// production Camera component's path); the legacy live engine
|
|
290
|
-
/// still pays the per-frame JPEG cost.
|
|
291
|
-
internal val isBatchKeyframeMode: Boolean
|
|
292
|
-
get() = batchKeyframeMode
|
|
293
|
-
|
|
294
280
|
init {
|
|
295
281
|
// Static back-pointer so `RNSARCameraView` can call into
|
|
296
282
|
// the singleton-style bridge module without a DI dance. RN
|
|
@@ -332,15 +318,17 @@ class IncrementalStitcher(
|
|
|
332
318
|
// F8.4 — frameSourceMode honoured on Android. Pre-F8.4,
|
|
333
319
|
// Android ignored this option (only iOS interpreted it).
|
|
334
320
|
// Now `"frameProcessor"` unlocks `consumeFrameFromPlugin`'s
|
|
335
|
-
// producer-thread ingest path;
|
|
336
|
-
//
|
|
337
|
-
//
|
|
338
|
-
//
|
|
339
|
-
|
|
321
|
+
// producer-thread ingest path; `"arSession"` (the default)
|
|
322
|
+
// keeps it dormant so the ARCore-driven
|
|
323
|
+
// `ingestFromARCameraView` path runs unmodified.
|
|
324
|
+
// Default to "arSession" for parity with iOS — pre-v0.6 this
|
|
325
|
+
// defaulted to "jsDriver", but that mode (the JS-driver
|
|
326
|
+
// processFrameAtPath path) was removed in v0.6. Raw
|
|
327
|
+
// NativeModules callers that omit `frameSourceMode` get the
|
|
328
|
+
// AR-mode behaviour (the production <Camera> always sets
|
|
329
|
+
// 'arSession' explicitly for AR captures anyway).
|
|
330
|
+
val frameSourceMode = options.getString("frameSourceMode") ?: "arSession"
|
|
340
331
|
frameProcessorIngestEnabled.set(frameSourceMode == "frameProcessor")
|
|
341
|
-
// F8.6 perf-diagnostic — re-arm the route log for the
|
|
342
|
-
// new capture.
|
|
343
|
-
f8_6_routeLoggedThisCapture.set(false)
|
|
344
332
|
val rotation = options.getIntOrDefault("frameRotationDegrees", 90)
|
|
345
333
|
val composeW = options.getIntOrDefault("composeWidth", 960)
|
|
346
334
|
val composeH = options.getIntOrDefault("composeHeight", 720)
|
|
@@ -593,14 +581,6 @@ class IncrementalStitcher(
|
|
|
593
581
|
}
|
|
594
582
|
}
|
|
595
583
|
|
|
596
|
-
/**
|
|
597
|
-
* Feed one frame at a JPEG path into the engine. Pose inputs
|
|
598
|
-
* drive the same FoV-overlap gate as iOS. When a pose source
|
|
599
|
-
* isn't available pass yaw=0, pitch=0, fovHorizDegrees=0 — the
|
|
600
|
-
* engine treats fov<=0 as a sentinel for "no intrinsics" and
|
|
601
|
-
* substitutes a 65° default, so frames will still be processed,
|
|
602
|
-
* just less gated.
|
|
603
|
-
*/
|
|
604
584
|
/**
|
|
605
585
|
* Copy a (non-persistent) source JPEG to a persistent per-keyframe
|
|
606
586
|
* path under the React context's cache dir. The ARCameraView's
|
|
@@ -622,67 +602,6 @@ class IncrementalStitcher(
|
|
|
622
602
|
* for a Phase 3 follow-up; this MVP is just enough to make
|
|
623
603
|
* batch-keyframe work end-to-end on Android.
|
|
624
604
|
*/
|
|
625
|
-
/**
|
|
626
|
-
* 2026-05-21 (v0.3) — JPEG-to-grayscale decode for the JS-driver
|
|
627
|
-
* (non-AR) keyframe-gate path. Used by the batch-keyframe branch
|
|
628
|
-
* of processFrameAtPath to feed the C++ KeyframeGate with real
|
|
629
|
-
* pixel data so its Flow strategy actually runs (pre-0.3 the
|
|
630
|
-
* non-AR call was `evaluate(pose, null)` which silently fell back
|
|
631
|
-
* to the Pose strategy because no pixel data was supplied).
|
|
632
|
-
*
|
|
633
|
-
* Uses OpenCV's Imgcodecs.imread with IMREAD_GRAYSCALE, which
|
|
634
|
-
* decodes the JPEG straight into a single-channel CV_8UC1 Mat —
|
|
635
|
-
* faster than BitmapFactory + manual luma loop (~10-20 ms here
|
|
636
|
-
* vs ~100+ ms in interpreted Kotlin for a 1920×1080 image).
|
|
637
|
-
*
|
|
638
|
-
* The non-AR snapshot cadence is ~4 FPS so the per-call cost is
|
|
639
|
-
* well under the inter-snapshot interval. v0.4 will replace
|
|
640
|
-
* this code path entirely by moving non-AR capture to
|
|
641
|
-
* vision-camera's Frame Processor API (which delivers raw
|
|
642
|
-
* pixels directly with no JPEG roundtrip). Tracked at issue #11.
|
|
643
|
-
*
|
|
644
|
-
* Returns null when the decode fails (corrupt JPEG, OOM, etc.);
|
|
645
|
-
* caller is expected to fall back to the pose-only evaluate
|
|
646
|
-
* path so the capture doesn't lock up.
|
|
647
|
-
*/
|
|
648
|
-
private data class GrayscaleFrame(
|
|
649
|
-
val bytes: ByteArray,
|
|
650
|
-
val width: Int,
|
|
651
|
-
val height: Int,
|
|
652
|
-
val stride: Int,
|
|
653
|
-
)
|
|
654
|
-
|
|
655
|
-
private fun decodeJpegToGrayscale(path: String): GrayscaleFrame? {
|
|
656
|
-
val mat = Imgcodecs.imread(path, Imgcodecs.IMREAD_GRAYSCALE)
|
|
657
|
-
if (mat.empty()) {
|
|
658
|
-
android.util.Log.w(
|
|
659
|
-
"IncrementalStitcher",
|
|
660
|
-
"decodeJpegToGrayscale: imread returned empty Mat for $path"
|
|
661
|
-
)
|
|
662
|
-
return null
|
|
663
|
-
}
|
|
664
|
-
return try {
|
|
665
|
-
val width = mat.cols()
|
|
666
|
-
val height = mat.rows()
|
|
667
|
-
// step1() returns bytes-per-row for a CV_8UC1 Mat. For a
|
|
668
|
-
// continuous Mat from imread (no ROI) stride == width.
|
|
669
|
-
val stride = mat.step1().toInt()
|
|
670
|
-
val size = stride * height
|
|
671
|
-
val bytes = ByteArray(size)
|
|
672
|
-
mat.get(0, 0, bytes)
|
|
673
|
-
GrayscaleFrame(bytes, width, height, stride)
|
|
674
|
-
} catch (e: Exception) {
|
|
675
|
-
android.util.Log.w(
|
|
676
|
-
"IncrementalStitcher",
|
|
677
|
-
"decodeJpegToGrayscale: failed to copy Mat bytes for $path: ${e.message}",
|
|
678
|
-
e,
|
|
679
|
-
)
|
|
680
|
-
null
|
|
681
|
-
} finally {
|
|
682
|
-
mat.release()
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
|
|
686
605
|
private fun copyKeyframeToStore(srcPath: String): String? {
|
|
687
606
|
// V16 Phase 2 (Android Fix-1) — write into the per-session
|
|
688
607
|
// subdir created by start(). If start() didn't run (defensive
|
|
@@ -716,8 +635,8 @@ class IncrementalStitcher(
|
|
|
716
635
|
// ── V16 Phase 1 → P3-F migration note ────────────────────────
|
|
717
636
|
// The frame-counter placeholder gate `handleBatchKeyframeFrame`
|
|
718
637
|
// that lived here has been REMOVED. Both the AR-driven path
|
|
719
|
-
// (ingestFromARCameraView) and the
|
|
720
|
-
// (
|
|
638
|
+
// (`ingestFromARCameraView`) and the Frame Processor path
|
|
639
|
+
// (`consumeFrameFromPlugin`) now route through the shared-C++
|
|
721
640
|
// `KeyframeGate` (cpp/keyframe_gate.{hpp,cpp}) — same algorithm
|
|
722
641
|
// iOS has used since the V16 ship. See
|
|
723
642
|
// `private val keyframeGate = KeyframeGate()` above for the
|
|
@@ -730,204 +649,6 @@ class IncrementalStitcher(
|
|
|
730
649
|
// we ever add a "force every Nth frame regardless of overlap"
|
|
731
650
|
// override.
|
|
732
651
|
|
|
733
|
-
@ReactMethod
|
|
734
|
-
fun processFrameAtPath(options: ReadableMap, promise: Promise) {
|
|
735
|
-
val hybrid = this.engine
|
|
736
|
-
val firstwins = this.firstwinsEngine
|
|
737
|
-
// batch-keyframe mode runs without a live engine — handle it
|
|
738
|
-
// up-front before the null-check rejects.
|
|
739
|
-
//
|
|
740
|
-
// P3-F: this path uses the same shared-C++ KeyframeGate as
|
|
741
|
-
// the AR view path, but with translation = 0 (gyro-derived
|
|
742
|
-
// poses have only rotation, not position). The gate's
|
|
743
|
-
// internal logic detects the missing plane and uses the
|
|
744
|
-
// camera-forward angular-delta fallback automatically.
|
|
745
|
-
if (batchKeyframeMode) {
|
|
746
|
-
val path = options.getString("path")
|
|
747
|
-
?: return promise.reject("invalid-options", "path required")
|
|
748
|
-
val pose = RNSARFramePose(
|
|
749
|
-
tx = options.getDoubleOrDefault("tx", 0.0) ?: 0.0,
|
|
750
|
-
ty = options.getDoubleOrDefault("ty", 0.0) ?: 0.0,
|
|
751
|
-
tz = options.getDoubleOrDefault("tz", 0.0) ?: 0.0,
|
|
752
|
-
qx = options.getDoubleOrDefault("qx", 0.0) ?: 0.0,
|
|
753
|
-
qy = options.getDoubleOrDefault("qy", 0.0) ?: 0.0,
|
|
754
|
-
qz = options.getDoubleOrDefault("qz", 0.0) ?: 0.0,
|
|
755
|
-
qw = options.getDoubleOrDefault("qw", 1.0) ?: 1.0,
|
|
756
|
-
fx = options.getDoubleOrDefault("fx", 1000.0) ?: 1000.0,
|
|
757
|
-
fy = options.getDoubleOrDefault("fy", 1000.0) ?: 1000.0,
|
|
758
|
-
cx = options.getDoubleOrDefault("cx", 540.0) ?: 540.0,
|
|
759
|
-
cy = options.getDoubleOrDefault("cy", 960.0) ?: 960.0,
|
|
760
|
-
imageWidth = options.getIntOrDefault("imageWidth", 1080),
|
|
761
|
-
imageHeight = options.getIntOrDefault("imageHeight", 1920),
|
|
762
|
-
timestampMs = 0.0,
|
|
763
|
-
trackingState = RNSARSession.TRACKING_TRACKING,
|
|
764
|
-
)
|
|
765
|
-
// Vision-camera path: no plane available (gyro can't fit
|
|
766
|
-
// planes). Pass null → C++ skips the plane-overlap math.
|
|
767
|
-
//
|
|
768
|
-
// 2026-05-21 (v0.3) — pixel-aware Flow-strategy evaluation.
|
|
769
|
-
// The JS driver hands us a JPEG file path; we decode it to
|
|
770
|
-
// grayscale here so the C++ gate actually runs sparse-flow
|
|
771
|
-
// novelty (pre-0.3 the call was `evaluate(pose, null)` and
|
|
772
|
-
// the C++ side silently fell back to Pose strategy because
|
|
773
|
-
// no pixel data was supplied — same bug as Android AR
|
|
774
|
-
// mode, both fixed in v0.3). BitmapFactory + manual luma
|
|
775
|
-
// is fine here — non-AR snapshot cadence is ~4 FPS, the
|
|
776
|
-
// ~15-30 ms JPEG-decode-to-grayscale per call is well
|
|
777
|
-
// under the inter-snapshot interval. (v0.4 will move
|
|
778
|
-
// non-AR to vision-camera's Frame Processor API, at which
|
|
779
|
-
// point the JPEG step goes away entirely. Tracked at
|
|
780
|
-
// https://github.com/bhargavkanda/react-native-image-stitcher/issues/11)
|
|
781
|
-
val gray = decodeJpegToGrayscale(path)
|
|
782
|
-
val decision = if (gray != null) {
|
|
783
|
-
keyframeGate.evaluateWithFrame(
|
|
784
|
-
pose, null,
|
|
785
|
-
gray.bytes, gray.width, gray.height, gray.stride,
|
|
786
|
-
)
|
|
787
|
-
} else {
|
|
788
|
-
// JPEG decode failed (corrupt file? OOM?). Fall back
|
|
789
|
-
// to pose-only path so the capture doesn't lock up;
|
|
790
|
-
// this matches the C++ side's own defensive fallback
|
|
791
|
-
// for null grayData.
|
|
792
|
-
keyframeGate.evaluate(pose, null)
|
|
793
|
-
}
|
|
794
|
-
val result = Arguments.createMap()
|
|
795
|
-
// Outcome mapping for iOS-parity JS contract:
|
|
796
|
-
// 1 = accepted, 2 = rejected (gate), 3 = rejected (cap).
|
|
797
|
-
// C++ enum → outcome int:
|
|
798
|
-
val outcome = when {
|
|
799
|
-
decision.accept -> 1
|
|
800
|
-
decision.reason == "max-reached" -> 3
|
|
801
|
-
else -> 2
|
|
802
|
-
}
|
|
803
|
-
result.putInt("outcome", outcome)
|
|
804
|
-
if (!decision.accept) {
|
|
805
|
-
// 2026-05-22 (audit follow-up) — emit reject-state on
|
|
806
|
-
// the non-AR JS-driver path too so the debug overlay
|
|
807
|
-
// sees overlap % update on every snapshot (~4 Hz).
|
|
808
|
-
// Same rationale as the AR path's reject emission
|
|
809
|
-
// above; iOS parity.
|
|
810
|
-
emitBatchKeyframeRejectState(
|
|
811
|
-
decision = decision,
|
|
812
|
-
keyframeCount = batchKeyframePaths.size,
|
|
813
|
-
keyframeMax = keyframeGate.maxCount,
|
|
814
|
-
isLandscape = pose.imageWidth >= pose.imageHeight,
|
|
815
|
-
)
|
|
816
|
-
}
|
|
817
|
-
if (decision.accept) {
|
|
818
|
-
batchKeyframePaths.add(path) // vision-camera path
|
|
819
|
-
// gives us a unique
|
|
820
|
-
// per-snapshot file
|
|
821
|
-
// already (no copy
|
|
822
|
-
// needed).
|
|
823
|
-
// Emit the same state event the AR path emits so
|
|
824
|
-
// the JS LiveFrameStrip + "Keyframes n/max" pill
|
|
825
|
-
// work identically on the vision-camera fallback
|
|
826
|
-
// path.
|
|
827
|
-
emitBatchKeyframeAcceptedState(
|
|
828
|
-
thumbnailPath = path,
|
|
829
|
-
keyframeIndex = batchKeyframePaths.size - 1,
|
|
830
|
-
keyframeCount = batchKeyframePaths.size,
|
|
831
|
-
keyframeMax = keyframeGate.maxCount,
|
|
832
|
-
isLandscape = pose.imageWidth >= pose.imageHeight,
|
|
833
|
-
newContentFraction = decision.newContentFraction,
|
|
834
|
-
)
|
|
835
|
-
}
|
|
836
|
-
result.putInt("acceptedCount", batchKeyframePaths.size)
|
|
837
|
-
promise.resolve(result)
|
|
838
|
-
return
|
|
839
|
-
}
|
|
840
|
-
if (hybrid == null && firstwins == null) {
|
|
841
|
-
return promise.reject(
|
|
842
|
-
"incremental-not-running",
|
|
843
|
-
"Call start() before processFrameAtPath().",
|
|
844
|
-
)
|
|
845
|
-
}
|
|
846
|
-
val path = options.getString("path")
|
|
847
|
-
?: return promise.reject("invalid-options", "path required")
|
|
848
|
-
val yaw = options.getDoubleOrDefault("yaw", 0.0)
|
|
849
|
-
val pitch = options.getDoubleOrDefault("pitch", 0.0)
|
|
850
|
-
val fovH = options.getDoubleOrDefault("fovHorizDegrees", 65.0)
|
|
851
|
-
val fovV = options.getDoubleOrDefault("fovVertDegrees", 50.0)
|
|
852
|
-
// V6 pose-driven params. Defaults removed per critic finding
|
|
853
|
-
// #3: previously qw=1.0 default meant frames without explicit
|
|
854
|
-
// quaternion produced an identity rotation, and EVERY
|
|
855
|
-
// subsequent frame had R_rel = R_first^T (constant), so
|
|
856
|
-
// strip placement never advanced and `acceptedCount` froze
|
|
857
|
-
// at 1 after the first frame. Now every quaternion field is
|
|
858
|
-
// required; missing → reject as RejectedAlignmentLost so the
|
|
859
|
-
// gyro driver upstream notices instantly.
|
|
860
|
-
if (!options.hasKey("qx") || !options.hasKey("qy")
|
|
861
|
-
|| !options.hasKey("qz") || !options.hasKey("qw")) {
|
|
862
|
-
return promise.reject(
|
|
863
|
-
"invalid-options",
|
|
864
|
-
"qx/qy/qz/qw all required (no identity-quaternion fallback)",
|
|
865
|
-
)
|
|
866
|
-
}
|
|
867
|
-
val qx = options.getDouble("qx")
|
|
868
|
-
val qy = options.getDouble("qy")
|
|
869
|
-
val qz = options.getDouble("qz")
|
|
870
|
-
val qw = options.getDouble("qw")
|
|
871
|
-
val fx = options.getDoubleOrDefault("fx", 0.0)
|
|
872
|
-
val fy = options.getDoubleOrDefault("fy", 0.0)
|
|
873
|
-
val cx = options.getDoubleOrDefault("cx", 0.0)
|
|
874
|
-
val cy = options.getDoubleOrDefault("cy", 0.0)
|
|
875
|
-
val imageWidth = options.getIntOrDefault("imageWidth", 0)
|
|
876
|
-
val imageHeight = options.getIntOrDefault("imageHeight", 0)
|
|
877
|
-
val trackingPoor = options.getBooleanOrDefault("trackingPoor", false)
|
|
878
|
-
|
|
879
|
-
workScope.launch {
|
|
880
|
-
// Critic #4 fix: re-check isRunning synchronously here in
|
|
881
|
-
// case finalize/cancel ran on the JS thread between the
|
|
882
|
-
// null-check above and this dispatch landing. Skip the
|
|
883
|
-
// ingest if we're no longer running — matches iOS' V12.1
|
|
884
|
-
// pattern (synchronous-stop + worker re-check).
|
|
885
|
-
if (!isRunning.get()) {
|
|
886
|
-
promise.resolve(Arguments.createMap().apply { putInt("outcome", -1) })
|
|
887
|
-
return@launch
|
|
888
|
-
}
|
|
889
|
-
try {
|
|
890
|
-
val telemetry: FrameTelemetry
|
|
891
|
-
val state: WritableMap?
|
|
892
|
-
val accepted: Int
|
|
893
|
-
if (firstwins != null) {
|
|
894
|
-
telemetry = firstwins.addFrameAtPath(
|
|
895
|
-
path = path,
|
|
896
|
-
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
897
|
-
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
898
|
-
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
899
|
-
yaw = yaw, pitch = pitch,
|
|
900
|
-
fovHorizDegrees = fovH, fovVertDegrees = fovV,
|
|
901
|
-
trackingPoor = trackingPoor,
|
|
902
|
-
)
|
|
903
|
-
state = firstwins.snapshotIfDue(telemetry)
|
|
904
|
-
accepted = firstwins.acceptedCount
|
|
905
|
-
} else {
|
|
906
|
-
telemetry = hybrid!!.addFrameAtPath(
|
|
907
|
-
path = path,
|
|
908
|
-
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
909
|
-
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
910
|
-
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
911
|
-
yaw = yaw, pitch = pitch,
|
|
912
|
-
fovHorizDegrees = fovH, fovVertDegrees = fovV,
|
|
913
|
-
trackingPoor = trackingPoor,
|
|
914
|
-
)
|
|
915
|
-
state = hybrid.snapshotIfDue(telemetry)
|
|
916
|
-
accepted = hybrid.acceptedCount
|
|
917
|
-
}
|
|
918
|
-
emitState(state)
|
|
919
|
-
val result = Arguments.createMap()
|
|
920
|
-
result.putInt("outcome", telemetry.outcome.ordinal)
|
|
921
|
-
result.putDouble("confidence", telemetry.confidence)
|
|
922
|
-
result.putDouble("overlapPercent", telemetry.overlapPercent)
|
|
923
|
-
result.putDouble("processingMs", telemetry.processingMs)
|
|
924
|
-
result.putInt("acceptedCount", accepted)
|
|
925
|
-
promise.resolve(result)
|
|
926
|
-
} catch (t: Throwable) {
|
|
927
|
-
promise.reject("incremental-process-failed", t.message, t)
|
|
928
|
-
}
|
|
929
|
-
}
|
|
930
|
-
}
|
|
931
652
|
|
|
932
653
|
@ReactMethod
|
|
933
654
|
fun finalize(options: ReadableMap, promise: Promise) {
|
|
@@ -971,9 +692,10 @@ class IncrementalStitcher(
|
|
|
971
692
|
// frames slip into the engine while we serialize the canvas.
|
|
972
693
|
arCameraViewRef?.setIncrementalIngestionActive(false)
|
|
973
694
|
// Critic #4 fix: synchronously flip isRunning=false BEFORE
|
|
974
|
-
// dispatching the finalize body, so any in-flight
|
|
975
|
-
//
|
|
976
|
-
//
|
|
695
|
+
// dispatching the finalize body, so any in-flight per-frame
|
|
696
|
+
// ingest workers about to launch on workScope (today:
|
|
697
|
+
// `ingestFromARCameraView` or `consumeFrameFromPlugin`) bail
|
|
698
|
+
// at the re-check inside their workScope.launch blocks.
|
|
977
699
|
// Matches iOS V12.1 fix.
|
|
978
700
|
isRunning.set(false)
|
|
979
701
|
frameProcessorIngestEnabled.set(false) // F8.4 — cut producer-thread ingest at finalize
|
|
@@ -1266,9 +988,10 @@ class IncrementalStitcher(
|
|
|
1266
988
|
// is false (the legacy hybrid/firstwins live-engine path,
|
|
1267
989
|
// which feeds JPEG paths into addFrameAtPath for each ARCore
|
|
1268
990
|
// frame). Pass null when batchKeyframeMode is true; the
|
|
1269
|
-
// batch path uses `grayData` + `onAccept` instead.
|
|
1270
|
-
//
|
|
1271
|
-
//
|
|
991
|
+
// batch path uses `grayData` + `onAccept` instead. Modern
|
|
992
|
+
// callers prefer `nv21PixelData` below — `legacyJpegPath` is
|
|
993
|
+
// kept only as a defensive fallback for older call sites
|
|
994
|
+
// that have not yet been migrated.
|
|
1272
995
|
legacyJpegPath: String? = null,
|
|
1273
996
|
// F8.6 — pixel-data path for live engines. When supplied
|
|
1274
997
|
// (and `batchKeyframeMode == false`), takes precedence over
|
|
@@ -1473,26 +1196,13 @@ class IncrementalStitcher(
|
|
|
1473
1196
|
val hasPixelData = nv21PixelData != null
|
|
1474
1197
|
&& nv21PixelWidth > 0
|
|
1475
1198
|
&& nv21PixelHeight > 0
|
|
1476
|
-
// F8.6 perf-diagnostic — log route choice once per capture
|
|
1477
|
-
// so logcat shows whether the new pixel-data path is
|
|
1478
|
-
// actually getting exercised. Throttled to first-hit per
|
|
1479
|
-
// capture (counter resets in start()/cancel()).
|
|
1480
|
-
if (!f8_6_routeLoggedThisCapture.getAndSet(true)) {
|
|
1481
|
-
android.util.Log.i(
|
|
1482
|
-
"F8.6-route",
|
|
1483
|
-
"ingestFromARCameraView live-engine route: "
|
|
1484
|
-
+ if (hasPixelData) "pixel-data (F8.6, no JPEG round-trip)"
|
|
1485
|
-
else "jpeg-path (legacy, JPEG decode round-trip)",
|
|
1486
|
-
)
|
|
1487
|
-
}
|
|
1488
1199
|
val path = if (hasPixelData) null else legacyJpegPath ?: run {
|
|
1489
1200
|
android.util.Log.w(
|
|
1490
1201
|
"IncrementalStitcher",
|
|
1491
1202
|
"ingestFromARCameraView legacy: batchKeyframeMode=false " +
|
|
1492
1203
|
"but both legacyJpegPath and nv21PixelData are null — " +
|
|
1493
|
-
"dropping frame. Caller
|
|
1494
|
-
"
|
|
1495
|
-
"isBatchKeyframeMode == false.",
|
|
1204
|
+
"dropping frame. Caller must supply NV21 pixel data " +
|
|
1205
|
+
"(preferred) or a JPEG path for the live engine path.",
|
|
1496
1206
|
)
|
|
1497
1207
|
return
|
|
1498
1208
|
}
|
|
@@ -1611,8 +1321,8 @@ class IncrementalStitcher(
|
|
|
1611
1321
|
) {
|
|
1612
1322
|
// F8.4 — drop the call unless this capture was started in
|
|
1613
1323
|
// frameProcessor mode. Otherwise the plugin would double-
|
|
1614
|
-
// feed the engine alongside the
|
|
1615
|
-
//
|
|
1324
|
+
// feed the engine alongside the AR-mode
|
|
1325
|
+
// `ingestFromARCameraView` path. See the flag's declaration
|
|
1616
1326
|
// for the full reasoning. Mirrors iOS H1.
|
|
1617
1327
|
if (!frameProcessorIngestEnabled.get()) return
|
|
1618
1328
|
|
|
@@ -2635,7 +2345,7 @@ internal class IncrementalEngine(
|
|
|
2635
2345
|
// See iOS' equivalent fix for the architectural rationale.
|
|
2636
2346
|
val frame = downsampleToCompose(srcRaw)
|
|
2637
2347
|
if (frame !== srcRaw) srcRaw.release()
|
|
2638
|
-
|
|
2348
|
+
return addFrameMat(
|
|
2639
2349
|
frame,
|
|
2640
2350
|
qx, qy, qz, qw,
|
|
2641
2351
|
fx, fy, cx, cy,
|
|
@@ -2644,8 +2354,6 @@ internal class IncrementalEngine(
|
|
|
2644
2354
|
fovHorizDegrees, fovVertDegrees,
|
|
2645
2355
|
t0,
|
|
2646
2356
|
)
|
|
2647
|
-
f8_6_logPerf("hybrid/jpeg", t0, tele.outcome)
|
|
2648
|
-
return tele
|
|
2649
2357
|
}
|
|
2650
2358
|
|
|
2651
2359
|
/**
|
|
@@ -2695,7 +2403,7 @@ internal class IncrementalEngine(
|
|
|
2695
2403
|
}
|
|
2696
2404
|
val frame = downsampleToCompose(srcRaw)
|
|
2697
2405
|
if (frame !== srcRaw) srcRaw.release()
|
|
2698
|
-
|
|
2406
|
+
return addFrameMat(
|
|
2699
2407
|
frame,
|
|
2700
2408
|
qx, qy, qz, qw,
|
|
2701
2409
|
fx, fy, cx, cy,
|
|
@@ -2704,27 +2412,6 @@ internal class IncrementalEngine(
|
|
|
2704
2412
|
fovHorizDegrees, fovVertDegrees,
|
|
2705
2413
|
t0,
|
|
2706
2414
|
)
|
|
2707
|
-
f8_6_logPerf("hybrid/pixel", t0, tele.outcome)
|
|
2708
|
-
return tele
|
|
2709
|
-
}
|
|
2710
|
-
|
|
2711
|
-
/**
|
|
2712
|
-
* F8.6 perf-diagnostic counter (mirror of the firstwins one).
|
|
2713
|
-
* Remove after v0.5.1 ships and the numbers are baked in.
|
|
2714
|
-
*/
|
|
2715
|
-
@Volatile private var f8_6_perfCallCounter: Long = 0L
|
|
2716
|
-
private fun f8_6_logPerf(
|
|
2717
|
-
path: String,
|
|
2718
|
-
t0Nanos: Long,
|
|
2719
|
-
outcome: FrameOutcome,
|
|
2720
|
-
) {
|
|
2721
|
-
val n = ++f8_6_perfCallCounter
|
|
2722
|
-
if (n == 1L || n % 5L == 0L) {
|
|
2723
|
-
android.util.Log.i(
|
|
2724
|
-
"F8.6-perf",
|
|
2725
|
-
"$path took ${msSince(t0Nanos)}ms outcome=$outcome (call #$n)",
|
|
2726
|
-
)
|
|
2727
|
-
}
|
|
2728
2415
|
}
|
|
2729
2416
|
|
|
2730
2417
|
/**
|
|
@@ -247,9 +247,12 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
247
247
|
* the Y plane from the ARCore camera image (YUV_420_888) and
|
|
248
248
|
* hands it through. Zero-copy on the way in (the byte[] is
|
|
249
249
|
* pinned via GetPrimitiveArrayCritical in the JNI).
|
|
250
|
-
* - Non-AR mode (`
|
|
251
|
-
*
|
|
252
|
-
*
|
|
250
|
+
* - Non-AR mode (`CvFlowGateFrameProcessor` via
|
|
251
|
+
* `IncrementalStitcher.consumeFrameFromPlugin`): extracts the
|
|
252
|
+
* Y plane from the vision-camera Frame's YUV_420_888 image on
|
|
253
|
+
* the producer thread and hands it through. (Pre-v0.6 a
|
|
254
|
+
* JS-driver `processFrameAtPath` path also called this with
|
|
255
|
+
* JPEG-decoded grayscale; both were removed in v0.6.)
|
|
253
256
|
*
|
|
254
257
|
* @param grayData The grayscale plane bytes. Length must be
|
|
255
258
|
* at least `grayStride * grayHeight`.
|
|
@@ -18,7 +18,6 @@ import com.google.ar.core.exceptions.CameraNotAvailableException
|
|
|
18
18
|
import com.google.ar.core.exceptions.SessionPausedException
|
|
19
19
|
import io.imagestitcher.rn.ar.BackgroundRenderer
|
|
20
20
|
import io.imagestitcher.rn.ar.YuvImageConverter
|
|
21
|
-
import java.io.File
|
|
22
21
|
import java.util.concurrent.atomic.AtomicReference
|
|
23
22
|
import javax.microedition.khronos.egl.EGLConfig
|
|
24
23
|
import javax.microedition.khronos.opengles.GL10
|
|
@@ -76,13 +75,6 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
76
75
|
private var surfaceWidth: Int = 0
|
|
77
76
|
private var surfaceHeight: Int = 0
|
|
78
77
|
|
|
79
|
-
/// Tmp directory for the per-frame JPEG file we hand to the
|
|
80
|
-
/// incremental engine. Created lazily and reused across frames
|
|
81
|
-
/// — no per-frame allocation.
|
|
82
|
-
private val tmpJpegFile: File by lazy {
|
|
83
|
-
File(context.cacheDir, "rlis-arframe.jpg")
|
|
84
|
-
}
|
|
85
|
-
|
|
86
78
|
/// Whether to feed the AR session's frames into the incremental
|
|
87
79
|
/// engine. Toggled by IncrementalStitcher.start/stop
|
|
88
80
|
/// via setIncrementalIngestionActive() below.
|
|
@@ -510,28 +502,18 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
510
502
|
val rotationForEncode = if (lastDisplayRotation >= 0)
|
|
511
503
|
lastDisplayRotation else android.view.Surface.ROTATION_0
|
|
512
504
|
|
|
513
|
-
//
|
|
514
|
-
//
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
//
|
|
518
|
-
//
|
|
519
|
-
//
|
|
505
|
+
// F8.6 (v0.6) — the eager JPEG encode for live-engine mode
|
|
506
|
+
// is gone. Pass the already-packed NV21 directly via
|
|
507
|
+
// `nv21PixelData`; the engine's new `addFramePixelData`
|
|
508
|
+
// path builds the BGR cv::Mat in-process via cvtColor,
|
|
509
|
+
// skipping the JPEG decode round-trip downstream. In
|
|
510
|
+
// batch-keyframe mode the engine ignores `nv21PixelData`
|
|
511
|
+
// (it uses `grayData` + `onAccept` lazily); no behaviour
|
|
512
|
+
// change there.
|
|
520
513
|
//
|
|
521
|
-
//
|
|
522
|
-
//
|
|
523
|
-
//
|
|
524
|
-
// hold time.
|
|
525
|
-
val legacyJpegPath: String? = if (module.isBatchKeyframeMode) {
|
|
526
|
-
null
|
|
527
|
-
} else {
|
|
528
|
-
YuvImageConverter.encodeJpegFromNV21(
|
|
529
|
-
packed,
|
|
530
|
-
tmpJpegFile.absolutePath,
|
|
531
|
-
jpegQuality = 70,
|
|
532
|
-
displayRotation = rotationForEncode,
|
|
533
|
-
)
|
|
534
|
-
}
|
|
514
|
+
// (Was: eager JPEG encode for non-batch-keyframe modes,
|
|
515
|
+
// written to `tmpJpegFile`, passed as `legacyJpegPath`.
|
|
516
|
+
// See the v0.3 / F8.6 entries in CHANGELOG.md.)
|
|
535
517
|
module.ingestFromARCameraView(
|
|
536
518
|
tx = tArr[0].toDouble(),
|
|
537
519
|
ty = tArr[1].toDouble(),
|
|
@@ -553,7 +535,12 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
553
535
|
grayWidth = packed.width,
|
|
554
536
|
grayHeight = packed.height,
|
|
555
537
|
grayStride = packed.width,
|
|
556
|
-
legacyJpegPath =
|
|
538
|
+
legacyJpegPath = null,
|
|
539
|
+
// F8.6 — pixel-data path for live engines. Batch-
|
|
540
|
+
// keyframe mode ignores these (bails earlier).
|
|
541
|
+
nv21PixelData = packed.nv21,
|
|
542
|
+
nv21PixelWidth = packed.width,
|
|
543
|
+
nv21PixelHeight = packed.height,
|
|
557
544
|
onAccept = { targetPath ->
|
|
558
545
|
// Lazy JPEG encode. Runs ONLY if the C++ KeyframeGate
|
|
559
546
|
// accepted the frame. Encodes from the pre-packed
|