react-native-image-stitcher 0.5.0 → 0.5.1

package/CHANGELOG.md

@@ -16,6 +16,72 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.5.1] — 2026-05-25
+
+### Added — F8.6 Android pixel-buffer engine parity
+
+Closes the v0.5.0 follow-up tracked in the [0.5.0] section.
+
+**Live engine ingest no longer requires a JPEG round-trip.**
+The `IncrementalFirstwinsEngine` (slit-scan / first-wins) and the
+hybrid `IncrementalEngine` both gained a new
+`addFramePixelData(nv21, w, h, ...)` method.  It builds the BGR
+`cv::Mat` in-process via
+`Imgproc.cvtColor(yuv, COLOR_YUV2BGR_NV21)`, then delegates to a
+newly-extracted shared `addFrameMat` helper that runs the original
+engine pipeline verbatim.  The legacy `addFrameAtPath(path, ...)`
+is now a thin wrapper: `imread → downsample → addFrameMat`.
+
+**Routing.**  `IncrementalStitcher.ingestFromARCameraView` got
+three optional parameters — `nv21PixelData: ByteArray?`,
+`nv21PixelWidth: Int`, `nv21PixelHeight: Int`.  When supplied (and
+`batchKeyframeMode == false`), the live engine ingests via
+`addFramePixelData`; otherwise falls back to `addFrameAtPath` with
+`legacyJpegPath`.  Backwards-compatible — all-null defaults
+preserve every existing caller.
+
+**Frame Processor wiring.**  `consumeFrameFromPlugin` now packs the
+incoming `Image` NV21 once at the top (was twice — gate consumed
+Y only, then the `onAccept` lambda re-packed for JPEG encode) and
+threads the bytes through to both the gate (which reads only the
+Y subset) AND the new `nv21PixelData` parameter.  Net: single
+`packNV21` per producer-thread frame.
+
+**Measured on Galaxy A35, `engine: 'firstwins-rectilinear'`,
+non-AR Frame Processor capture:**
+
+| Outcome | F8.6 pixel-data | Legacy JPEG path (estimated) |
+|---|---|---|
+| `AcceptedHigh` (first-frame init) | 7–11 ms | 50–70 ms |
+| `SkippedTooClose` (gate bail) | 0.5–2 ms | 50–60 ms (imread is unconditional) |
+
+`SkippedTooClose` dominates the producer-thread frame budget
+(~95% of frames at 30 fps with a slow pan).  Eliminating the
+imread on those frames is the bulk of the F8.6 win.
+
+### Added
+
+* New `<Camera engine={...}>` prop exposes the live engine
+  selection (`'batch-keyframe'` (default) / `'firstwins-rectilinear'`
+  / `'hybrid'` / `'slitscan-*'`).  Lets hosts opt into in-flight
+  stitching for low-latency previews; previously the choice was
+  hardcoded.
+
+### Changed
+
+* `New: F8.6 perf-diagnostic logs` (`F8.6-route`, `F8.6-perf`) fire
+  in live-engine mode only — inert under the default
+  `batch-keyframe`.  Will be removed in v0.6 once F8.6 is baked in
+  production.
+
+### Fixed
+
+* In `IncrementalStitcher.consumeFrameFromPlugin`, the `onAccept`
+  lambda was re-packing the live `Image` instead of reusing the
+  already-packed NV21 from the outer scope.  Now it reuses the
+  outer `packed` — saves a redundant `packNV21` call on every
+  accepted frame.
+
 ## [0.5.0] — 2026-05-25
 
 ### Added — F8 Frame Processor port
@@ -91,13 +157,29 @@ Frame Processor** on the camera producer thread instead of the
 
 ### Tracking — known follow-ups (don't gate this release)
 
-- **F8.6** — Android engine refactor for pixel-buffer-direct
-  ingest (true zero-copy parity with iOS).
-- **F8.3.H2-target** — `swift test` currently can't run the
-  iOS test target due to mixed Swift/.mm sources.  The
-  `FrameProcessorPluginSelectorTests` selector guard is in place
-  as a documentation artifact; CI test-runner fix is a separate
-  task.
+- **F8.6 (v0.5.1)** — Android engine refactor for pixel-buffer-
+  direct ingest (true zero-copy parity with iOS).  Would extract
+  an `addFrameMat` helper from `IncrementalFirstwinsEngine` and
+  `IncrementalEngine`'s `addFrameAtPath`, add a parallel
+  `addFramePixelData` that constructs the BGR `cv::Mat` from NV21
+  bytes via `cvtColor`, and rewire `RNSARCameraView` to skip the
+  per-frame JPEG encode.  Expected gain: ~30–50 ms per accepted
+  frame.  Deferred because the engine bodies are 400+ lines of
+  complex AR-mode code; needs A35 device verification before
+  merge, which the v0.5.0 prep session didn't have.
+
+- **F8.3-followup-roll** — resolved in v0.5.0.
+
+- **F8.3.H2-target** — RESOLVED in v0.5.0 via a different
+  mechanism than originally planned.  The selector pin is now a
+  compile-time `#selector(...)` reference inside
+  `IncrementalStitcher.swift` plus a dev-build runtime assert in
+  `IncrementalStitcher.init()` — both fire if the Swift method
+  signature drifts from what `KeyframeGateFrameProcessor.mm`
+  expects.  The obsolete test file was deleted.  `swift test` now
+  runs the (8-test) `QualityCheckerTests` suite cleanly because
+  `Package.swift` switched from an exclude list (broke every time
+  a new `.mm` landed) to an explicit `sources` allowlist.
 
 ## [0.4.1] — 2026-05-23
 

package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt

@@ -197,7 +197,155 @@ internal class IncrementalFirstwinsEngine(
         }
         val frameBGR = downsampleToCompose(srcRaw)
         if (frameBGR !== srcRaw) srcRaw.release()
+        val tele = addFrameMat(
+            frameBGR,
+            qx, qy, qz, qw,
+            fx, fy, cx, cy,
+            imageWidth, imageHeight,
+            yaw, pitch,
+            fovHorizDegrees, fovVertDegrees,
+            t0,
+        )
+        f8_6_logPerf("firstwins/jpeg", t0, tele.outcome)
+        return tele
+    }
+
+    /**
+     * F8.6 — pixel-data twin of [addFrameAtPath].  Accepts the
+     * camera frame as an NV21 byte buffer instead of a JPEG file
+     * path; skips the JPEG decode round-trip (~30–50 ms per
+     * accepted frame on a mid-tier device).
+     *
+     * Use this from the vision-camera Frame Processor path (where
+     * we already have direct producer-thread access to YUV bytes)
+     * and from the ARCore path (where the previous
+     * JPEG-encode-on-every-frame in `RNSARCameraView` was a
+     * measurable hot-spot).
+     *
+     * The body matches [addFrameAtPath] one-for-one except for the
+     * Mat construction: an `Mat(h*3/2, w, CV_8UC1)` wraps the
+     * NV21 bytes, then `Imgproc.cvtColor` produces the BGR Mat the
+     * engine pipeline already expects.  Everything downstream is
+     * the shared [addFrameMat] helper.
+     *
+     * `nv21Width`/`nv21Height` describe the buffer's actual
+     * dimensions.  `imageWidth`/`imageHeight` describe the
+     * camera's reported sensor dims used for intrinsics scaling —
+     * these can differ when the camera is downsampling for Frame
+     * Processor output.
+     */
+    fun addFramePixelData(
+        nv21: ByteArray,
+        nv21Width: Int,
+        nv21Height: Int,
+        qx: Double, qy: Double, qz: Double, qw: Double,
+        fx: Double, fy: Double, cx: Double, cy: Double,
+        imageWidth: Int, imageHeight: Int,
+        yaw: Double, pitch: Double,
+        fovHorizDegrees: Double, fovVertDegrees: Double,
+        trackingPoor: Boolean,
+    ): FrameTelemetry {
+        val t0 = System.nanoTime()
+        if (trackingPoor) {
+            return FrameTelemetry(
+                FrameOutcome.SkippedTrackingPoor, -1.0, 0, yaw, pitch,
+                msSince(t0),
+                isLandscape = isLandscape,
+            )
+        }
+        // NV21 layout: Y plane (w*h bytes) + interleaved VU
+        // (w*h/2 bytes).  A single CV_8UC1 Mat of height h*3/2
+        // packs the whole thing; cvtColor with COLOR_YUV2BGR_NV21
+        // does the planar-aware decode in one call.
+        //
+        // F8.6 IS-1 — length guard.  If the caller supplied a
+        // short buffer, `yuv.put(0,0,nv21)` would copy only
+        // `nv21.size` bytes and leave the rest zero-init; cvtColor
+        // would then read stale/zero UV and produce silently
+        // corrupt colour.  Fail fast instead.
+        val expectedBytes = nv21Width * nv21Height * 3 / 2
+        require(nv21.size >= expectedBytes) {
+            "addFramePixelData: nv21 buffer too small " +
+                "(${nv21.size} bytes < $expectedBytes for " +
+                "${nv21Width}x${nv21Height})"
+        }
+        val yuv = Mat(nv21Height + nv21Height / 2, nv21Width, CvType.CV_8UC1)
+        yuv.put(0, 0, nv21)
+        val srcRaw = Mat()
+        Imgproc.cvtColor(yuv, srcRaw, Imgproc.COLOR_YUV2BGR_NV21)
+        yuv.release()
+        if (srcRaw.empty()) {
+            return FrameTelemetry(
+                FrameOutcome.RejectedAlignmentLost, -1.0, 0, yaw, pitch,
+                msSince(t0),
+                isLandscape = isLandscape,
+            )
+        }
+        val frameBGR = downsampleToCompose(srcRaw)
+        if (frameBGR !== srcRaw) srcRaw.release()
+        val tele = addFrameMat(
+            frameBGR,
+            qx, qy, qz, qw,
+            fx, fy, cx, cy,
+            imageWidth, imageHeight,
+            yaw, pitch,
+            fovHorizDegrees, fovVertDegrees,
+            t0,
+        )
+        f8_6_logPerf("firstwins/pixel", t0, tele.outcome)
+        return tele
+    }
 
+    /**
+     * F8.6 perf-diagnostic counter.  Logs ingest timing every Nth
+     * call so a single capture session yields enough samples to
+     * eyeball the JPEG-path vs pixel-data-path delta.  Remove this
+     * once F8.6 is verified in production.
+     */
+    @Volatile private var f8_6_perfCallCounter: Long = 0L
+    private fun f8_6_logPerf(
+        path: String,
+        t0Nanos: Long,
+        outcome: FrameOutcome,
+    ) {
+        val n = ++f8_6_perfCallCounter
+        // Every 5th call ≈ ~1 line/sec at 5 Hz live-engine rate;
+        // first call always logs so we see something on capture
+        // start without waiting.
+        if (n == 1L || n % 5L == 0L) {
+            Log.i(
+                "F8.6-perf",
+                "$path took ${msSince(t0Nanos)}ms outcome=$outcome (call #$n)",
+            )
+        }
+    }
+
+    /**
+     * F8.6 — the body extracted from [addFrameAtPath].  Takes a
+     * BGR `Mat` (downsampled to compose dims) and runs the full
+     * engine pipeline: first-frame init or subsequent-frame paste
+     * via either rectilinear or cylindrical warp.
+     *
+     * Behaviour is identical to the pre-F8.6 `addFrameAtPath`
+     * (the body is a verbatim move).  Both `addFrameAtPath` and
+     * `addFramePixelData` delegate here after their respective
+     * Mat constructions.
+     */
+    private fun addFrameMat(
+        frameBGR: Mat,
+        qx: Double, qy: Double, qz: Double, qw: Double,
+        fx: Double, fy: Double, cx: Double, cy: Double,
+        imageWidth: Int, imageHeight: Int,
+        yaw: Double, pitch: Double,
+        // FOV params kept for symmetry with `IncrementalEngine.addFrameMat`
+        // (the hybrid engine, which uses them in `computeOverlapPct`).
+        // The firstwins/slit-scan engine here doesn't consume them —
+        // its paste decision is driven by pose-projected pixel
+        // displacement, not FoV-overlap percent.
+        @Suppress("UNUSED_PARAMETER") fovHorizDegrees: Double,
+        @Suppress("UNUSED_PARAMETER") fovVertDegrees: Double,
+        t0: Long,
+    ): FrameTelemetry {
         val rNew = quaternionToRotationMat(qx, qy, qz, qw)
 
         if (!hasFirstFrame) {

package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt

@@ -240,6 +240,11 @@ class IncrementalStitcher(
     /// (start/cancel/finalize) writes via `set()`/`compareAndSet()`.
     /// Mirror of iOS' `frameProcessorIngestEnabled` ivar.
     private val frameProcessorIngestEnabled = AtomicBoolean(false)
+
+    /// F8.6 perf-diagnostic — one-shot guard so the "live-engine
+    /// route" log fires exactly once per capture session.  Reset
+    /// in start().
+    private val f8_6_routeLoggedThisCapture = AtomicBoolean(false)
     /// Critic #5 fix: serial dispatcher so concurrent
     /// processFrameAtPath() calls can't race on the engine's canvas.
     /// `limitedParallelism(1)` guarantees one-at-a-time execution
@@ -333,6 +338,9 @@ class IncrementalStitcher(
             // / ARCore paths run unmodified.
             val frameSourceMode = options.getString("frameSourceMode") ?: "jsDriver"
             frameProcessorIngestEnabled.set(frameSourceMode == "frameProcessor")
+            // F8.6 perf-diagnostic — re-arm the route log for the
+            // new capture.
+            f8_6_routeLoggedThisCapture.set(false)
             val rotation = options.getIntOrDefault("frameRotationDegrees", 90)
             val composeW = options.getIntOrDefault("composeWidth",  960)
             val composeH = options.getIntOrDefault("composeHeight", 720)
@@ -1262,6 +1270,24 @@ class IncrementalStitcher(
         // can check `isBatchKeyframeMode` to elide the per-frame
         // JPEG encode for the batch path.
         legacyJpegPath: String? = null,
+        // F8.6 — pixel-data path for live engines.  When supplied
+        // (and `batchKeyframeMode == false`), takes precedence over
+        // `legacyJpegPath`: the live engine ingests via
+        // `addFramePixelData` (NV21 → BGR Mat in-process) instead of
+        // `addFrameAtPath` (JPEG decode round-trip).  Saves ~30-50 ms
+        // per accepted frame on a mid-tier device.  Pass null to use
+        // the legacy JPEG path.
+        //
+        // OWNERSHIP: the engine retains a reference to `nv21PixelData`
+        // until `workScope`'s coroutine consumes it (~50 ms later).
+        // Callers MUST treat the array as transferred — do not
+        // mutate it or return it to a buffer pool after calling
+        // this method.  If a caller needs to recycle the buffer,
+        // pass `.copyOf()` (currently no caller does — the F8.4
+        // Frame Processor plugin allocates a fresh array per frame).
+        nv21PixelData: ByteArray? = null,
+        nv21PixelWidth: Int = 0,
+        nv21PixelHeight: Int = 0,
     ) {
         // ── V16 batch-keyframe: AR-driven path ─────────────────────
         //
@@ -1439,40 +1465,91 @@ class IncrementalStitcher(
         // we only get here when batchKeyframeMode == false.  Caller
         // (RNSARCameraView) was expected to supply legacyJpegPath in
         // that case — defensively drop the frame if it didn't.
-        val path = legacyJpegPath ?: run {
+        // F8.6 — prefer the pixel-data path when the caller supplied
+        // NV21 bytes (Frame Processor / refactored ARCore path),
+        // otherwise fall back to legacyJpegPath (un-migrated ARCore
+        // path).  At least one of them must be present; drop the
+        // frame defensively otherwise.
+        val hasPixelData = nv21PixelData != null
+            && nv21PixelWidth > 0
+            && nv21PixelHeight > 0
+        // F8.6 perf-diagnostic — log route choice once per capture
+        // so logcat shows whether the new pixel-data path is
+        // actually getting exercised.  Throttled to first-hit per
+        // capture (counter resets in start()/cancel()).
+        if (!f8_6_routeLoggedThisCapture.getAndSet(true)) {
+            android.util.Log.i(
+                "F8.6-route",
+                "ingestFromARCameraView live-engine route: "
+                + if (hasPixelData) "pixel-data (F8.6, no JPEG round-trip)"
+                  else "jpeg-path (legacy, JPEG decode round-trip)",
+            )
+        }
+        val path = if (hasPixelData) null else legacyJpegPath ?: run {
             android.util.Log.w(
                 "IncrementalStitcher",
                 "ingestFromARCameraView legacy: batchKeyframeMode=false " +
-                    "but legacyJpegPath is null — dropping frame.  " +
-                    "Caller should have encoded a JPEG when " +
+                    "but both legacyJpegPath and nv21PixelData are null — " +
+                    "dropping frame.  Caller should have encoded a JPEG " +
+                    "OR supplied NV21 pixel data when " +
                     "isBatchKeyframeMode == false.",
             )
             return
         }
         workScope.launch {
             val state: WritableMap? = if (firstwins != null) {
-                val tele = firstwins.addFrameAtPath(
-                    path = path,
-                    qx = qx, qy = qy, qz = qz, qw = qw,
-                    fx = fx, fy = fy, cx = cx, cy = cy,
-                    imageWidth = imageWidth, imageHeight = imageHeight,
-                    yaw = yaw, pitch = pitch,
-                    fovHorizDegrees = fovHorizDegrees,
-                    fovVertDegrees = fovVertDegrees,
-                    trackingPoor = trackingPoor,
-                )
+                val tele = if (hasPixelData) {
+                    firstwins.addFramePixelData(
+                        nv21 = nv21PixelData!!,
+                        nv21Width = nv21PixelWidth,
+                        nv21Height = nv21PixelHeight,
+                        qx = qx, qy = qy, qz = qz, qw = qw,
+                        fx = fx, fy = fy, cx = cx, cy = cy,
+                        imageWidth = imageWidth, imageHeight = imageHeight,
+                        yaw = yaw, pitch = pitch,
+                        fovHorizDegrees = fovHorizDegrees,
+                        fovVertDegrees = fovVertDegrees,
+                        trackingPoor = trackingPoor,
+                    )
+                } else {
+                    firstwins.addFrameAtPath(
+                        path = path!!,
+                        qx = qx, qy = qy, qz = qz, qw = qw,
+                        fx = fx, fy = fy, cx = cx, cy = cy,
+                        imageWidth = imageWidth, imageHeight = imageHeight,
+                        yaw = yaw, pitch = pitch,
+                        fovHorizDegrees = fovHorizDegrees,
+                        fovVertDegrees = fovVertDegrees,
+                        trackingPoor = trackingPoor,
+                    )
+                }
                 firstwins.snapshotIfDue(tele)
             } else {
-                val tele = hybrid!!.addFrameAtPath(
-                    path = path,
-                    qx = qx, qy = qy, qz = qz, qw = qw,
-                    fx = fx, fy = fy, cx = cx, cy = cy,
-                    imageWidth = imageWidth, imageHeight = imageHeight,
-                    yaw = yaw, pitch = pitch,
-                    fovHorizDegrees = fovHorizDegrees,
-                    fovVertDegrees = fovVertDegrees,
-                    trackingPoor = trackingPoor,
-                )
+                val tele = if (hasPixelData) {
+                    hybrid!!.addFramePixelData(
+                        nv21 = nv21PixelData!!,
+                        nv21Width = nv21PixelWidth,
+                        nv21Height = nv21PixelHeight,
+                        qx = qx, qy = qy, qz = qz, qw = qw,
+                        fx = fx, fy = fy, cx = cx, cy = cy,
+                        imageWidth = imageWidth, imageHeight = imageHeight,
+                        yaw = yaw, pitch = pitch,
+                        fovHorizDegrees = fovHorizDegrees,
+                        fovVertDegrees = fovVertDegrees,
+                        trackingPoor = trackingPoor,
+                    )
+                } else {
+                    hybrid!!.addFrameAtPath(
+                        path = path!!,
+                        qx = qx, qy = qy, qz = qz, qw = qw,
+                        fx = fx, fy = fy, cx = cx, cy = cy,
+                        imageWidth = imageWidth, imageHeight = imageHeight,
+                        yaw = yaw, pitch = pitch,
+                        fovHorizDegrees = fovHorizDegrees,
+                        fovVertDegrees = fovVertDegrees,
+                        trackingPoor = trackingPoor,
+                    )
+                }
                 hybrid.snapshotIfDue(tele)
             }
             emitState(state)
@@ -1539,18 +1616,31 @@ class IncrementalStitcher(
         // for the full reasoning.  Mirrors iOS H1.
         if (!frameProcessorIngestEnabled.get()) return
 
-        val width = image.width
-        val height = image.height
-        val yPlane = image.planes[0]
-        val yRowStride = yPlane.rowStride
-
-        // Read Y plane bytes.  ByteBuffer.get() advances position;
-        // copy into our own ByteArray so the engine's downstream
-        // workScope can safely outlive this method (the Image
-        // closes after callback returns, but we've already copied).
-        val yBuffer = yPlane.buffer
-        val yBytes = ByteArray(yBuffer.remaining())
-        yBuffer.get(yBytes)
+        // F8.6 — pack the full NV21 (Y + interleaved VU) once,
+        // then reuse it for BOTH the gate's Y-plane read AND the
+        // live engine's pixel-data ingest.  Previously the plugin
+        // only extracted Y; the live engine then had to JPEG-decode
+        // a separately-encoded path to recover BGR colour.  Now we
+        // skip both round-trips: the packed NV21 → BGR cvtColor
+        // inside `addFramePixelData` produces the BGR Mat directly.
+        //
+        // YuvImageConverter.packNV21 is stride-aware and densely
+        // repacks Y (so the gate's `grayStride = grayWidth = width`
+        // works), then interleaves VU per the standard NV21 layout
+        // [Y...][VU...].  Returns null only on degenerate Images
+        // (closed mid-callback or non-YUV format).
+        val packed = io.imagestitcher.rn.ar.YuvImageConverter.packNV21(image)
+            ?: return
+        val width = packed.width
+        val height = packed.height
+        val nv21Bytes = packed.nv21
+        // The gate reads `grayHeight` rows of `grayWidth` pixels
+        // at stride=width starting from offset 0.  That's exactly
+        // the Y plane region of nv21Bytes — the gate naturally
+        // stops before the UV bytes start.  No need to slice into
+        // a separate ByteArray.
+        val yBytes = nv21Bytes
+        val yRowStride = width
 
         // Compute derived params expected by the existing ingest
         // API.  Quaternion-to-yaw/pitch follows the same convention
@@ -1596,13 +1686,22 @@ class IncrementalStitcher(
             grayWidth = width,
             grayHeight = height,
             grayStride = yRowStride,
+            // F8.6 — pass the already-packed NV21 so the live
+            // engine branch (hybrid / firstwins) can ingest via
+            // `addFramePixelData` instead of JPEG-decoding a
+            // separately-written path.  Batch-keyframe mode
+            // ignores these (it uses `grayData` + `onAccept`).
+            nv21PixelData = nv21Bytes,
+            nv21PixelWidth = width,
+            nv21PixelHeight = height,
             onAccept = { targetPath ->
                 // Synchronous JPEG encode via the existing
                 // YuvImageConverter (also used by RNSARCameraView's
-                // ARCore path).  Handles both the NV21 conversion
-                // (stride / pixelStride aware) and the EXIF
-                // Orientation tag write so the JPEG displays upright
-                // in the UI thumbnail strip + any RN Image consumer.
+                // ARCore path).  Reuses the NV21 already packed at
+                // the top of `consumeFrameFromPlugin` — F8.6 saves
+                // a duplicate packNV21 call here (the previous
+                // version repacked the live `image` inside the
+                // lambda).
                 //
                 // EXIF rotation is BAKED-AS-METADATA, not pixel-
                 // rotated.  cv::imread in the stitcher ignores EXIF
@@ -1614,32 +1713,20 @@ class IncrementalStitcher(
                 // Returning `true` tells the engine the keyframe was
                 // persisted; `false` tells it to drop the accept.
                 try {
-                    val packed = YuvImageConverter.packNV21(image)
-                        ?: run {
-                            android.util.Log.w(
-                                "IncrementalStitcher",
-                                "consumeFrameFromPlugin: packNV21 returned null for $targetPath",
-                            )
-                            return@run null
-                        }
-                    if (packed == null) {
-                        false
-                    } else {
-                        val displayRotation = when (sensorRotationDegrees) {
-                            0   -> android.view.Surface.ROTATION_90
-                            90  -> android.view.Surface.ROTATION_0
-                            180 -> android.view.Surface.ROTATION_270
-                            270 -> android.view.Surface.ROTATION_180
-                            else -> android.view.Surface.ROTATION_0
-                        }
-                        val outPath = YuvImageConverter.encodeJpegFromNV21(
-                            packed,
-                            targetPath,
-                            jpegQuality = 80,
-                            displayRotation = displayRotation,
-                        )
-                        outPath != null
+                    val displayRotation = when (sensorRotationDegrees) {
+                        0   -> android.view.Surface.ROTATION_90
+                        90  -> android.view.Surface.ROTATION_0
+                        180 -> android.view.Surface.ROTATION_270
+                        270 -> android.view.Surface.ROTATION_180
+                        else -> android.view.Surface.ROTATION_0
                     }
+                    val outPath = YuvImageConverter.encodeJpegFromNV21(
+                        packed,
+                        targetPath,
+                        jpegQuality = 80,
+                        displayRotation = displayRotation,
+                    )
+                    outPath != null
                 } catch (e: Throwable) {
                     android.util.Log.w(
                         "IncrementalStitcher",
@@ -2548,7 +2635,114 @@ internal class IncrementalEngine(
         // See iOS' equivalent fix for the architectural rationale.
         val frame = downsampleToCompose(srcRaw)
         if (frame !== srcRaw) srcRaw.release()
+        val tele = addFrameMat(
+            frame,
+            qx, qy, qz, qw,
+            fx, fy, cx, cy,
+            imageWidth, imageHeight,
+            yaw, pitch,
+            fovHorizDegrees, fovVertDegrees,
+            t0,
+        )
+        f8_6_logPerf("hybrid/jpeg", t0, tele.outcome)
+        return tele
+    }
+
+    /**
+     * F8.6 — pixel-data twin of [addFrameAtPath].  Accepts the
+     * camera frame as an NV21 byte buffer instead of a JPEG file
+     * path; skips the JPEG decode round-trip.  See
+     * `IncrementalFirstwinsEngine.addFramePixelData` for the
+     * sibling implementation rationale.
+     */
+    fun addFramePixelData(
+        nv21: ByteArray,
+        nv21Width: Int,
+        nv21Height: Int,
+        qx: Double, qy: Double, qz: Double, qw: Double,
+        fx: Double, fy: Double, cx: Double, cy: Double,
+        imageWidth: Int, imageHeight: Int,
+        yaw: Double, pitch: Double,
+        fovHorizDegrees: Double, fovVertDegrees: Double,
+        trackingPoor: Boolean,
+    ): FrameTelemetry {
+        val t0 = System.nanoTime()
+        if (trackingPoor) {
+            return FrameTelemetry(
+                FrameOutcome.SkippedTrackingPoor, -1.0, 0, 0.0, 0.0,
+                msSince(t0),
+            )
+        }
+        // F8.6 IS-1 — length guard; see
+        // `IncrementalFirstwinsEngine.addFramePixelData` for the
+        // failure-mode rationale.
+        val expectedBytes = nv21Width * nv21Height * 3 / 2
+        require(nv21.size >= expectedBytes) {
+            "addFramePixelData: nv21 buffer too small " +
+                "(${nv21.size} bytes < $expectedBytes for " +
+                "${nv21Width}x${nv21Height})"
+        }
+        val yuv = Mat(nv21Height + nv21Height / 2, nv21Width, CvType.CV_8UC1)
+        yuv.put(0, 0, nv21)
+        val srcRaw = Mat()
+        Imgproc.cvtColor(yuv, srcRaw, Imgproc.COLOR_YUV2BGR_NV21)
+        yuv.release()
+        if (srcRaw.empty()) {
+            return FrameTelemetry(
+                FrameOutcome.SkippedTrackingPoor, -1.0, 0, 0.0, 0.0,
+                msSince(t0),
+            )
+        }
+        val frame = downsampleToCompose(srcRaw)
+        if (frame !== srcRaw) srcRaw.release()
+        val tele = addFrameMat(
+            frame,
+            qx, qy, qz, qw,
+            fx, fy, cx, cy,
+            imageWidth, imageHeight,
+            yaw, pitch,
+            fovHorizDegrees, fovVertDegrees,
+            t0,
+        )
+        f8_6_logPerf("hybrid/pixel", t0, tele.outcome)
+        return tele
+    }
 
+    /**
+     * F8.6 perf-diagnostic counter (mirror of the firstwins one).
+     * Remove after v0.5.1 ships and the numbers are baked in.
+     */
+    @Volatile private var f8_6_perfCallCounter: Long = 0L
+    private fun f8_6_logPerf(
+        path: String,
+        t0Nanos: Long,
+        outcome: FrameOutcome,
+    ) {
+        val n = ++f8_6_perfCallCounter
+        if (n == 1L || n % 5L == 0L) {
+            android.util.Log.i(
+                "F8.6-perf",
+                "$path took ${msSince(t0Nanos)}ms outcome=$outcome (call #$n)",
+            )
+        }
+    }
+
+    /**
+     * F8.6 — the body extracted from [addFrameAtPath].  Takes a
+     * BGR `Mat` (already downsampled to compose dims) and runs the
+     * pose-driven homography paste pipeline.  Behaviour is
+     * identical to the pre-F8.6 `addFrameAtPath` — the body is a
+     * verbatim move.
+     */
+    private fun addFrameMat(
+        frame: Mat,
+        qx: Double, qy: Double, qz: Double, qw: Double,
+        fx: Double, fy: Double, cx: Double, cy: Double,
+        imageWidth: Int, imageHeight: Int,
+        yaw: Double, pitch: Double,
+        fovHorizDegrees: Double, fovVertDegrees: Double,
+        t0: Long,
+    ): FrameTelemetry {
         // Build R_new from quaternion.
         val rNew = quaternionToRotationMat(qx, qy, qz, qw)
 

package/dist/camera/Camera.d.ts

@@ -140,6 +140,24 @@ export interface CameraProps {
     enablePanoramaMode?: boolean;
     showSettingsButton?: boolean;
     style?: StyleProp<ViewStyle>;
+    /**
+     * Which incremental stitcher engine to drive.  Default
+     * `'batch-keyframe'` — collects accepted JPEGs and runs
+     * `cv::Stitcher` once at finalize time.  This is the v0.4+
+     * production default and what the v0.5 Frame Processor migration
+     * exercises.
+     *
+     * Switch to a live engine (`'firstwins-rectilinear'` or
+     * `'hybrid'`) for low-latency in-flight stitching.  Live engines
+     * exercise the F8.6 pixel-buffer ingest path (skipping the JPEG
+     * encode/decode round-trip; ~30–50 ms saved per accept) when the
+     * Frame Processor driver is active.
+     *
+     * See `docs/f8-frame-processor-plan.md` and the v0.5.0
+     * CHANGELOG for the trade-offs between batch-keyframe and live
+     * engines.
+     */
+    engine?: 'batch-keyframe' | 'hybrid' | 'slitscan-rotate' | 'slitscan-both' | 'firstwins' | 'firstwins-zoomed' | 'firstwins-rectilinear' | 'slitscan';
     /**
      * Optional destination directory for captures.  When set, the lib
      * lands tap-photos at `${outputDir}/photo-${ts}.jpg` and panoramas

package/dist/camera/Camera.js

@@ -271,7 +271,7 @@ function extractPanoramaOverrides(props) {
  * The public `<Camera>` component.
  */
 function Camera(props) {
-    const { defaultCaptureSource = 'ar', defaultLens = '1x', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, frameProcessor: hostFrameProcessor, legacyDriver = false, } = props;
+    const { defaultCaptureSource = 'ar', defaultLens = '1x', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, frameProcessor: hostFrameProcessor, legacyDriver = false, engine = 'batch-keyframe', } = props;
     const insets = (0, react_native_safe_area_context_1.useSafeAreaInsets)();
     // ── State ───────────────────────────────────────────────────────
     const [arPreference, setArPreference] = (0, react_1.useState)(defaultCaptureSource === 'ar');
@@ -662,7 +662,7 @@ function Camera(props) {
                 composeHeight: 1080,
                 canvasWidth: 5000,
                 canvasHeight: 5000,
-                engine: 'batch-keyframe',
+                engine,
                 config: (0, PanoramaSettingsBridge_1.panoramaSettingsToNativeConfig)({
                     ...settings,
                     captureSource: effectiveCaptureSource,
@@ -706,6 +706,7 @@ function Camera(props) {
         jsDriver,
         fpDriver,
         legacyDriver,
+        engine,
         onError,
     ]);
     const handleHoldEnd = (0, react_1.useCallback)(async () => {

package/ios/Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version:5.9
+// swift-tools-version:5.10
 //
 // Package.swift — SwiftPM manifest used **only for command-line testing**
 // of the algorithm layer (QualityChecker.swift).  Production builds
@@ -38,26 +38,40 @@ let package = Package(
     .target(
       name: "RNImageStitcher",
       path: "Sources/RNImageStitcher",
-      // Excluded from `swift test` because they depend on either
-      // React (which isn't a SwiftPM dep) or OpenCV (which only
-      // ships as an iOS XCFramework via the podspec — no macOS
-      // build).  The host app's CocoaPods workspace picks them up.
-      exclude: [
-        // React-dependent
-        "QualityCheckerBridge.swift",
-        "QualityCheckerBridge.m",
-        "StitcherBridge.swift",
-        "StitcherBridge.m",
-        // OpenCV-dependent (Phase 2 stitcher)
-        "OpenCVStitcher.h",
-        "OpenCVStitcher.mm",
-        // OpenCV-dependent (V16 Phase 1 keyframe collector)
-        "OpenCVKeyframeCollector.h",
-        "OpenCVKeyframeCollector.mm",
-        // Stitcher.swift is `#if canImport(UIKit)`-gated so it
-        // compiles to nothing on macOS; including it keeps the
-        // file available to the Pods build without breaking
-        // `swift test`.
+      // F8.3.H2-target — instead of an `exclude` list (which broke
+      // every time a new .mm landed, e.g.
+      // `KeyframeGateFrameProcessor.mm` in F8.1, because SwiftPM
+      // still scans the directory and rejects "mixed language
+      // source files" if it sees both .swift and .mm), we use an
+      // explicit `sources` allowlist of files that compile cleanly
+      // on macOS (where `swift test` runs).
+      //
+      // What's in the allowlist:
+      //   * QualityChecker.swift — Accelerate / CoreImage; macOS-OK.
+      //   * KeyframeGate.swift — Foundation + simd; macOS-OK.
+      //
+      // What's NOT (intentionally):
+      //   * Anything with `import UIKit` / `import ARKit` — iOS only.
+      //     CocoaPods compiles them for the host app via the podspec
+      //     source_files glob; SwiftPM macOS doesn't need them.
+      //   * .mm / .m / .h files — same.  Picked up by CocoaPods.
+      //   * RN-bridge Swift files (`*Bridge.swift`) — `import React`,
+      //     not a SwiftPM dep.
+      //
+      // The Frame Processor plugin's Swift⇄ObjC selector pin
+      // (formerly relied on by `FrameProcessorPluginSelectorTests`)
+      // is enforced as a compile-time `#selector(...)` reference
+      // inside `IncrementalStitcher.swift` itself — see the
+      // `_consumeFrameFromPluginSelectorPin` static.  Drift breaks
+      // the SDK build, which is a stronger guarantee than a test
+      // that needs iOS-Simulator infrastructure to run.
+      sources: [
+        "QualityChecker.swift",
+        // KeyframeGate.swift depends on `KeyframeGateBridge` (ObjC
+        // class in .mm) and `RNSARFramePose` (from a UIKit-using
+        // Swift file), so it doesn't compile standalone under
+        // SwiftPM on macOS — only the CocoaPods build sees the
+        // full type graph.
       ]
     ),
     .testTarget(

package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift

@@ -476,6 +476,13 @@ public final class IncrementalStitcher: NSObject {
 
     private override init() {
         super.init()
+        // F8.3.H2 — runtime check that Swift's auto-bridged ObjC
+        // selector for `consumeFrameFromPlugin(...)` matches the
+        // selector string the plugin's .mm dispatches.  Asserts in
+        // dev builds; no-ops in release.  See the
+        // `_consumeFrameFromPluginSelectorPin` declaration below for
+        // the full rationale.
+        IncrementalStitcher._verifyConsumeFrameFromPluginSelector()
     }
 
     /// 2026-05-18 (iOS cross-orientation fix) — bridge entry-point
@@ -3158,4 +3165,57 @@ extension IncrementalStitcher {
         )
         consumeFrame(pixelBuffer: pixelBuffer, pose: pose)
     }
+
+    // F8.3.H2 — compile-time + runtime guard for the Swift⇄ObjC
+    // selector contract that `KeyframeGateFrameProcessor.mm`
+    // depends on.
+    //
+    // The .mm file forward-declares `IncrementalStitcher` and
+    // dispatches `[shared consumeFrameFromPluginWithPixelBuffer:tx:
+    // …:trackingStateRaw:]` by NAME — ObjC's late-binding means
+    // signature drift would silently link but crash at runtime
+    // with `NSInvalidArgumentException: unrecognized selector`
+    // on the first non-AR frame.
+    //
+    // This `#selector(...)` reference forces the Swift compiler
+    // to resolve the exact method signature.  If anyone renames a
+    // parameter label or adds/removes an argument, the
+    // `_consumeFrameFromPluginSelectorPin` expression fails to
+    // compile — the SDK won't build until the .mm's forward
+    // declaration is updated to match.  Stronger guarantee than a
+    // test that needs iOS-Simulator infrastructure to run.
+    //
+    // The runtime check below additionally pins the exact
+    // SELECTOR STRING the .mm dispatches.  In dev/debug builds it
+    // asserts; in release builds it's a no-op (the static let is
+    // initialised lazily and never read otherwise, so the runtime
+    // cost is one-time + tiny).  Drift between Swift's auto-
+    // generated selector name and the .mm's expected string
+    // (e.g., if Swift's bridging rules change) trips the assert.
+    private static let _consumeFrameFromPluginSelectorPin: Selector =
+        #selector(IncrementalStitcher.consumeFrameFromPlugin(
+            pixelBuffer:
+            tx: ty: tz:
+            qx: qy: qz: qw:
+            fx: fy: cx: cy:
+            imageWidth: imageHeight:
+            timestampMs:
+            trackingStateRaw:))
+
+    @inline(never)
+    private static func _verifyConsumeFrameFromPluginSelector() {
+        let expected =
+            "consumeFrameFromPluginWithPixelBuffer:tx:ty:tz:"
+            + "qx:qy:qz:qw:fx:fy:cx:cy:"
+            + "imageWidth:imageHeight:timestampMs:trackingStateRaw:"
+        let actual = NSStringFromSelector(_consumeFrameFromPluginSelectorPin)
+        assert(
+            actual == expected,
+            "Frame Processor selector drift — Swift's auto-bridged "
+            + "ObjC selector for consumeFrameFromPlugin is "
+            + "\(actual) but KeyframeGateFrameProcessor.mm's "
+            + "forward declaration expects \(expected).  Update the "
+            + ".mm to match (or fix the assumption here).",
+        )
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-image-stitcher",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/camera/Camera.tsx

@@ -235,6 +235,29 @@ export interface CameraProps {
   showSettingsButton?: boolean;
   style?: StyleProp<ViewStyle>;
 
+  /**
+   * Which incremental stitcher engine to drive.  Default
+   * `'batch-keyframe'` — collects accepted JPEGs and runs
+   * `cv::Stitcher` once at finalize time.  This is the v0.4+
+   * production default and what the v0.5 Frame Processor migration
+   * exercises.
+   *
+   * Switch to a live engine (`'firstwins-rectilinear'` or
+   * `'hybrid'`) for low-latency in-flight stitching.  Live engines
+   * exercise the F8.6 pixel-buffer ingest path (skipping the JPEG
+   * encode/decode round-trip; ~30–50 ms saved per accept) when the
+   * Frame Processor driver is active.
+   *
+   * See `docs/f8-frame-processor-plan.md` and the v0.5.0
+   * CHANGELOG for the trade-offs between batch-keyframe and live
+   * engines.
+   */
+  engine?: 'batch-keyframe'
+    | 'hybrid'
+    | 'slitscan-rotate' | 'slitscan-both'
+    | 'firstwins' | 'firstwins-zoomed' | 'firstwins-rectilinear'
+    | 'slitscan';
+
   /**
    * Optional destination directory for captures.  When set, the lib
    * lands tap-photos at `${outputDir}/photo-${ts}.jpg` and panoramas
@@ -587,6 +610,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
     onError,
     frameProcessor: hostFrameProcessor,
     legacyDriver = false,
+    engine = 'batch-keyframe',
   } = props;
 
   const insets = useSafeAreaInsets();
@@ -1031,7 +1055,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
         composeHeight: 1080,
         canvasWidth: 5000,
         canvasHeight: 5000,
-        engine: 'batch-keyframe',
+        engine,
         config: panoramaSettingsToNativeConfig({
           ...settings,
           captureSource: effectiveCaptureSource,
@@ -1079,6 +1103,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
     jsDriver,
     fpDriver,
     legacyDriver,
+    engine,
     onError,
   ]);