react-native-image-stitcher 0.7.1 → 0.9.0

package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt

@@ -1001,14 +1001,17 @@ class IncrementalStitcher(
         // per accepted frame on a mid-tier device.  Pass null to use
         // the legacy JPEG path.
         //
-        // OWNERSHIP: the engine retains a reference to `nv21PixelData`
-        // until `workScope`'s coroutine consumes it (~50 ms later).
-        // Callers MUST treat the array as transferred — do not
-        // mutate it or return it to a buffer pool after calling
-        // this method.  If a caller needs to recycle the buffer,
-        // pass `.copyOf()` (currently no caller does — the F8.4
-        // Frame Processor plugin allocates a fresh array per frame).
-        nv21PixelData: ByteArray? = null,
+        // OWNERSHIP: wrapped in `TransferredNV21` (audit #4A,
+        // v0.10.0).  The wrapper enforces single-use: the engine
+        // calls `.takeOnce()` on the producer thread before
+        // dispatching to `workScope`; subsequent attempts to extract
+        // the bytes throw.  Callers MUST construct a fresh
+        // `TransferredNV21` per frame and MUST NOT hand the same
+        // instance to two consumers (e.g., a sync gate-eval + an
+        // async workScope.launch).  The Frame Processor plugin and
+        // the AR camera view both allocate fresh NV21 arrays per
+        // frame; the wrapper is a defensive-programming guard.
+        nv21PixelData: TransferredNV21? = null,
         nv21PixelWidth: Int = 0,
         nv21PixelHeight: Int = 0,
     ) {
@@ -1215,11 +1218,20 @@ class IncrementalStitcher(
             )
             return
         }
+        // v0.10.0 audit #4A — extract the wrapped bytes ONCE on the
+        // producer thread before dispatching to workScope.  This
+        // makes the transfer-of-ownership explicit + caught early:
+        // if a caller accidentally passes the same TransferredNV21
+        // instance to a sync consumer earlier, takeOnce() would
+        // have already thrown there.  Capturing `pixelBytes` by
+        // value inside the coroutine sidesteps any chance of the
+        // wrapper being read from two threads.
+        val pixelBytes: ByteArray? = if (hasPixelData) nv21PixelData!!.takeOnce() else null
         workScope.launch {
             val state: WritableMap? = if (firstwins != null) {
                 val tele = if (hasPixelData) {
                     firstwins.addFramePixelData(
-                        nv21 = nv21PixelData!!,
+                        nv21 = pixelBytes!!,
                         nv21Width = nv21PixelWidth,
                         nv21Height = nv21PixelHeight,
                         qx = qx, qy = qy, qz = qz, qw = qw,
@@ -1246,7 +1258,7 @@ class IncrementalStitcher(
             } else {
                 val tele = if (hasPixelData) {
                     hybrid!!.addFramePixelData(
-                        nv21 = nv21PixelData!!,
+                        nv21 = pixelBytes!!,
                         nv21Width = nv21PixelWidth,
                         nv21Height = nv21PixelHeight,
                         qx = qx, qy = qy, qz = qz, qw = qw,
@@ -1410,7 +1422,14 @@ class IncrementalStitcher(
             // `addFramePixelData` instead of JPEG-decoding a
             // separately-written path.  Batch-keyframe mode
             // ignores these (it uses `grayData` + `onAccept`).
-            nv21PixelData = nv21Bytes,
+            //
+            // v0.10.0 audit #4A — wrap in TransferredNV21 so the
+            // engine takes ownership exactly once on the producer
+            // thread (engine calls `.takeOnce()` before workScope).
+            // Misuse (handing this same instance to two consumers)
+            // throws at the second `.takeOnce()` site, not silently
+            // corrupting frames.
+            nv21PixelData = TransferredNV21(nv21Bytes),
             nv21PixelWidth = width,
             nv21PixelHeight = height,
             onAccept = { targetPath ->

package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt

@@ -55,19 +55,33 @@ class RNImageStitcherPackage : ReactPackage {
                 ) { proxy, options ->
                     CvFlowGateFrameProcessor(proxy, options)
                 }
+                // v0.9.0 Layer 1 — register `save_frame_as_jpeg`
+                // alongside the cv_flow_gate plugin.  Same lifecycle,
+                // same defensive error handling (the outer try/catch
+                // covers both registrations).  Either both register
+                // or neither does — if vc isn't on the classpath,
+                // both calls are skipped together.
+                FrameProcessorPluginRegistry.addFrameProcessorPlugin(
+                    SaveFrameAsJpegPlugin.PLUGIN_NAME,
+                ) { proxy, options ->
+                    SaveFrameAsJpegPlugin(proxy, options)
+                }
                 fpPluginRegistered = true
             } catch (e: NoClassDefFoundError) {
                 android.util.Log.i(
                     "RNImageStitcherPackage",
                     "vision-camera FrameProcessorPluginRegistry not on classpath — "
-                    + "skipping cv_flow_gate_process_frame plugin registration "
-                    + "(host app doesn't appear to use Frame Processors).",
+                    + "skipping cv_flow_gate_process_frame + save_frame_as_jpeg "
+                    + "plugin registration (host app doesn't appear to use "
+                    + "Frame Processors).",
                 )
                 fpPluginRegistered = true  // don't retry every package init
             } catch (e: Throwable) {
                 android.util.Log.w(
                     "RNImageStitcherPackage",
-                    "Failed to register cv_flow_gate_process_frame plugin: ${e.message}",
+                    "Failed to register Frame Processor plugins "
+                    + "(cv_flow_gate_process_frame / save_frame_as_jpeg): "
+                    + e.message,
                 )
                 fpPluginRegistered = true
             }
@@ -87,6 +101,10 @@ class RNImageStitcherPackage : ReactPackage {
             RNSARSession(reactContext),
             IncrementalStitcher(reactContext),
             FileBridge(reactContext),
+            // v0.8.0 Phase 4b.ii — Android JSI installer for the
+            // host-worklet `__stitcherProxy` global.  Mirror of
+            // iOS' `StitcherJsiInstaller`.
+            StitcherJsiInstallerModule(reactContext),
         )
     }
 

package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt

@@ -287,8 +287,20 @@ class RNSARCameraView @JvmOverloads constructor(
         // contract was already in place for Phase 4.
         appendPose(camera, frame.timestamp)
 
-        // Forward to the incremental stitcher if engaged.
-        if (ingestActive) {
+        // Forward to the incremental stitcher if engaged, OR if any
+        // host worklets are registered (v0.8.0 Phase 4b.iii).  iOS'
+        // `RNSARWorkletRuntime.dispatchFrame:pose:` fires on every
+        // AR frame regardless of capture state; Android needs the
+        // same semantic so host worklets see the AR-mode preview
+        // stream, not just capture frames.
+        //
+        // `hasHostWorklets()` is a microsecond atomic-read on the
+        // native registry — cheap enough to hit per frame.  When
+        // no host worklets are registered AND no capture is active,
+        // the entire forwardToIncremental branch (including the
+        // ~3-5ms NV21 pack) is skipped — same cost envelope as
+        // before Phase 4b.iii.
+        if (ingestActive || StitcherWorkletRuntime.hasHostWorklets()) {
             forwardToIncremental(frame, camera)
         }
 
@@ -514,6 +526,23 @@ class RNSARCameraView @JvmOverloads constructor(
         // (Was: eager JPEG encode for non-batch-keyframe modes,
         //  written to `tmpJpegFile`, passed as `legacyJpegPath`.
         //  See the v0.3 / F8.6 entries in CHANGELOG.md.)
+        //
+        // v0.8.0 Phase 3c — route through the worklet runtime's
+        // `runFirstParty` indirection.  The lambda body is the
+        // unchanged engine ingest call; the indirection sets up
+        // the seam where Phase 4 will fan out to host worklets
+        // without touching this first-party path.  Synchronous
+        // invocation preserves the ARCore Image ownership contract
+        // — the engine consumes the TransferredNV21 inside the
+        // lambda before ARCore recycles the Image.
+        StitcherWorkletRuntime.installIfNeeded()
+        // v0.8.0 Phase 4b.iii — only run first-party stitching when
+        // the host has actively engaged capture (`setIncrementalIngestionActive(true)`).
+        // The host-worklet dispatch below runs regardless, so AR-mode
+        // preview frames stream through registered host worklets even
+        // before/after capture.
+        if (ingestActive) {
+        StitcherWorkletRuntime.runFirstParty {
         module.ingestFromARCameraView(
             tx = tArr[0].toDouble(),
             ty = tArr[1].toDouble(),
@@ -538,7 +567,17 @@ class RNSARCameraView @JvmOverloads constructor(
             legacyJpegPath = null,
             // F8.6 — pixel-data path for live engines.  Batch-
             // keyframe mode ignores these (bails earlier).
-            nv21PixelData = packed.nv21,
+            //
+            // v0.10.0 audit #4A — wrap `packed.nv21` in
+            // TransferredNV21 so ownership is enforced at runtime.
+            // The AR caller passes the SAME `packed.nv21` array as
+            // both `grayData` (sync, gate-eval read) and
+            // `nv21PixelData` (async, engine ingest).  Today no race
+            // because grayData is consumed inside evaluateWithFrame
+            // before workScope.launch fires; the wrapper makes a
+            // future refactor that reorders consumption fail loudly
+            // instead of silently corrupting frames.
+            nv21PixelData = TransferredNV21(packed.nv21),
             nv21PixelWidth = packed.width,
             nv21PixelHeight = packed.height,
             onAccept = { targetPath ->
@@ -555,6 +594,42 @@ class RNSARCameraView @JvmOverloads constructor(
                 ) != null
             },
         )
+        }  // closes StitcherWorkletRuntime.runFirstParty { … } (v0.8.0 Phase 3c)
+        }  // closes `if (ingestActive)` (v0.8.0 Phase 4b.iii)
+
+        // ── v0.8.0 Phase 4b.iii — host-worklet fan-out ─────────────
+        //
+        // Dispatch the AR frame to every host worklet registered via
+        // `globalThis.__stitcherProxy.install(workletFn)` (the
+        // `useFrameProcessor` hook's AR-mode path).  The native side
+        // fast-path early-exits when the registry is empty (~ns
+        // cost), so this call is free for first-party-only deployments.
+        //
+        // Map the trackingState back to the JS-visible string set.
+        // `RNSARSession.TRACKING_*` are int codes; we re-derive the
+        // string here instead of plumbing it through.  (Could be
+        // refactored into a helper if/when other call sites need
+        // it.)
+        val trackingStateStr = when (camera.trackingState) {
+            TrackingState.TRACKING -> "normal"
+            TrackingState.PAUSED -> "limited"
+            TrackingState.STOPPED -> "notAvailable"
+            else -> ""
+        }
+        StitcherWorkletRuntime.dispatchToHostWorklets(
+            nv21Bytes = packed.nv21,
+            width = packed.width,
+            height = packed.height,
+            qx = qarr[0].toDouble(),
+            qy = qarr[1].toDouble(),
+            qz = qarr[2].toDouble(),
+            qw = qarr[3].toDouble(),
+            tx = tArr[0].toDouble(),
+            ty = tArr[1].toDouble(),
+            tz = tArr[2].toDouble(),
+            timestampNs = frame.timestamp.toDouble(),
+            trackingState = trackingStateStr,
+        )
     }
 
     private fun applyDisplayGeometry() {

package/android/src/main/java/io/imagestitcher/rn/SaveFrameAsJpegPlugin.kt

@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+package io.imagestitcher.rn
+
+import android.graphics.ImageFormat
+import android.media.Image
+import android.util.Log
+import androidx.annotation.Keep
+import com.facebook.proguard.annotations.DoNotStrip
+import com.mrousavy.camera.frameprocessors.Frame
+import com.mrousavy.camera.frameprocessors.FrameProcessorPlugin
+import com.mrousavy.camera.frameprocessors.VisionCameraProxy
+import io.imagestitcher.rn.ar.YuvImageConverter
+
+/**
+ * v0.9.0 Layer 1 — Android vc Frame Processor plugin that JPEG-
+ * encodes the supplied frame to a host-supplied path.  Mirror of
+ * iOS' `SaveFrameAsJpegPlugin.mm`.
+ *
+ * Plugin name (must match iOS): `save_frame_as_jpeg`.
+ *
+ * ## Wrapping the existing encoder
+ *
+ * The lib already encodes JPEGs from NV21 bytes via
+ * `YuvImageConverter.encodeJpegFromNV21` — that's the path
+ * `RNSARCameraView.kt`'s keyframe-accept callback uses (line 589,
+ * `onAccept = { targetPath -> ... encodeJpegFromNV21(...) }`).
+ * This plugin reuses that exact encoder so:
+ *   - JPEG output is byte-equivalent to the keyframe-accept output
+ *     (same encoder, same quality knob)
+ *   - No new encoder maintenance burden
+ *
+ * vision-camera's `Frame.image` is an `android.media.Image` in
+ * `YUV_420_888` format (the camera's native).  We pass it through
+ * `YuvImageConverter.packNV21(image)` to extract the dense NV21
+ * byte array + dims, then `encodeJpegFromNV21(packed, file, q, rot)`
+ * does the JPEG write.
+ *
+ * ## Plugin contract (matches iOS surface exactly)
+ *
+ * Arguments dict:
+ *   - `path` (string, REQUIRED): absolute output path.
+ *   - `quality` (number, optional): 0-100 JPEG quality.  Default 75.
+ *     Clamped to [1, 100].
+ *
+ * Returns:
+ *   - On success: `{ "ok" => true, "path" => ..., "width" => ...,
+ *                    "height" => ... }`
+ *   - On failure: `{ "ok" => false, "error" => "..." }`
+ *
+ * Errors surfaced via the result map (not thrown) — host worklets
+ * can branch on `result.ok` without try/catch.  Same convention
+ * as iOS.
+ *
+ * ## Lifetime / threading
+ *
+ * The supplied `Frame` (and its `Image`) is valid only for the
+ * duration of this callback — vision-camera closes the underlying
+ * `Image` on return.  All Image access (NV21 pack + JPEG encode)
+ * happens synchronously inside `callback()`.
+ *
+ * ## Format restriction
+ *
+ * Only `YUV_420_888` input is supported (vc Android's standard).
+ * Anything else returns `{ ok: false, error: "unsupported format" }`.
+ * No format conversion fallback — that would mask bugs in the host
+ * camera config.
+ *
+ * ## Registration
+ *
+ * Registered in `RNImageStitcherPackage.kt`'s companion-object
+ * `ensureFrameProcessorPluginRegistered()`, alongside
+ * `cv_flow_gate_process_frame`.  Same defensive
+ * NoClassDefFoundError handling — if vc isn't on the host's
+ * classpath, registration is silently skipped (and the plugin's
+ * `init { … }` calls below never happen).
+ */
+@DoNotStrip
+@Keep
+class SaveFrameAsJpegPlugin(
+    @Suppress("UNUSED_PARAMETER") proxy: VisionCameraProxy,
+    @Suppress("UNUSED_PARAMETER") options: Map<String, Any>?,
+) : FrameProcessorPlugin() {
+
+    override fun callback(frame: Frame, params: Map<String, Any>?): Any {
+        val path = params?.get("path") as? String
+            ?: return mapOf(
+                "ok" to false,
+                "error" to "missing required `path` argument",
+            )
+        val rawQuality = (params["quality"] as? Number)?.toInt() ?: 75
+        val quality = rawQuality.coerceIn(1, 100)
+
+        // Frame may throw if vc already released it.
+        val image: Image = try {
+            frame.image
+        } catch (e: Throwable) {
+            return mapOf(
+                "ok" to false,
+                "error" to "frame invalid: ${e.message}",
+            )
+        }
+
+        if (image.format != ImageFormat.YUV_420_888) {
+            return mapOf(
+                "ok" to false,
+                "error" to "unsupported format ${image.format} (need YUV_420_888)",
+            )
+        }
+
+        // Pack NV21 — same call site RNSARCameraView uses.
+        val packed = YuvImageConverter.packNV21(image)
+            ?: return mapOf(
+                "ok" to false,
+                "error" to "YuvImageConverter.packNV21 returned null",
+            )
+
+        // Reuse the lib's existing JPEG encoder.  Rotation is 0 here
+        // (the host's frame is in camera-native orientation; if they
+        // want display orientation they can pass an `orientation`
+        // arg in a future version — for v0.9.0 the worklet emits
+        // raw-camera-oriented JPEGs, matching the keyframe-accept
+        // pipeline's behaviour).
+        //
+        // Signature: `encodeJpegFromNV21(packed, outputPath: String,
+        // jpegQuality: Int, displayRotation: Int): String?` — returns
+        // the written path on success, null on failure.
+        val encodedPath: String? = try {
+            YuvImageConverter.encodeJpegFromNV21(
+                packed,
+                path,
+                jpegQuality = quality,
+                displayRotation = 0,
+            )
+        } catch (e: Throwable) {
+            return mapOf(
+                "ok" to false,
+                "error" to "encodeJpegFromNV21 threw: ${e.message}",
+            )
+        }
+        if (encodedPath == null) {
+            return mapOf(
+                "ok" to false,
+                "error" to "encodeJpegFromNV21 returned null",
+            )
+        }
+
+        return mapOf(
+            "ok" to true,
+            "path" to encodedPath,
+            "width" to packed.width,
+            "height" to packed.height,
+        )
+    }
+
+    companion object {
+        private const val TAG = "SaveFrameAsJpegPlugin"
+
+        /// Plugin name; MUST match iOS + the JS-side
+        /// `initFrameProcessorPlugin('save_frame_as_jpeg')` call.
+        const val PLUGIN_NAME = "save_frame_as_jpeg"
+    }
+}

package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt

@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: Apache-2.0
+package io.imagestitcher.rn
+
+import android.util.Log
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReactContextBaseJavaModule
+import com.facebook.react.bridge.ReactMethod
+
+/**
+ * v0.8.0 Phase 4b.ii — Android-side JSI installer for the host
+ * worklet proxy.  Mirror of iOS' `StitcherJsiInstaller`.
+ *
+ * The module exposes one synchronous method, `install()`, which JS
+ * calls once at lib bootstrap (via the
+ * `ensureStitcherProxyInstalled` helper in
+ * `src/stitching/ensureStitcherProxyInstalled.ts`).  We reach into
+ * the main JS runtime via `ReactApplicationContext.getJavaScriptContextHolder().get()`
+ * — the canonical bridgeless-compatible accessor in modern RN
+ * (worklets-core's `WorkletsModule` uses the same pattern, verified
+ * working on RN 0.84.1 + new arch + Hermes).
+ *
+ * The native `nativeInstall(jsiRuntimeRef)` JNI then casts the long
+ * back to a `jsi::Runtime*` and calls into the shared C++
+ * `retailens::installStitcherProxy(runtime)` (in
+ * `cpp/stitcher_proxy_jsi.{hpp,cpp}`).  Identical destination on
+ * both platforms — `globalThis.__stitcherProxy` exposes the same
+ * `install` / `uninstall` / `count` host functions.
+ *
+ * ## Returning `Boolean` (not `Promise`) from a sync method
+ *
+ * `isBlockingSynchronousMethod = true` + `Boolean` return is the
+ * documented pattern for "I'm doing one-shot native setup that
+ * needs to complete before the next JS line runs."  Same shape as
+ * `WorkletsModule.install()`.
+ *
+ * ## What we DON'T do here (Phase 4b.ii follow-up)
+ *
+ * Phase 4b.ii's MVP installs the proxy ONLY.  Host worklets that
+ * register through `__stitcherProxy.install` land in the native
+ * `retailens::StitcherWorkletRegistry`.  Per-frame fan-out from
+ * Android's `StitcherWorkletRuntime` is a separate piece of work
+ * (Phase 4b.ii follow-up) — needs the Kotlin↔JNI bridge that
+ * constructs a `StitcherFrameJsiHostObject` from an `ArImage` +
+ * pose and posts it through a worklet runtime.  Until that lands,
+ * Android-registered worklets behave exactly like iOS-registered
+ * worklets BEFORE Phase 4b.i: they exist in the registry but
+ * aren't invoked.
+ *
+ * The proxy install itself is still useful as a foundation —
+ * verifies the JNI handshake works, exercises the bridgeless
+ * runtime accessor, and gives us a `count()` smoke test for the
+ * device verification step.
+ */
+class StitcherJsiInstallerModule(
+    private val reactContext: ReactApplicationContext,
+) : ReactContextBaseJavaModule(reactContext) {
+    override fun getName(): String = NAME
+
+    @ReactMethod(isBlockingSynchronousMethod = true)
+    fun install(): Boolean {
+        return try {
+            // `getJavaScriptContextHolder().get()` returns a raw
+            // `jsi::Runtime*` boxed as `Long`.  Same accessor
+            // worklets-core's `WorkletsModule.install()` uses;
+            // documented to work in both legacy + bridgeless modes
+            // on RN 0.71+.
+            val holder = reactContext.javaScriptContextHolder
+            if (holder == null) {
+                Log.e(TAG, "getJavaScriptContextHolder() returned null; runtime unreachable")
+                return false
+            }
+            val runtimeRef = holder.get()
+            if (runtimeRef == 0L) {
+                Log.e(TAG, "JavaScriptContextHolder.get() returned 0; runtime not initialized yet")
+                return false
+            }
+            val ok = nativeInstall(runtimeRef)
+            if (!ok) {
+                Log.e(TAG, "nativeInstall(runtimeRef=$runtimeRef) returned false")
+            }
+            ok
+        } catch (t: Throwable) {
+            Log.e(TAG, "install() threw — falling back to JS-side registry", t)
+            false
+        }
+    }
+
+    private external fun nativeInstall(jsiRuntimeRef: Long): Boolean
+
+    companion object {
+        const val NAME = "StitcherJsiInstaller"
+        private const val TAG = "StitcherJsiInstaller"
+
+        init {
+            // The Phase 3a JNI shim (`libimage_stitcher.so`) absorbed
+            // the JSI-install JNI binding from Phase 4b.ii.  Loading
+            // it once is enough — Android's loader deduplicates,
+            // so even if `IncrementalStitcher.kt`'s init block
+            // already loaded the lib, calling again is a cheap no-op.
+            System.loadLibrary("image_stitcher")
+        }
+    }
+}