react-native-image-stitcher 0.18.0 → 0.20.0

package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt

@@ -11,7 +11,9 @@ import android.util.Log
 import android.view.Surface
 import android.view.WindowManager
 import android.widget.FrameLayout
+import com.google.ar.core.Anchor
 import com.google.ar.core.Camera
+import com.google.ar.core.Pose
 import com.google.ar.core.Session
 import com.google.ar.core.TrackingState
 import com.google.ar.core.exceptions.CameraNotAvailableException
@@ -78,6 +80,38 @@ class RNSARCameraView @JvmOverloads constructor(
     private val backgroundRenderer = BackgroundRenderer()
     private val sessionRef = AtomicReference<Session?>(null)
     private var sessionTextureBound = false
+
+    // ── 0.20.0 — AR overlay/annotation renderer ─────────────────────────
+    //
+    // [overlayStore] is the single source of truth for the overlays to
+    // draw (UNION of JS-set + native-plugin-set; see [AROverlayStore]);
+    // [overlayRenderer] is the transparent Canvas View stacked ABOVE the
+    // GLSurfaceView in this FrameLayout.  Per frame, [onDrawFrame] snapshots
+    // the camera view/projection matrices + the letterbox box and pushes
+    // them into the renderer (which reprojects + redraws on the UI thread).
+    //
+    // The store is exposed (via [overlayStore] internal) so the native
+    // plugin path ([RNSARPluginRegistry.setOverlays] etc.) and the JS
+    // imperative path ([RNSARSession] @ReactMethods → bound view) both write
+    // to it; the JS imperative path uses the JS namespace, plugins the
+    // plugin namespace.
+    val overlayStore = AROverlayStore()
+    private val overlayRenderer = AROverlayRenderer(context, overlayStore)
+
+    // v0.20.0 — one ARCore Anchor per world-anchored overlay (parity with
+    // iOS' ARAnchor) so ARCore refines the pose against drift / re-localization
+    // instead of trusting a frozen world coordinate.  GL-thread only (created /
+    // read / detached inside [reconcileOverlayAnchors] from onDrawFrame).
+    private val overlayAnchors = HashMap<String, Anchor>()
+    // The source world point each anchor was created at — used to detect when
+    // an overlay's position changed (JS re-set it) and recreate the anchor.
+    private val overlayAnchorSrc = HashMap<String, FloatArray>()
+
+    // Scratch matrices for the per-frame overlay camera snapshot — reused
+    // each frame so the GL thread does no per-frame allocation when overlays
+    // are active.  ARCore returns COLUMN-MAJOR (OpenGL) matrices.
+    private val overlayViewMatrix = FloatArray(16)
+    private val overlayProjMatrix = FloatArray(16)
     /// Last known display rotation; consulted on each setDisplayGeometry
     /// call so we can recompute when the user rotates the device.
     private var lastDisplayRotation: Int = -1
@@ -131,6 +165,21 @@ class RNSARCameraView @JvmOverloads constructor(
         glView.requestRender()
     }
 
+    /// Pending crosshair raycast (v0.20.0).  Fulfilled on the next render
+    /// tick with the live ARCore frame — `Frame.hitTest` must run on the GL
+    /// thread, so a JS `raycast()` call can't resolve synchronously.
+    private val pendingRaycast =
+        AtomicReference<com.facebook.react.bridge.Promise?>(null)
+
+    /// Called from the bridge (RNSARSession.raycast @ReactMethod).  Stores a
+    /// promise fulfilled on the next render tick by hitTest from the screen
+    /// centre.  Supersedes any queued raycast.
+    internal fun requestRaycast(promise: com.facebook.react.bridge.Promise) {
+        val previous = pendingRaycast.getAndSet(promise)
+        previous?.resolve(null) // superseded → null (caller falls back)
+        glView.requestRender()
+    }
+
     init {
         // Black background avoids a flash before the GL surface starts
         // clearing itself black each frame (the GL-level letterbox draws
@@ -140,6 +189,14 @@ class RNSARCameraView @JvmOverloads constructor(
             glView,
             LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT),
         )
+        // 0.20.0 — overlay renderer stacked ABOVE the GLSurfaceView (added
+        // after glView so it draws on top).  Full-screen + transparent; it
+        // reprojects world overlays into the same letterbox box the camera
+        // feed fills, so its coordinate space matches the preview.
+        addView(
+            overlayRenderer,
+            LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT),
+        )
     }
 
     override fun onAttachedToWindow() {
@@ -215,6 +272,11 @@ class RNSARCameraView @JvmOverloads constructor(
         // Pause the GL thread so we stop drawing frames.
         glView.onPause()
         sessionTextureBound = false
+        // 0.20.0 — stop drawing overlays; we drop the camera snapshot so a
+        // stale pose isn't reused if the view re-attaches.  The overlay SET
+        // itself is left intact (the host may keep the same overlays across
+        // a remount); only the per-frame camera state is cleared.
+        overlayRenderer.clear()
         IncrementalStitcher.bridgeInstance?.unbindArCameraView(this)
         RNSARSession.instance?.unbindCameraView(this)
         // iOS parity (didMoveToWindow else-branch): stop the session
@@ -398,28 +460,58 @@ class RNSARCameraView @JvmOverloads constructor(
         // contract was already in place for Phase 4.
         appendPose(camera, frame.timestamp)
 
-        // onArFrame (v0.18.0) — LIGHT AR-metadata event channel.  Built
-        // + emitted INDEPENDENTLY of the stitcher ingest / host-worklet
-        // fan-out below: a host that only wants per-frame AR metadata
-        // (no capture, no worklet) still gets it.  Gated + throttled
-        // internally; near-free (one volatile read + one nanoTime
-        // compare) when disabled or inside the throttle window.
-        maybeEmitArFrameMeta(frame, camera)
+        // ── 0.20.0 — feed the overlay renderer this frame's camera ───────
+        //
+        // Only when overlays exist (cheap AtomicReference emptiness check),
+        // snapshot the view + projection matrices and the letterbox box so
+        // the overlay View can reproject world points → screen and redraw.
+        // Runs every render frame so overlays track at display rate.
+        //
+        // First reconcile ARCore anchors + publish their drift-corrected poses
+        // so the renderer projects the refined positions, not frozen coords.
+        reconcileOverlayAnchors(session)
+        maybeUpdateOverlayCamera(camera, box)
 
         // Forward to the incremental stitcher when capture is engaged,
         // OR when an AR frame-processor host worklet is registered (the
         // v0.8.0 Phase 4b.iii fan-out forwards preview frames whenever
         // host worklets exist, even with capture off — the host worklet
-        // observes the live AR stream).  `forwardToIncremental` does the
+        // observes the live AR stream), OR when a native AR plugin is
+        // registered (0.19.0 — `forwardToIncremental` builds the
+        // ARFrameContext + runs the plugins; their SYNC results are stashed
+        // for the onArFrame meta below).  `forwardToIncremental` does the
         // NV21 pack once and gates the first-party ingest internally on
-        // `ingestActive`; the host-worklet dispatch is gated on the
-        // native registry count.  `hasHostWorklets()` is a cheap atomic
-        // read (microseconds) so the common capture-off / no-worklet
-        // preview path stays near-free.
-        if (ingestActive || StitcherWorkletRuntime.hasHostWorklets()) {
+        // `ingestActive`; the host-worklet dispatch is gated on the native
+        // worklet registry count; the plugin invocation on the plugin
+        // registry.  All three checks are cheap atomic reads so the common
+        // idle preview path (no capture, no worklet, no plugin) stays
+        // near-free.
+        //
+        // ORDER (0.19.0): run forwardToIncremental BEFORE maybeEmitArFrameMeta
+        // so the native-plugin SYNC results computed in the former are
+        // available to fold into the onArFrame `plugins` field built in the
+        // latter (same render frame).
+        if (ingestActive ||
+            StitcherWorkletRuntime.hasHostWorklets() ||
+            !RNSARPluginRegistry.isEmpty()
+        ) {
             forwardToIncremental(frame, camera)
+        } else {
+            // No consumer this frame — make sure last frame's stashed plugin
+            // sync results don't leak into a later onArFrame meta.
+            lastPluginSyncResults = null
         }
 
+        // onArFrame (v0.18.0) — LIGHT AR-metadata event channel.  Built
+        // + emitted INDEPENDENTLY of the stitcher ingest / host-worklet
+        // fan-out above: a host that only wants per-frame AR metadata
+        // (no capture, no worklet) still gets it.  Gated + throttled
+        // internally; near-free (one volatile read + one nanoTime
+        // compare) when disabled or inside the throttle window.  Native-
+        // plugin SYNC results (0.19.0) stashed by forwardToIncremental
+        // above ride along under the meta's `plugins` field.
+        maybeEmitArFrameMeta(frame, camera)
+
         // takePhoto consumer — runs on EVERY render tick (not just
         // when ingest is active), since the host calls takePhoto in
         // photo mode where ingest is off.  No-op when no request is
@@ -427,6 +519,65 @@ class RNSARCameraView @JvmOverloads constructor(
         pendingTakePhoto.getAndSet(null)?.let { req ->
             fulfilTakePhoto(frame, req)
         }
+
+        // raycast consumer (v0.20.0) — same pattern: a JS raycast() request
+        // is fulfilled here with the live frame's hitTest from the crosshair.
+        pendingRaycast.getAndSet(null)?.let { promise ->
+            fulfilRaycast(frame, camera, promise)
+        }
+    }
+
+    /// Raycast from the screen-centre crosshair to the nearest real-world
+    /// surface; resolve `{ worldPosition: [x,y,z] }` (or null when nothing is
+    /// hit / not tracking).  Runs on the GL render thread from onDrawFrame.
+    /// hitTest coordinates are in the `setDisplayGeometry` space — which we
+    /// feed the LETTERBOX BOX — so the crosshair is the box centre.  Mirrors
+    /// iOS `RNSARSession.raycast`; the JS controller falls back to a fixed
+    /// 1 m-ahead point when this resolves null.
+    private fun fulfilRaycast(
+        frame: com.google.ar.core.Frame,
+        camera: Camera,
+        promise: com.facebook.react.bridge.Promise,
+    ) {
+        try {
+            if (camera.trackingState != TrackingState.TRACKING) {
+                promise.resolve(null)
+                return
+            }
+            val box = letterboxBox()
+            val cx = box[2] / 2f // box width  / 2
+            val cy = box[3] / 2f // box height / 2
+            val hits = frame.hitTest(cx, cy)
+            // hits are sorted near→far.  Prefer a plane hit inside the
+            // detected polygon, then a depth / feature point, then any hit.
+            val hit = hits.firstOrNull { h ->
+                when (val t = h.trackable) {
+                    is com.google.ar.core.Plane ->
+                        t.trackingState == TrackingState.TRACKING &&
+                            t.isPoseInPolygon(h.hitPose)
+                    is com.google.ar.core.DepthPoint -> true
+                    is com.google.ar.core.Point -> true
+                    else -> false
+                }
+            } ?: hits.firstOrNull()
+            if (hit == null) {
+                promise.resolve(null)
+                return
+            }
+            val p = hit.hitPose
+            val wp = com.facebook.react.bridge.Arguments.createArray().apply {
+                pushDouble(p.tx().toDouble())
+                pushDouble(p.ty().toDouble())
+                pushDouble(p.tz().toDouble())
+            }
+            val result = com.facebook.react.bridge.Arguments.createMap().apply {
+                putArray("worldPosition", wp)
+            }
+            promise.resolve(result)
+        } catch (t: Throwable) {
+            Log.w(TAG, "raycast failed: ${t.message}")
+            promise.resolve(null)
+        }
     }
 
     /// Capture the current ARCore frame to JPEG and resolve / reject
@@ -529,6 +680,163 @@ class RNSARCameraView @JvmOverloads constructor(
         RNSARSession.instance?.updateTrackingState(camera.trackingState)
     }
 
+    /// v0.20.0 — keep one ARCore Anchor per world-anchored overlay so ARCore
+    /// refines its pose against drift / re-localization (parity with iOS'
+    /// ARAnchor).  Runs on the GL thread each frame: create anchors for new
+    /// overlays, recreate when an overlay's source pose changes, detach removed
+    /// ones, then publish the live anchor positions to the renderer (which uses
+    /// them instead of the frozen world coordinates).  Cheap no-op when there
+    /// are neither overlays nor lingering anchors.
+    private fun reconcileOverlayAnchors(session: Session) {
+        if (overlayStore.isEmpty() && overlayAnchors.isEmpty()) return
+
+        val overlays = overlayStore.snapshot()
+        val live = HashSet<String>(overlays.size)
+        val positions = HashMap<String, FloatArray>(overlays.size)
+
+        for (o in overlays) {
+            val target = anchorTarget(o) ?: continue
+            live.add(o.id)
+            val prev = overlayAnchorSrc[o.id]
+            if (prev == null || !prev.contentEquals(target)) {
+                // New overlay, or its position changed → (re)create the anchor.
+                overlayAnchors.remove(o.id)?.detach()
+                val anchor = try {
+                    session.createAnchor(
+                        Pose(target, floatArrayOf(0f, 0f, 0f, 1f)),
+                    )
+                } catch (t: Throwable) {
+                    Log.w(TAG, "createAnchor failed for '${o.id}': ${t.message}")
+                    null
+                }
+                if (anchor != null) {
+                    overlayAnchors[o.id] = anchor
+                    overlayAnchorSrc[o.id] = target.copyOf()
+                } else {
+                    overlayAnchorSrc.remove(o.id)
+                }
+            }
+            // Publish the live (tracking) anchor pose, else the source pose.
+            val a = overlayAnchors[o.id]
+            positions[o.id] = if (a != null && a.trackingState == TrackingState.TRACKING) {
+                val p = a.pose
+                floatArrayOf(p.tx(), p.ty(), p.tz())
+            } else {
+                overlayAnchorSrc[o.id] ?: target
+            }
+        }
+
+        // Detach anchors for overlays that no longer exist.
+        if (overlayAnchors.keys.any { it !in live }) {
+            for (id in overlayAnchors.keys.filter { it !in live }) {
+                overlayAnchors.remove(id)?.detach()
+                overlayAnchorSrc.remove(id)
+            }
+        }
+
+        overlayRenderer.setAnchorPositions(positions)
+    }
+
+    /// The world point to anchor an overlay at: its `worldPosition`, or the
+    /// centroid of its `worldQuad`.  null when it has no world geometry.
+    private fun anchorTarget(o: AROverlayData): FloatArray? {
+        o.worldPosition?.let { return it }
+        val q = o.worldQuad ?: return null
+        if (q.isEmpty()) return null
+        var cx = 0f; var cy = 0f; var cz = 0f
+        for (v in q) { cx += v[0]; cy += v[1]; cz += v[2] }
+        val n = q.size.toFloat()
+        return floatArrayOf(cx / n, cy / n, cz / n)
+    }
+
+    // ── 0.20.0 — per-frame overlay camera snapshot + JS imperative API ──
+
+    /**
+     * Snapshot this frame's camera view + projection matrices and the
+     * letterbox box into the [overlayRenderer], then trigger a redraw.
+     *
+     * Cheap no-op when no overlays are set (single AtomicReference
+     * emptiness check) so the common no-overlay preview path pays almost
+     * nothing.  ARCore's `getViewMatrix` / `getProjectionMatrix` are
+     * COLUMN-MAJOR (OpenGL) — exactly what `android.opengl.Matrix` and the
+     * renderer expect.  The near/far planes (0.05 m / 100 m) bound depth
+     * precision; they don't affect XY projection.
+     *
+     * @param camera the ARCore camera for this frame (pose + intrinsics).
+     * @param glBox  the letterbox box [x, y, w, h] in GL pixel space
+     *               (origin BOTTOM-left, as used by `glViewport`).  We flip
+     *               Y to the overlay View's TOP-left origin here.
+     */
+    private fun maybeUpdateOverlayCamera(camera: Camera, glBox: IntArray) {
+        if (overlayStore.isEmpty()) return
+
+        // ARCore projection matrix: near=0.05 m, far=100 m (matches the
+        // BackgroundRenderer's typical range; depth bounds only).
+        try {
+            camera.getViewMatrix(overlayViewMatrix, 0)
+            camera.getProjectionMatrix(overlayProjMatrix, 0, 0.05f, 100f)
+        } catch (t: Throwable) {
+            // Camera not ready (early frames) — skip this frame's overlay
+            // update; the feed keeps drawing and we retry next frame.
+            return
+        }
+
+        // GL viewport box → overlay View box.  GL origin is bottom-left;
+        // the View is top-left.  Both share surfaceWidth × surfaceHeight.
+        val boxX = glBox[0].toFloat()
+        val boxW = glBox[2].toFloat()
+        val boxH = glBox[3].toFloat()
+        val boxYTop = (surfaceHeight - (glBox[1] + glBox[3])).toFloat()
+
+        val tracking = camera.trackingState == TrackingState.TRACKING
+
+        overlayRenderer.updateCamera(
+            viewMatrix = overlayViewMatrix,
+            projectionMatrix = overlayProjMatrix,
+            boxX = boxX,
+            boxY = boxYTop,
+            boxW = boxW,
+            boxH = boxH,
+            tracking = tracking,
+        )
+    }
+
+    // ── JS imperative overlay API (forwarded from RNSARSession) ──────────
+    //
+    // The JS `ARCameraViewHandle` / `<Camera>` ref methods (setOverlays /
+    // addOverlay / updateOverlay / removeOverlay / clearOverlays) route
+    // through the singleton `RNSARSession` native module — the SAME idiom
+    // takePhoto uses — which forwards to the bound view's [overlayStore]
+    // (JS namespace).  These run on the bridge thread; the store's
+    // AtomicReferences make that safe against the GL thread's per-frame
+    // read.  An overlay set change triggers an immediate redraw so the
+    // overlay appears without waiting for the next AR frame's snapshot.
+
+    internal fun setOverlaysFromJs(overlays: List<AROverlayData>) {
+        overlayStore.setJsOverlays(overlays)
+        overlayRenderer.postInvalidateOnAnimation()
+    }
+
+    internal fun addOverlayFromJs(overlay: AROverlayData) {
+        overlayStore.addJsOverlay(overlay)
+        overlayRenderer.postInvalidateOnAnimation()
+    }
+
+    internal fun updateOverlayFromJs(id: String, patch: com.facebook.react.bridge.ReadableMap) {
+        overlayStore.updateJsOverlay(id, patch)
+        overlayRenderer.postInvalidateOnAnimation()
+    }
+
+    internal fun removeOverlayFromJs(id: String) {
+        overlayStore.removeJsOverlay(id)
+        overlayRenderer.postInvalidateOnAnimation()
+    }
+
+    internal fun clearOverlaysFromJs() {
+        overlayStore.clearJsOverlays()
+        overlayRenderer.postInvalidateOnAnimation()
+    }
+
     /// P3-G diagnostic — rate-limit the per-frame log so we can see
     /// at-a-glance whether forwardToIncremental is even running, vs
     /// being short-circuited at the `if (ingestActive)` guard in
@@ -813,6 +1121,132 @@ class RNSARCameraView @JvmOverloads constructor(
             anchorAlignments = anchorAlignments,
             anchorExtents = anchorExtents,
         )
+
+        // ── 0.19.0 — native AR-plugin per-frame invocation ───────────────
+        //
+        // Mirror of iOS' RNSARSession.session(_:didUpdate:) plugin loop.
+        // Only build the ARFrameContext + call plugins when the registry is
+        // NON-EMPTY (the onDrawFrame gate already let us in via that check,
+        // but a worklet-only frame can reach here with an empty plugin
+        // registry — re-check so those frames pay nothing).  Runs on the AR
+        // (GL render) thread, synchronously.  Reuses the already-packed
+        // `packed.nv21`, the depth/anchors collected above, and the pose +
+        // intrinsics already read — no extra Image acquire, no second pack.
+        // Depth is passed ONLY when the host opted into enableDepth (a
+        // mesh-only host acquired depth for its mesh, but the contract says
+        // the context's depth is null unless enableDepth).
+        runArPlugins(
+            packed, qarr, tArr, arTracking, frame, intrinsics, anchors,
+            depth = if (flags.depth) depth else null,
+        )
+    }
+
+    /// 0.19.0 — last frame's native-plugin SYNC results, keyed by plugin
+    /// name.  Written by [runArPlugins] on the GL render thread, read by
+    /// [maybeEmitArFrameMeta] on the same thread one step later in the same
+    /// onDrawFrame tick.  Null = no plugins ran / no sync results this
+    /// frame.  Single-threaded handoff (both on the GL thread) so no
+    /// synchronisation is needed, but @Volatile is cheap insurance against
+    /// any future cross-thread read.
+    @Volatile
+    private var lastPluginSyncResults: Map<String, Any?>? = null
+
+    /**
+     * 0.19.0 — build one [ARFrameContext] from the current frame and invoke
+     * every registered [ARFramePlugin].
+     *
+     *  - Non-null SYNC results are collected into a `{ name -> result }` map
+     *    and stashed in [lastPluginSyncResults] for [maybeEmitArFrameMeta]
+     *    to fold into the onArFrame `plugins` field this same tick.
+     *  - A plugin returning `null` defers to the ASYNC channel
+     *    ([RNSARPluginRegistry.emit] → `RNImageStitcherARPluginResult`).
+     *
+     * A throwing plugin is isolated (logged, skipped) so one bad plugin
+     * can't take down the AR render loop.
+     *
+     * Reuses caller-collected data (no extra Image work):
+     * @param packed     the already-packed NV21 camera image.
+     * @param qarr       pose rotation quaternion [x,y,z,w].
+     * @param tArr       pose translation [x,y,z] (world metres).
+     * @param tracking   contract tracking string ("normal"|"limited"|"notAvailable").
+     * @param frame      the ARCore frame (for the timestamp).
+     * @param intrinsics camera intrinsics (fx,fy,cx,cy + image dims).
+     * @param anchors    anchor descriptors already collected for onArFrame
+     *                   (enableAnchors-gated; empty otherwise).
+     * @param depth      row-packed DEPTH16 or null (enableDepth-gated).
+     */
+    private fun runArPlugins(
+        packed: YuvImageConverter.PackedYuv,
+        qarr: FloatArray,
+        tArr: FloatArray,
+        tracking: String,
+        frame: com.google.ar.core.Frame,
+        intrinsics: com.google.ar.core.CameraIntrinsics,
+        anchors: List<ArAnchorData>,
+        depth: ArDepthData?,
+    ) {
+        val plugins = RNSARPluginRegistry.plugins()
+        if (plugins.isEmpty()) {
+            lastPluginSyncResults = null
+            return
+        }
+
+        // Flatten the already-collected anchor descriptors into plain maps
+        // (id/type/transform + optional alignment/extent) so plugins get the
+        // same shape as the JS `ARAnchor` contract without a JSI dependency.
+        val anchorMaps: List<Map<String, Any?>> =
+            if (anchors.isEmpty()) emptyList()
+            else anchors.map { a ->
+                val m = HashMap<String, Any?>(5)
+                m["id"] = a.id
+                m["type"] = a.type
+                m["transform"] = a.transform
+                if (a.alignment.isNotEmpty()) m["alignment"] = a.alignment
+                a.extent?.let { m["extent"] = it }
+                m
+            }
+
+        val ctx = ARFrameContext(
+            nv21 = packed.nv21,
+            width = packed.width,
+            height = packed.height,
+            timestampNs = frame.timestamp.toDouble(),
+            fx = intrinsics.focalLength[0].toDouble(),
+            fy = intrinsics.focalLength[1].toDouble(),
+            cx = intrinsics.principalPoint[0].toDouble(),
+            cy = intrinsics.principalPoint[1].toDouble(),
+            imageWidth = intrinsics.imageDimensions[0],
+            imageHeight = intrinsics.imageDimensions[1],
+            poseRotation = doubleArrayOf(
+                qarr[0].toDouble(), qarr[1].toDouble(),
+                qarr[2].toDouble(), qarr[3].toDouble(),
+            ),
+            poseTranslation = doubleArrayOf(
+                tArr[0].toDouble(), tArr[1].toDouble(), tArr[2].toDouble(),
+            ),
+            trackingState = tracking,
+            depthBytes = depth?.bytes,
+            depthWidth = depth?.width ?: 0,
+            depthHeight = depth?.height ?: 0,
+            anchors = anchorMaps,
+        )
+
+        var sync: HashMap<String, Any?>? = null
+        for (plugin in plugins) {
+            val result = try {
+                plugin.process(ctx)
+            } catch (t: Throwable) {
+                if (forwardLogTick % 30 == 1) {
+                    Log.w(TAG, "AR plugin '${plugin.name()}' threw in process(): ${t.message}")
+                }
+                null
+            }
+            if (result != null) {
+                if (sync == null) sync = HashMap()
+                sync[plugin.name()] = result
+            }
+        }
+        lastPluginSyncResults = sync
     }
 
     /// Packed DEPTH16 result: dense (no row padding) uint16-per-pixel
@@ -1186,6 +1620,31 @@ class RNSARCameraView @JvmOverloads constructor(
             meta.putNull("mesh")
         }
 
+        // ── plugins (0.19.0) — native-plugin SYNC results, if any ────────
+        //    `lastPluginSyncResults` was stashed by `runArPlugins` earlier
+        //    in THIS same onDrawFrame tick (forwardToIncremental runs before
+        //    maybeEmitArFrameMeta).  Each value is the WritableMap a plugin
+        //    returned from `process()`; we re-key it under `plugins[name]`.
+        //    Omitted entirely when no plugin produced a sync result (JS sees
+        //    `meta.plugins === undefined`), matching the optional `plugins?`
+        //    field in the ARFrameMeta contract.
+        val pluginResults = lastPluginSyncResults
+        if (!pluginResults.isNullOrEmpty()) {
+            val pluginsMap = com.facebook.react.bridge.Arguments.createMap()
+            for ((name, value) in pluginResults) {
+                when (value) {
+                    is com.facebook.react.bridge.WritableMap ->
+                        pluginsMap.putMap(name, value)
+                    null -> pluginsMap.putNull(name)
+                    // Defensive: a plugin should only ever return a
+                    // WritableMap, but never let an unexpected type crash the
+                    // emit — drop it.
+                    else -> { /* skip unsupported result type */ }
+                }
+            }
+            meta.putMap("plugins", pluginsMap)
+        }
+
         session.emitArFrameMeta(meta)
     }
 

package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt

@@ -1,8 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 package io.imagestitcher.rn
 
+import com.facebook.react.bridge.ReadableArray
 import com.facebook.react.uimanager.SimpleViewManager
 import com.facebook.react.uimanager.ThemedReactContext
+import com.facebook.react.uimanager.annotations.ReactProp
 
 /**
  * RN ViewManager for `RNSARCameraView`.
@@ -16,11 +18,19 @@ import com.facebook.react.uimanager.ThemedReactContext
  * in `src/camera/ARCameraView.tsx` which auto-selects the native
  * component for iOS vs Android.
  *
- * The view itself is config-free for now (no JS-side props beyond
- * `style`) since lifecycle is driven by mount/unmount + the
- * incremental stitcher's start/finalize methods.  Future phases may
- * add props like `enabled` to allow JS-controlled pause/resume of
- * the GL render loop.
+ * Props:
+ *   - `overlays` (0.20.0) — declarative `AROverlay[]` drawn on the AR
+ *     overlay layer above the camera preview (see [AROverlayStore] /
+ *     [AROverlayRenderer]).  React state-driven: when the array changes,
+ *     RN re-sends it and we REPLACE the view's JS overlay namespace.  The
+ *     imperative ref API (setOverlays / addOverlay / updateOverlay /
+ *     removeOverlay / clearOverlays) routes through the `RNSARSession`
+ *     native module instead (the same idiom takePhoto uses); both write the
+ *     SAME JS namespace, and the renderer draws the union of JS + native-
+ *     plugin overlays.
+ *
+ * Lifecycle remains driven by mount/unmount + the incremental stitcher's
+ * start/finalize methods.
  */
 class RNSARCameraViewManager : SimpleViewManager<RNSARCameraView>() {
 
@@ -29,6 +39,21 @@ class RNSARCameraViewManager : SimpleViewManager<RNSARCameraView>() {
     override fun createViewInstance(reactContext: ThemedReactContext): RNSARCameraView =
         RNSARCameraView(reactContext)
 
+    /**
+     * 0.20.0 — declarative `overlays` prop.  Parses the `AROverlay[]` array
+     * (the shared TS contract shape) and REPLACES the view's JS overlay
+     * namespace.  A null / empty array clears the JS overlays (native-plugin
+     * overlays are untouched).  Malformed entries are dropped (see
+     * [AROverlayData.fromReadableArray]).
+     *
+     * React diffs the prop by value before re-sending, so we don't diff
+     * here — a fresh array means "this is the new full JS overlay set".
+     */
+    @ReactProp(name = "overlays")
+    fun setOverlays(view: RNSARCameraView, overlays: ReadableArray?) {
+        view.setOverlaysFromJs(AROverlayData.fromReadableArray(overlays))
+    }
+
     companion object {
         const val REACT_CLASS = "RNSARCameraView"
     }