react-native-image-stitcher 0.17.0 → 0.18.0

package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt

@@ -120,6 +120,72 @@ class RNSARSession(reactContext: ReactApplicationContext)
         }
     }
 
+    /**
+     * Android side of the JS `<Camera planeDetection=...>` prop.  Sets
+     * which plane orientations are EMITTED into `arAnchors`
+     * (`"vertical"` | `"horizontal"` | `"both"`); an unrecognised value
+     * falls back to `"vertical"`.
+     *
+     * Unlike iOS — where the prop narrows ARKit's `planeDetection`
+     * option set — we deliberately do NOT narrow ARCore's
+     * `planeFindingMode`: it stays `HORIZONTAL_AND_VERTICAL` so ARCore
+     * keeps bootstrapping tracking from horizontal planes (a plain
+     * vertical wall alone leaves ARCore unable to establish a
+     * gravity-aligned world; see the start()/startForView() config
+     * comments).  We only FILTER which plane orientations are surfaced
+     * into `arAnchors` (in [RNSARCameraView.collectTrackingAnchors]).
+     *
+     * Void (no Promise): a fire-and-forget setter mirroring the other
+     * config-prop bridge calls.
+     */
+    @ReactMethod
+    fun setPlaneDetection(mode: String) {
+        planeDetectionMode = when (mode) {
+            "vertical", "horizontal", "both" -> mode
+            else -> "vertical"
+        }
+    }
+
+    // ── onArFrame — LIGHT AR metadata channel (v0.18.0) ───────────────
+    //
+    // Android side of the shared `onArFrame` contract.  Delivers light
+    // AR metadata (tracking state, pose, intrinsics, depth dims, anchor
+    // descriptors, mesh counts) to JS on the main thread as a normal
+    // RCTDeviceEventEmitter event — NO worklets, NO zero-copy buffers.
+    //
+    // This is the deliberate counterpart to the `arFrameProcessor`
+    // host-worklet path: worklets-core's closure-wrap crashes when an
+    // AR worklet captures host objects, so `onArFrame` bypasses worklets
+    // entirely and uses the bridge event channel the rest of this module
+    // already uses (see `IncrementalStitcher.emitState`).
+    //
+    // TS sets the gate via `setArFrameMetaEnabled(true, intervalMs)` when
+    // a host passes the `onArFrame` prop, and `setArFrameMetaEnabled(false, _)`
+    // on unmount / prop removal.  The per-frame build + emit happens in
+    // `RNSARCameraView.onDrawFrame` (the only thread guaranteed to run
+    // once the AR session is live), gated + throttled by the companion
+    // state set here.
+
+    /**
+     * JS-facing gate for the `onArFrame` metadata channel.
+     *
+     *  - `enabled`    — true while a host supplies the `onArFrame` prop.
+     *  - `intervalMs` — throttle floor in milliseconds (default contract
+     *                   value 100 ≈ 10 Hz).  Clamped to ≥ 0; a 0 interval
+     *                   means "emit every render frame" (no throttle).
+     *
+     * Fire-and-forget (no Promise) — mirrors the other config-prop bridge
+     * setters (`setPlaneDetection`, `lockPortrait`).
+     */
+    @ReactMethod
+    fun setArFrameMetaEnabled(enabled: Boolean, intervalMs: Double) {
+        arFrameMetaEnabled = enabled
+        arFrameMetaIntervalMs = if (intervalMs.isNaN()) 100L else intervalMs.toLong().coerceAtLeast(0L)
+        // Reset the throttle clock so the first frame after enabling emits
+        // immediately rather than waiting out a stale interval window.
+        arFrameMetaLastEmitNs = 0L
+    }
+
     @ReactMethod
     fun isSupported(promise: Promise) {
         // `checkAvailability` can return UNKNOWN_CHECKING if the
@@ -832,6 +898,48 @@ class RNSARSession(reactContext: ReactApplicationContext)
         if (attachedView === view) attachedView = null
     }
 
+    /**
+     * Emit a pre-built [ARFrameMeta] WritableMap to JS over the shared
+     * `RNImageStitcherARFrame` device event.  Called from
+     * [RNSARCameraView.maybeEmitArFrameMeta] on the GL render thread.
+     *
+     * Uses the same `DeviceEventManagerModule.RCTDeviceEventEmitter`
+     * channel as [IncrementalStitcher.emitState] — RN drops the event
+     * when no JS listener is attached, so no extra gating is needed
+     * beyond the enabled flag the caller already checked.  The
+     * TS-required `addListener`/`removeListeners` no-op pair already
+     * exists on the `IncrementalStitcher` module; the `NativeEventEmitter`
+     * the TS layer constructs over `RNSARSession` needs the same pair, so
+     * they're declared below.
+     */
+    internal fun emitArFrameMeta(meta: com.facebook.react.bridge.WritableMap) {
+        try {
+            reactApplicationContext
+                .getJSModule(
+                    com.facebook.react.modules.core.DeviceEventManagerModule.RCTDeviceEventEmitter::class.java,
+                )
+                .emit(AR_FRAME_META_EVENT, meta)
+        } catch (t: Throwable) {
+            // Catalyst instance torn down mid-emit (reload / unmount race),
+            // or no JS context yet — drop the frame silently.  AR metadata
+            // is best-effort and re-emitted every interval, so a dropped
+            // frame is harmless.
+            Log.d(TAG, "emitArFrameMeta: emit failed (ignoring): ${t.message}")
+        }
+    }
+
+    /// Required by RN's `NativeEventEmitter` contract — the TS
+    /// `onArFrame` wiring constructs a `NativeEventEmitter` over this
+    /// module, which calls `addListener`/`removeListeners` on subscribe /
+    /// unsubscribe.  No-op on Android: `DeviceEventManagerModule` does its
+    /// own listener tracking and drops events when none are attached
+    /// (same rationale as `IncrementalStitcher.addListener`).
+    @ReactMethod
+    fun addListener(eventName: String) { /* no-op — see KDoc */ }
+
+    @ReactMethod
+    fun removeListeners(count: Int) { /* no-op — see KDoc */ }
+
     private fun clearPoseLogInternal() {
         poseLogLock.write { poseLog.clear() }
     }
@@ -915,6 +1023,54 @@ class RNSARSession(reactContext: ReactApplicationContext)
         @Volatile
         var instance: RNSARSession? = null
             private set
+
+        /**
+         * Which plane orientations reach `arAnchors`:
+         * `"vertical"` | `"horizontal"` | `"both"`.
+         *
+         * Default `"vertical"` preserves the legacy plane-projected
+         * stitch path (the shutter-gate / evaluatePlanesForFrame logic
+         * only ever cared about vertical planes), so existing hosts see
+         * no change unless they opt into a wider filter via the JS
+         * `<Camera planeDetection=...>` prop (→ [setPlaneDetection]).
+         *
+         * Read on the GL render thread in
+         * [RNSARCameraView.collectTrackingAnchors]; written from the JS
+         * thread via [setPlaneDetection] — hence `@Volatile`.
+         */
+        @JvmStatic
+        @Volatile
+        var planeDetectionMode: String = "vertical"
+
+        // ── onArFrame gate + throttle (v0.18.0) ──────────────────────
+        //
+        // Written from the JS thread via [setArFrameMetaEnabled];
+        // read on the GL render thread in
+        // [RNSARCameraView.maybeEmitArFrameMeta] — hence `@Volatile`.
+        //
+        //  - `arFrameMetaEnabled`     — gate: only build+emit when true.
+        //  - `arFrameMetaIntervalMs`  — throttle floor (ms); contract
+        //                               default 100 (≈10 Hz).
+        //  - `arFrameMetaLastEmitNs`  — monotonic clock of the last emit
+        //                               (System.nanoTime()); 0 = "never",
+        //                               reset on every enable so the first
+        //                               post-enable frame emits at once.
+        @JvmStatic
+        @Volatile
+        var arFrameMetaEnabled: Boolean = false
+
+        @JvmStatic
+        @Volatile
+        var arFrameMetaIntervalMs: Long = 100L
+
+        @JvmStatic
+        @Volatile
+        var arFrameMetaLastEmitNs: Long = 0L
+
+        /// Event name carrying the [ARFrameMeta] payload to JS.  MUST
+        /// match the shared contract + the iOS `supportedEvents` entry +
+        /// the TS `NativeEventEmitter` subscription string exactly.
+        const val AR_FRAME_META_EVENT = "RNImageStitcherARFrame"
     }
 
     init {

package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt

@@ -40,7 +40,7 @@ import com.facebook.react.bridge.ReactMethod
  * `retailens::StitcherWorkletRegistry`.  Per-frame fan-out from
  * Android's `StitcherWorkletRuntime` is a separate piece of work
  * (Phase 4b.ii follow-up) — needs the Kotlin↔JNI bridge that
- * constructs a `StitcherFrameJsiHostObject` from an `ArImage` +
+ * constructs a `CameraFrameJsiHostObject` from an `ArImage` +
  * pose and posts it through a worklet runtime.  Until that lands,
  * Android-registered worklets behave exactly like iOS-registered
  * worklets BEFORE Phase 4b.i: they exist in the registry but

package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt

@@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicBoolean
  *   the GLSurfaceView GL render thread (audit caveat #4 from the
  *   Phase-0 audit).
  * - `dispatchFrame()` stub — Phase 3c will fill in:
- *     1. Build `StitcherFrameHostObject` from ARCore Frame + pose
+ *     1. Build `CameraFrameHostObject` from ARCore Frame + pose
  *        (via the shared C++ JSI host object now linked into
  *        `libimage_stitcher.so` post-Phase-3a).
  *     2. Run first-party stitching synchronously on the caller
@@ -165,7 +165,7 @@ object StitcherWorkletRuntime {
     /// **When host worklets ARE registered:** the JNI layer copies
     /// the NV21 byte array into an owned C++ `std::vector` (so the
     /// async dispatch can outlive ARCore's `Image.close()` scope),
-    /// builds a `StitcherFrameJsiHostObject`, and posts a lambda
+    /// builds a `CameraFrameJsiHostObject`, and posts a lambda
     /// onto worklets-core's default `JsiWorkletContext`'s worklet
     /// thread.  The lambda iterates the registry's
     /// `WorkletInvoker`s, calls each with the JSI host object as
@@ -196,6 +196,26 @@ object StitcherWorkletRuntime {
     /// @param trackingState  One of "" / "notAvailable" / "limited"
     ///                       / "normal".  Empty string ⇒ JS-side
     ///                       `arTrackingState` is `undefined`.
+    /// @param depthBytes     Raw ARCore DEPTH16 bytes, dense row-packed
+    ///                       (`depthWidth*depthHeight*2` bytes, uint16/px,
+    ///                       low 13 bits = mm, high 3 bits = confidence).
+    ///                       `null` when depth is unavailable this frame —
+    ///                       the JNI then leaves `data.arDepth == nullopt`.
+    /// @param depthWidth     Depth-map width (px); 0 when no depth.
+    /// @param depthHeight    Depth-map height (px); 0 when no depth.
+    /// @param anchorIds      Parallel arrays describing every TRACKING
+    /// @param anchorTypes    anchor: stable id, coarse type
+    /// @param anchorTransforms ("plane"/"image"/"point"/"mesh"), and a
+    ///                       16-element ROW-MAJOR (anchor->world) transform
+    ///                       (identity for the depth-derived "mesh" anchor —
+    ///                       its vertices are camera-local).  Empty when no
+    ///                       anchors/mesh were collected.
+    /// @param anchorMeshVertices Parallel per-anchor mesh byte arrays: a
+    /// @param anchorMeshFaces    Float32-xyz vertex buffer + a Uint32 index
+    ///                       buffer for the depth-derived mesh anchor, `null`
+    ///                       for every non-mesh anchor.  Carried verbatim to
+    ///                       the JNI which sets `ArAnchor.hasMesh` + the
+    ///                       mesh vectors when both are non-null.
     @JvmStatic
     fun dispatchToHostWorklets(
         nv21Bytes: ByteArray,
@@ -205,6 +225,18 @@ object StitcherWorkletRuntime {
         tx: Double, ty: Double, tz: Double,
         timestampNs: Double,
         trackingState: String,
+        depthBytes: ByteArray?,
+        depthWidth: Int,
+        depthHeight: Int,
+        anchorIds: Array<String>,
+        anchorTypes: Array<String>,
+        anchorTransforms: Array<DoubleArray>,
+        anchorMeshVertices: Array<ByteArray?>,
+        anchorMeshFaces: Array<ByteArray?>,
+        fx: Double, fy: Double, cx: Double, cy: Double,
+        intrinsicsImageWidth: Int, intrinsicsImageHeight: Int,
+        anchorAlignments: Array<String>,
+        anchorExtents: Array<DoubleArray?>,
     ) {
         if (!installed.get()) return
         nativeDispatchToHostWorklets(
@@ -212,6 +244,12 @@ object StitcherWorkletRuntime {
             qx, qy, qz, qw,
             tx, ty, tz,
             timestampNs, trackingState,
+            depthBytes, depthWidth, depthHeight,
+            anchorIds, anchorTypes, anchorTransforms,
+            anchorMeshVertices, anchorMeshFaces,
+            fx, fy, cx, cy,
+            intrinsicsImageWidth, intrinsicsImageHeight,
+            anchorAlignments, anchorExtents,
         )
     }
 
@@ -228,9 +266,41 @@ object StitcherWorkletRuntime {
         return nativeRegistryCount() > 0
     }
 
+    /// Per-frame AR-metadata extraction toggles (the JS-driven
+    /// enableDepth/enableAnchors/enableMesh `<Camera>` props, written via
+    /// `__stitcherProxy.setExtractionConfig`).  Read once per frame in
+    /// `RNSARCameraView.forwardToIncremental` to GATE the costly ARCore
+    /// depth-acquire / anchor-collect / mesh-build work — all default OFF,
+    /// so a host pays zero AR-metadata cost until it opts in.
+    ///
+    /// Returns all-false (no extraction) before `installIfNeeded()` runs.
+    data class ExtractionFlags(
+        val depth: Boolean,
+        val anchors: Boolean,
+        val mesh: Boolean,
+    )
+
+    @JvmStatic
+    fun extractionFlags(): ExtractionFlags {
+        if (!installed.get()) return ExtractionFlags(false, false, false)
+        val bits = nativeExtractionFlags()
+        return ExtractionFlags(
+            depth = (bits and 0x1) != 0,
+            anchors = (bits and 0x2) != 0,
+            mesh = (bits and 0x4) != 0,
+        )
+    }
+
     @JvmStatic
     private external fun nativeRegistryCount(): Int
 
+    /// JNI binding: `nativeExtractionFlags` in
+    /// `android/src/main/cpp/stitcher_jsi_install_jni.cpp`.  Packs
+    /// `retailens::getExtractionConfig()` into a bitmask
+    /// (bit0=depth, bit1=anchors, bit2=mesh).
+    @JvmStatic
+    private external fun nativeExtractionFlags(): Int
+
     /// JNI binding: `android/src/main/cpp/stitcher_jsi_install_jni.cpp`'s
     /// `nativeDispatchToHostWorklets`.  Fast-path early-exit lives
     /// inside the native function — see its docstring.
@@ -243,6 +313,18 @@ object StitcherWorkletRuntime {
         tx: Double, ty: Double, tz: Double,
         timestampNs: Double,
         trackingState: String,
+        depthBytes: ByteArray?,
+        depthWidth: Int,
+        depthHeight: Int,
+        anchorIds: Array<String>,
+        anchorTypes: Array<String>,
+        anchorTransforms: Array<DoubleArray>,
+        anchorMeshVertices: Array<ByteArray?>,
+        anchorMeshFaces: Array<ByteArray?>,
+        fx: Double, fy: Double, cx: Double, cy: Double,
+        intrinsicsImageWidth: Int, intrinsicsImageHeight: Int,
+        anchorAlignments: Array<String>,
+        anchorExtents: Array<DoubleArray?>,
     )
 
     init {

package/cpp/{stitcher_frame_data.hpp → camera_frame_data.hpp}

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 //
-// stitcher_frame_data.hpp — platform-agnostic backing data for the
+// camera_frame_data.hpp — platform-agnostic backing data for the
 // v0.8.0 `StitcherFrame` JSI host object.
 //
 // ## Why this lives here
@@ -16,21 +16,24 @@
 // ## Memory model
 //
 // `PixelBufferReader` is an opaque interface; platform code (iOS
-// `StitcherFrameHostObject.mm`; Android `stitcher_frame_jni.cpp`)
+// `CameraFrameHostObject.mm`; Android `stitcher_frame_jni.cpp`)
 // implements it by wrapping the underlying `CVPixelBufferRef` /
 // `ArImage*`.  Lifetime: the reader holds a strong ref to its
 // source for the entire host-object lifetime; releases on
 // destruction (deterministic, RAII).
 //
-// `StitcherFrameData` is value-typed (cheap to copy; ~100 bytes).
+// `CameraFrameData` is value-typed (cheap to copy; ~100 bytes).
 // Construct on the worklet runtime's thread before each dispatch.
 
 #pragma once
 
+#include <array>
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <string>
+#include <vector>
 
 namespace retailens {
 
@@ -68,10 +71,75 @@ public:
     virtual std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) = 0;
 };
 
+/// One detected AR anchor (ARKit `ARAnchor` / ARCore `Anchor`).
+/// Mirrors the JS `ARAnchor`: a stable id, a coarse type, and a
+/// 4x4 transform.
+struct ArAnchor {
+    /// Stable per-session identifier (ARKit UUID / ARCore anchor id).
+    std::string id;
+    /// Coarse class: `"plane"`, `"image"`, `"point"`, or `"mesh"`.
+    std::string type;
+    /// 4x4 anchor->world transform, ROW-MAJOR (16 elements). Platform
+    /// code is responsible for emitting row-major regardless of the
+    /// native matrix's storage order.
+    std::array<double, 16> transform{};
+
+    /// Plane alignment: "" (n/a), "horizontal", or "vertical".  Set on
+    /// plane anchors; empty -> JS `alignment === undefined`.
+    std::string alignment;
+    /// Plane extent in metres along local x/z ([extentX, extentZ]); valid
+    /// only when hasExtent (plane anchors).
+    bool hasExtent = false;
+    double extentX = 0.0;
+    double extentZ = 0.0;
+    /// ARKit semantic class ("wall","floor","table",… or "none"); empty
+    /// string -> JS `classification === undefined` (iOS only).
+    std::string classification;
+
+    // ── Scene-reconstruction geometry (only when type == "mesh") ──
+    /// True when this anchor carries a mesh (gates JSI emission of
+    /// `meshGeometry`).  Raw bytes, emitted as ArrayBuffers verbatim
+    /// (no conversion) by the JSI layer:
+    ///   - meshVertices: Float32 xyz triplets, anchor-local.
+    ///   - meshFaces: Uint32 triangle indices into the vertices.
+    ///   - meshClassifications: optional Uint8 per-face class (iOS
+    ///     ARMeshAnchor; empty on Android — depth-derived meshes have
+    ///     no semantics).
+    bool hasMesh = false;
+    std::vector<uint8_t> meshVertices;
+    std::vector<uint8_t> meshFaces;
+    std::vector<uint8_t> meshClassifications;
+};
+
+/// AR depth map for one frame. The platforms encode depth differently,
+/// so we carry the raw bytes plus a `format` tag and NORMALISE to a
+/// single JS shape (Float32 metres + Uint8 confidence 0..2) in the JSI
+/// layer (`camera_frame_jsi.cpp`):
+///   - iOS (ARKit `ARDepthData`): `depthBytes` = Float32 metres
+///     (row-packed); `confidenceBytes` = Uint8 `ARConfidenceLevel`
+///     (0=low,1=medium,2=high). `format = "f32m"`.
+///   - Android (ARCore DEPTH16): `depthBytes` = uint16 packed (low 13
+///     bits = millimetres, high 3 bits = confidence 0..7);
+///     `confidenceBytes` empty. `format = "u16packed"`.
+/// Depth maps are small (~256x192 iOS, ~160x120 Android) so the bytes
+/// are eager-copied at extraction time (the ARCore Image is closed
+/// in-scope; iOS copies for the same uniform contract).
+struct ArDepth {
+    int32_t width = 0;
+    int32_t height = 0;
+    /// Encoding of `depthBytes`: `"f32m"` or `"u16packed"`.
+    std::string format;
+    /// Raw depth bytes, interpreted per `format`.
+    std::vector<uint8_t> depthBytes;
+    /// Per-pixel confidence (Uint8 0..2). Populated on iOS; empty on
+    /// Android (confidence is packed into `depthBytes`).
+    std::vector<uint8_t> confidenceBytes;
+};
+
 /// Plain-old-data payload for one `StitcherFrame`.  Fully extracted
 /// at construction time (cheap fields) plus an opaque reader for
 /// the lazy pixel access.
-struct StitcherFrameData {
+struct CameraFrameData {
     /// Discriminator. `"ar"` for AR-mode frames, `"vc"` for
     /// vision-camera frames.  Used by worklets to gate on AR-only
     /// field access (translation, depth, anchors, tracking state).
@@ -127,15 +195,30 @@ struct StitcherFrameData {
     /// docstring for lifetime contract.
     std::shared_ptr<PixelBufferReader> pixelReader;
 
-    // ── AR-only optional fields (not populated in v0.8.0; stubs) ──
-    // These are deferred to v0.8.1+ because the host worklets that
-    // would consume them aren't shipping in v0.8.0 either.  Adding
-    // them here as plain data fields keeps the JSI host object code
-    // simple when they DO arrive.
-
-    /// arDepth, arAnchors stubs intentionally omitted — they're
-    /// fields the JSI dispatch will return `undefined` for in v0.8.0.
-    /// v0.8.1+ adds them here as `std::optional<ArDepth>` etc.
+    // ── AR-only optional fields ──────────────────────────────────────
+
+    /// AR depth map (ARKit sceneDepth / ARCore Depth API).  `nullopt`
+    /// when the device/session can't provide depth; the JSI host object
+    /// then exposes `arDepth === undefined`.  Normalised to a single JS
+    /// shape in the JSI layer regardless of the native `format`.
+    std::optional<ArDepth> arDepth;
+
+    /// Tracked AR anchors visible this frame.  Empty when none (or in
+    /// non-AR mode); the JSI host object exposes `arAnchors === undefined`
+    /// only when empty AND source != "ar" (an AR frame with no anchors
+    /// returns an empty array, per the JS contract).
+    std::vector<ArAnchor> arAnchors;
+
+    /// Per-frame camera intrinsics (fx,fy,cx,cy in pixels) + the capture
+    /// resolution they're expressed at.  Valid only when hasIntrinsics
+    /// (AR frames); the JSI exposes `intrinsics === undefined` otherwise.
+    bool hasIntrinsics = false;
+    double fx = 0.0;
+    double fy = 0.0;
+    double cx = 0.0;
+    double cy = 0.0;
+    int32_t intrinsicsImageWidth = 0;
+    int32_t intrinsicsImageHeight = 0;
 };
 
 }  // namespace retailens