react-native-image-stitcher 0.7.1 → 0.9.0

package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt

@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: Apache-2.0
+package io.imagestitcher.rn
+
+import android.os.HandlerThread
+import android.util.Log
+import java.util.concurrent.atomic.AtomicBoolean
+
+/**
+ * v0.8.0 Phase 3b — Android twin of iOS' `RNSARWorkletRuntime`.
+ * Owns the per-AR-frame worklet runtime + the thread it dispatches
+ * on.  Symmetric API shape to the iOS class so the cross-platform
+ * dispatch story (Phase 3c) lands in lockstep.
+ *
+ * ## Phase 3b scope (this commit)
+ *
+ * - Singleton accessor + lifecycle (installIfNeeded / isInstalled).
+ * - Dedicated `HandlerThread` for worklet dispatch — keeps work off
+ *   the GLSurfaceView GL render thread (audit caveat #4 from the
+ *   Phase-0 audit).
+ * - `dispatchFrame()` stub — Phase 3c will fill in:
+ *     1. Build `StitcherFrameHostObject` from ARCore Frame + pose
+ *        (via the shared C++ JSI host object now linked into
+ *        `libimage_stitcher.so` post-Phase-3a).
+ *     2. Run first-party stitching synchronously on the caller
+ *        thread (`onDrawFrame`'s GL thread, today).
+ *     3. If host worklets are registered, dispatch the host object
+ *        onto this runtime's `HandlerThread` + invoke each worklet
+ *        via JNI → `RNWorklet::WorkletInvoker::call`.
+ *     4. Invalidate the host object after all worklets return.
+ *
+ * ## Worklet-runtime construction model
+ *
+ * Unlike iOS (where the lib's `.mm` directly `std::make_shared`s
+ * a `RNWorklet::JsiWorkletContext`), Android can't construct the
+ * context purely from native C++ without JNI plumbing.  Phase 3c
+ * will choose between two paths:
+ *
+ *   - **Option A:** JS-side code calls
+ *     `Worklets.createContext("stitcher.ar")` at AR-mode start;
+ *     hands the resulting context pointer to this Kotlin class via
+ *     a small JSI plugin.  Minimal new JNI.  **Phase 3b's
+ *     HandlerThread becomes dead code** under this option — the
+ *     JS-side `Worklets.createContext` picks its own thread.  We'd
+ *     need to remove the HandlerThread + change `installIfNeeded`
+ *     into a no-op until a `setContextHandle(Long)` setter lands.
+ *   - **Option B:** Direct JNI binding to worklets-core's C++
+ *     constructor.  More native code but no JS dependency at runtime.
+ *     **Phase 3b's HandlerThread is exactly the right scaffold**
+ *     under this option — its looper becomes the JsiWorkletContext's
+ *     `workletCallInvoker` target.
+ *
+ * **Phase 3b assumption: Option B is the more likely path.**  The
+ * scaffolding below (HandlerThread + serial dispatch) fits Option
+ * B; if Phase 3c picks Option A instead, the HandlerThread becomes
+ * unused and Phase 3c will refactor accordingly.
+ *
+ * @see [RNSARWorkletRuntime] iOS equivalent
+ * @see docs/plans/handoff/2026-05-26-v0.8.0-phase-0-audit.md
+ *      worklets-core API rationale (Audit 2: `JsiWorkletContext`).
+ * @see docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md
+ *      Phase 3c implementation plan.
+ */
+object StitcherWorkletRuntime {
+    private const val TAG = "StitcherWorkletRuntime"
+
+    /// Single-flight install guard.  `compareAndSet` makes the
+    /// runtime construction race-safe across concurrent first-mount
+    /// calls from multiple `<Camera>` instances.
+    private val installed = AtomicBoolean(false)
+
+    /// Dedicated dispatch thread.  Constructed eagerly so we can
+    /// validate the thread starts cleanly during `installIfNeeded`.
+    /// Off the GLSurfaceView GL render thread (audit caveat #4)
+    /// + off the main thread.  Phase 3c will configure the worklet
+    /// context's `workletCallInvoker` to post onto this thread's
+    /// looper.
+    private val dispatchThread: HandlerThread by lazy {
+        HandlerThread("io.imagestitcher.ar-worklet-runtime").apply { start() }
+    }
+
+    /// Construct the underlying worklet context if not yet installed.
+    /// Idempotent — repeated calls are no-ops.
+    ///
+    /// Phase 3b: starts the dispatch thread; no JsiWorkletContext
+    /// construction yet (deferred to Phase 3c).
+    /// Phase 3c: also wires the JsiWorkletContext + binds it to the
+    /// dispatch thread's looper.
+    @JvmStatic
+    fun installIfNeeded() {
+        if (!installed.compareAndSet(false, true)) {
+            return
+        }
+        // Force the lazy `dispatchThread` to initialise.  If the
+        // OS denies thread creation (extreme memory pressure on a
+        // budget device), `HandlerThread.start()` won't throw but
+        // the looper won't be available — the Phase 3c dispatch
+        // logic will need to defend against that.  For Phase 3b
+        // we only care that this method returns without throwing.
+        //
+        // Log `Thread.id` (Java-side monotonic, always non-zero) —
+        // NOT `HandlerThread.threadId` (Linux tid set after first
+        // Looper-prepared message; reading from this caller thread
+        // immediately after .start() returns -1 until scheduled).
+        val javaThreadId = dispatchThread.id
+        Log.i(TAG, "installed runtime; dispatch java-thread id=$javaThreadId")
+    }
+
+    /// Diagnostics + tests.  Returns `true` after a successful
+    /// `installIfNeeded()`.
+    @JvmStatic
+    fun isInstalled(): Boolean = installed.get()
+
+    /// v0.8.0 Phase 3c — first-party stitching dispatch.  Invokes
+    /// the supplied block synchronously on the caller thread
+    /// (`onDrawFrame`'s GL render thread today).
+    ///
+    /// Phase 3c minimum-viable: this is the closure-based equivalent
+    /// of iOS' first-party callback.  The block is the original
+    /// `module.ingestFromARCameraView(...)` call site moved
+    /// verbatim into a lambda — no behaviour change, just an
+    /// indirection so Phase 4 can interpose host-worklet fanout
+    /// without touching the engine ingest path.
+    ///
+    /// **Why synchronous + on the caller thread:** the engine's
+    /// `ingestFromARCameraView` takes ownership of the ARCore
+    /// `Image`-derived NV21 buffer (via the v0.10.0 `TransferredNV21`
+    /// wrapper).  ARCore's `Image.close()` happens after this call
+    /// returns, so the consumer must finish reading the bytes before
+    /// we return — exactly what synchronous block invocation
+    /// provides.  Phase 4 will copy the buffer for off-thread
+    /// access in host worklets; Phase 3c keeps the sync contract.
+    ///
+    /// If `installIfNeeded()` hasn't been called yet, the block
+    /// still runs (no-op on the registry side).  Defensive — the
+    /// caller may call this method before `installIfNeeded` is
+    /// wired up.
+    @JvmStatic
+    fun runFirstParty(block: () -> Unit) {
+        // Synchronous invocation — Phase 4 will extend this to also
+        // post the registered host worklets onto `dispatchThread`.
+        // Not `inline`: Phase 4 will need to read `dispatchThread`
+        // (private) from inside this function body, and Kotlin's
+        // inline functions can't access private members from call
+        // sites outside the declaring class.  Per-frame lambda
+        // alloc is ~ns and the alternative (callers passing a
+        // method reference) doesn't materially change cost.
+        block()
+    }
+
+    /// v0.8.0 Phase 4b.iii — fan out one AR frame to every host
+    /// worklet registered in the shared C++ `StitcherWorkletRegistry`
+    /// (populated from JS via `__stitcherProxy.install(workletFn)`).
+    ///
+    /// Called from `RNSARCameraView.onDrawFrame` immediately after
+    /// `runFirstParty { ... }` returns, with the already-extracted
+    /// AR frame data (pose + NV21 bytes + dimensions + tracking
+    /// state).
+    ///
+    /// **Fast-path:** the native side queries the registry's count
+    /// FIRST and returns before copying any bytes when no host
+    /// worklets are registered.  In the common first-party-only
+    /// deployment, this method costs one JNI call + one C++ atomic
+    /// read per frame — negligible.
+    ///
+    /// **When host worklets ARE registered:** the JNI layer copies
+    /// the NV21 byte array into an owned C++ `std::vector` (so the
+    /// async dispatch can outlive ARCore's `Image.close()` scope),
+    /// builds a `StitcherFrameJsiHostObject`, and posts a lambda
+    /// onto worklets-core's default `JsiWorkletContext`'s worklet
+    /// thread.  The lambda iterates the registry's
+    /// `WorkletInvoker`s, calls each with the JSI host object as
+    /// its argument, and invalidates the host object after the
+    /// last invoker returns.  Per-worklet failure isolation: one
+    /// host worklet throwing does NOT stop the lib's stitching or
+    /// the other host worklets.
+    ///
+    /// **Threading:** this method returns synchronously on the
+    /// caller's thread.  The actual worklet invocations happen
+    /// asynchronously on the worklets-core thread; the caller does
+    /// NOT block on them.
+    ///
+    /// **Caller-thread contract:** the caller (`RNSARCameraView`'s
+    /// `onDrawFrame`) MUST have already invoked `runFirstParty`
+    /// before calling this method.  The first-party stitching
+    /// path holds the synchronous ARCore Image consumption
+    /// contract; the host-worklet dispatch does not.
+    ///
+    /// @param nv21Bytes      Pre-packed NV21 byte array.  COPIED
+    ///                       into a native owned buffer; caller can
+    ///                       release the reference after return.
+    /// @param width          Camera image width (pixels).
+    /// @param height         Camera image height (pixels).
+    /// @param qx,qy,qz,qw    Pose rotation quaternion (unit length).
+    /// @param tx,ty,tz       Pose translation (metres, world coords).
+    /// @param timestampNs    Frame timestamp in nanoseconds.
+    /// @param trackingState  One of "" / "notAvailable" / "limited"
+    ///                       / "normal".  Empty string ⇒ JS-side
+    ///                       `arTrackingState` is `undefined`.
+    @JvmStatic
+    fun dispatchToHostWorklets(
+        nv21Bytes: ByteArray,
+        width: Int,
+        height: Int,
+        qx: Double, qy: Double, qz: Double, qw: Double,
+        tx: Double, ty: Double, tz: Double,
+        timestampNs: Double,
+        trackingState: String,
+    ) {
+        if (!installed.get()) return
+        nativeDispatchToHostWorklets(
+            nv21Bytes, width, height,
+            qx, qy, qz, qw,
+            tx, ty, tz,
+            timestampNs, trackingState,
+        )
+    }
+
+    /// v0.8.0 Phase 4b.iii — number of registered host worklets.
+    /// Cheap (microsecond) call into the native registry.  Used by
+    /// `RNSARCameraView.onDrawFrame` to gate the per-frame
+    /// NV21-pack + dispatch path: when no worklets are registered
+    /// AND no capture is active, the entire `forwardToIncremental`
+    /// branch can be skipped, saving the ~3-5ms NV21 pack cost per
+    /// idle preview frame.
+    @JvmStatic
+    fun hasHostWorklets(): Boolean {
+        if (!installed.get()) return false
+        return nativeRegistryCount() > 0
+    }
+
+    @JvmStatic
+    private external fun nativeRegistryCount(): Int
+
+    /// JNI binding: `android/src/main/cpp/stitcher_jsi_install_jni.cpp`'s
+    /// `nativeDispatchToHostWorklets`.  Fast-path early-exit lives
+    /// inside the native function — see its docstring.
+    @JvmStatic
+    private external fun nativeDispatchToHostWorklets(
+        nv21Bytes: ByteArray,
+        width: Int,
+        height: Int,
+        qx: Double, qy: Double, qz: Double, qw: Double,
+        tx: Double, ty: Double, tz: Double,
+        timestampNs: Double,
+        trackingState: String,
+    )
+
+    init {
+        // The JSI install module (`StitcherJsiInstallerModule`)
+        // already loads `libimage_stitcher` at class load.  We
+        // load it again here defensively in case
+        // `StitcherWorkletRuntime` is referenced before the install
+        // module — `System.loadLibrary` is idempotent.
+        System.loadLibrary("image_stitcher")
+    }
+}

package/android/src/main/java/io/imagestitcher/rn/TransferredNV21.kt

@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: Apache-2.0
+package io.imagestitcher.rn
+
+/**
+ * v0.10.0 (audit #4A) — single-use NV21 byte-array handle that
+ * enforces the engine's pixel-data ownership contract at runtime.
+ *
+ * ## Why this exists
+ *
+ * `IncrementalStitcher.ingestFromARCameraView` accepts an
+ * `nv21PixelData` parameter that the engine retains for ~50 ms
+ * after the producer thread returns (until the `workScope`
+ * coroutine consumes it).  The documented contract is
+ * "callers MUST treat the array as transferred — do not mutate it
+ * or return it to a buffer pool after calling this method."
+ *
+ * The v0.10.0 audit (`docs/plans/handoff/2026-05-26-autonomous-run-handoff.md`
+ * finding #4A) noted this is by-convention only.  The current AR
+ * caller (`RNSARCameraView`) passes the same `packed.nv21` array
+ * as BOTH `grayData` (consumed synchronously inside the gate)
+ * AND `nv21PixelData` (consumed asynchronously).  Today no race
+ * because the sync read finishes before the async coroutine reads,
+ * but a future refactor that reorders consumption would silently
+ * corrupt frames.
+ *
+ * Wrapping the bytes in `TransferredNV21` turns the documentation
+ * contract into a runtime contract: callers can only extract the
+ * bytes once via `takeOnce()`; the second call throws.  The
+ * misuse is caught at the call site, not at the engine.
+ *
+ * ## Cost
+ *
+ * Construction: tens of ns (one heap allocation for the wrapper +
+ * one volatile write of the bytes reference).  `takeOnce()`: tens
+ * of ns (one synchronized read + one null-out).  Negligible vs the
+ * underlying NV21 array's KB-scale memory footprint and the
+ * ms-scale frame-processing cost — but not a free pointer hop.
+ *
+ * ## Thread-safety
+ *
+ * `takeOnce()` and `available` are `synchronized` on the wrapper
+ * itself.  Producers should still extract on a single thread (the
+ * frame producer); the synchronization defends against the
+ * pathological case where two threads race to extract.
+ */
+class TransferredNV21(bytes: ByteArray) {
+    init {
+        // Empty arrays would propagate as "0 bytes of pixel data with
+        // a non-zero width/height" downstream and crash inside the
+        // C++ ingest with a far less actionable error.  Catch at
+        // construction.  Critic-finding [MAJOR][B].
+        require(bytes.isNotEmpty()) {
+            "TransferredNV21 requires a non-empty byte array " +
+                "(received zero-length)"
+        }
+    }
+
+    @Volatile
+    private var bytes: ByteArray? = bytes
+
+    /**
+     * Take the wrapped bytes.  Throws on second call.
+     *
+     * Consumers should call this exactly once — typically once per
+     * frame, on the producer thread, immediately before handing
+     * the bytes to the async work queue:
+     *
+     * ```kotlin
+     * val pixelBytes: ByteArray? = if (hasPixelData) nv21PixelData!!.takeOnce() else null
+     * workScope.launch {
+     *     // pixelBytes is captured by value; no race.
+     *     engine.addFramePixelData(nv21 = pixelBytes!!, ...)
+     * }
+     * ```
+     *
+     * Concurrency note: `@Volatile` on the bytes field plus the
+     * `synchronized(this)` block here together guarantee both
+     * visibility AND atomicity across threads.  The `@Volatile` is
+     * defensive for any future non-synchronized read; today every
+     * accessor goes through the synchronized block.
+     */
+    fun takeOnce(): ByteArray = synchronized(this) {
+        val b = bytes ?: error(
+            "TransferredNV21.takeOnce() called twice — bytes already transferred. " +
+                "Check that you're not passing the same TransferredNV21 instance to " +
+                "two consumers (e.g., a sync gate-eval call AND an async workScope.launch)."
+        )
+        bytes = null
+        b
+    }
+
+    // Note: an `available` property was considered and removed in
+    // pre-merge review (critic-finding [MAJOR][B]).  Any
+    // `if (handle.available) handle.takeOnce()` pattern is
+    // inherently TOCTOU-racy — another thread could win the
+    // takeOnce() between the check and the use.  Consumers should
+    // call `takeOnce()` directly and catch the `IllegalStateException`
+    // if they need recovery semantics.  No internal caller used
+    // `available`; YAGNI removed it.
+}

package/cpp/stitcher_frame_data.hpp

@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// stitcher_frame_data.hpp — platform-agnostic backing data for the
+// v0.8.0 `StitcherFrame` JSI host object.
+//
+// ## Why this lives here
+//
+// The JSI host object's `get()` dispatch logic is platform-specific
+// (Obj-C++ on iOS includes `<jsi/jsi.h>` from React Native's
+// CocoaPod; Android needs a more elaborate CMake setup to link
+// against React Native's JSI library).  But the *data* the host
+// object exposes — pose, dimensions, the pixel-buffer reader
+// indirection — is identical on both platforms.  That data lives
+// here so iOS / Android JSI dispatch code references one source.
+//
+// ## Memory model
+//
+// `PixelBufferReader` is an opaque interface; platform code (iOS
+// `StitcherFrameHostObject.mm`; Android `stitcher_frame_jni.cpp`)
+// implements it by wrapping the underlying `CVPixelBufferRef` /
+// `ArImage*`.  Lifetime: the reader holds a strong ref to its
+// source for the entire host-object lifetime; releases on
+// destruction (deterministic, RAII).
+//
+// `StitcherFrameData` is value-typed (cheap to copy; ~100 bytes).
+// Construct on the worklet runtime's thread before each dispatch.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <string>
+
+namespace retailens {
+
+/// Opaque interface for reading the underlying camera pixel data.
+/// Platform code provides an implementation:
+///   - iOS: wraps a `CVPixelBufferRef` (locks/unlocks base address
+///     across copyTo, defers release until destruction).
+///   - Android: wraps an ARCore `ArImage*` (handles plane access via
+///     `ArImage_getPlaneData`, calls `ArImage_release` on destruct).
+///
+/// **Thread-affinity contract:** implementations need not be
+/// reentrant.  An instance MAY be constructed on thread A
+/// (typically the ARSession delegate queue) and used on thread B
+/// (the worklet-runtime thread), provided the construction-thread
+/// releases its `shared_ptr` reference before thread B uses the
+/// reader.  The `shared_ptr`'s atomic refcount serves as the
+/// happens-before barrier — fields set in the constructor are
+/// visible on the worklet thread once the construction-thread
+/// drops its ref.  Concurrent access from two threads simultaneously
+/// is NOT supported.
+class PixelBufferReader {
+public:
+    virtual ~PixelBufferReader() = default;
+
+    /// Total byte size of the buffer the reader exposes.  For Y-plane
+    /// access (the v0.8.0 default), this is `width * height`.
+    virtual std::size_t byteSize() const = 0;
+
+    /// Copy up to `maxBytes` of the underlying buffer into `dst`.
+    /// Returns bytes written.  Returns 0 if reader is invalidated.
+    ///
+    /// Implementations MUST handle the case where `maxBytes < byteSize()`
+    /// (clip silently).  This matches JS `ArrayBuffer.slice` semantics
+    /// even though the host object always allocates exactly `byteSize()`.
+    virtual std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) = 0;
+};
+
+/// Plain-old-data payload for one `StitcherFrame`.  Fully extracted
+/// at construction time (cheap fields) plus an opaque reader for
+/// the lazy pixel access.
+struct StitcherFrameData {
+    /// Discriminator. `"ar"` for AR-mode frames, `"vc"` for
+    /// vision-camera frames.  Used by worklets to gate on AR-only
+    /// field access (translation, depth, anchors, tracking state).
+    /// Mirrored to the JS `source` field (standard discriminated-
+    /// union pattern).
+    std::string source;
+
+    /// Width / height of the camera image in pixels.
+    int32_t width = 0;
+    int32_t height = 0;
+
+    /// String pixel-format identifier; matches the JS
+    /// `StitcherFrame.pixelFormat` union: `"yuv"` / `"rgb"` /
+    /// `"unknown"`.  Today's emitters always populate `"yuv"`
+    /// (NV12 on iOS, NV21 on Android).
+    std::string pixelFormat;
+
+    /// String orientation identifier; matches vision-camera's
+    /// `Frame.orientation`: `"portrait"`, `"portrait-upside-down"`,
+    /// `"landscape-left"`, `"landscape-right"`.
+    std::string orientation;
+
+    /// Monotonic timestamp in nanoseconds.  AR mode: from
+    /// `ARFrame.timestamp` (CFAbsoluteTime, converted to ns).
+    /// Non-AR mode: from `vision-camera Frame.timestamp` (already ns).
+    double timestampNs = 0.0;
+
+    /// Pose rotation as quaternion `(x, y, z, w)`.  Matches the
+    /// `q = q_yaw * q_pitch * q_roll` convention used elsewhere in
+    /// the lib (KeyframeGate, RNSARFramePose, AcceptedKeyframe).
+    double qx = 0.0;
+    double qy = 0.0;
+    double qz = 0.0;
+    double qw = 1.0;
+
+    /// Pose translation in metres (world coords).  AR mode: from
+    /// `ARFrame.camera.transform`.  Non-AR mode: undefined — the
+    /// `hasTranslation` flag is `false` and JS receives
+    /// `pose.translation === undefined`.
+    double tx = 0.0;
+    double ty = 0.0;
+    double tz = 0.0;
+    bool hasTranslation = false;
+
+    /// AR tracking state.  Empty string (`""`) means "not
+    /// applicable" (the JS host object exposes `arTrackingState ===
+    /// undefined` in that case).  Otherwise one of `"notAvailable"`,
+    /// `"limited"`, `"normal"`.
+    std::string arTrackingState;
+
+    /// Pixel data accessor.  Always present (even for AR mode where
+    /// arDepth might be the more interesting buffer).  See class
+    /// docstring for lifetime contract.
+    std::shared_ptr<PixelBufferReader> pixelReader;
+
+    // ── AR-only optional fields (not populated in v0.8.0; stubs) ──
+    // These are deferred to v0.8.1+ because the host worklets that
+    // would consume them aren't shipping in v0.8.0 either.  Adding
+    // them here as plain data fields keeps the JSI host object code
+    // simple when they DO arrive.
+
+    /// arDepth, arAnchors stubs intentionally omitted — they're
+    /// fields the JSI dispatch will return `undefined` for in v0.8.0.
+    /// v0.8.1+ adds them here as `std::optional<ArDepth>` etc.
+};
+
+}  // namespace retailens