react-native-image-stitcher 0.7.0 → 0.8.0

package/dist/stitching/useFrameProcessor.js

@@ -0,0 +1,196 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.useFrameProcessor = useFrameProcessor;
+const react_1 = require("react");
+const react_native_vision_camera_1 = require("react-native-vision-camera");
+const ensureStitcherProxyInstalled_1 = require("./ensureStitcherProxyInstalled");
+const StitcherWorkletRegistry_1 = require("./StitcherWorkletRegistry");
+/**
+ * v0.8.0 Phase 4a — public hook for hosts to attach a per-frame
+ * worklet that runs in BOTH AR and non-AR capture modes.
+ *
+ * ## Quick start
+ *
+ * ```tsx
+ * import { useFrameProcessor, type StitcherFrame } from 'react-native-image-stitcher';
+ *
+ * function MyOcrOverlay() {
+ *   const processor = useFrameProcessor((frame: StitcherFrame) => {
+ *     'worklet';
+ *     // Pixel data is in `frame.toArrayBuffer()`.
+ *     // AR-only fields: `frame.arDepth`, `frame.arAnchors`, `frame.arTrackingState`.
+ *     // Discriminate via `frame.source === 'ar'` / `'vc'`.
+ *   }, []);
+ *   return <Camera frameProcessor={processor} ... />;
+ * }
+ * ```
+ *
+ * ## Two behaviours, depending on mode
+ *
+ * **Non-AR mode (today, fully working):** the worklet runs on
+ * vision-camera's Frame Processor runtime.  Same thread + same
+ * cost envelope as a plain `useFrameProcessor` from
+ * `react-native-vision-camera`.  The lib's own first-party
+ * stitching plugin runs alongside on the same producer-thread
+ * runtime (composition is handled by vision-camera's own dispatch
+ * order).
+ *
+ * Your worklet receives whatever vision-camera delivers — vc's raw
+ * `Frame`.  This is a structural subset of `StitcherFrame`: the
+ * vc-shaped fields (`width`, `height`, `pixelFormat`, `orientation`,
+ * `timestamp`, `toArrayBuffer`) are guaranteed; the
+ * `StitcherFrame`-only fields (`source`, `pose`, `arDepth`,
+ * `arAnchors`, `arTrackingState`) are **undefined** at runtime
+ * because the lib does NOT wrap or augment vc's `Frame` in Phase 4a
+ * (cross-worklet-boundary field injection is Phase 4b work).
+ * Worklets that need to read `source` / `pose` MUST guard for
+ * `undefined`:
+ *
+ * ```ts
+ * if (frame.source === 'ar') { ... }   // false in non-AR mode
+ * if (frame.pose) { ... }              // skipped in non-AR mode
+ * ```
+ *
+ * **AR mode — iOS Phase 4b.i (this release):** the worklet is
+ * installed into the native registry via
+ * `globalThis.__stitcherProxy.install(workletFn)`, where
+ * `__stitcherProxy` is a JSI host object installed at lib
+ * bootstrap by the native `StitcherJsiInstaller` module.  The
+ * AR worklet runtime (`RNSARWorkletRuntime`) reads from the
+ * native registry on each `dispatchFrame:pose:` call and fans
+ * out invocations — your worklet fires alongside the lib's
+ * first-party stitching path.
+ *
+ * **AR mode — Android Phase 4b.ii (deferred):** the native
+ * installer + JNI bridge from `StitcherWorkletRuntime.kt`'s
+ * `runFirstParty {...}` path to a parallel C++ registry land in
+ * a follow-up release.  Until then, on Android the hook falls
+ * back to the JS-side `StitcherWorkletRegistry`; AR-mode host
+ * worklets register but do not invoke.  No regression vs.
+ * Phase 4a; iOS gets the API first.
+ *
+ * ### When Phase 4b.ii lands (Android)
+ *
+ * The hook's call signature does NOT change.  Android hosts that
+ * write code today against this API will see their worklets
+ * start firing in AR mode automatically when Phase 4b.ii is
+ * merged.  No migration required.
+ *
+ * ## Frame contract
+ *
+ * The worklet receives a {@link StitcherFrame} (see
+ * `src/stitching/StitcherFrame.ts` for the full contract +
+ * lifecycle).  Highlights:
+ *
+ *   - **`source`** discriminator: `'vc'` or `'ar'`.  Branch on this
+ *     before reading `arDepth` / `arAnchors` / `arTrackingState`
+ *     so non-AR captures don't break.
+ *   - **`pose`** always present.  `pose.translation` is `undefined`
+ *     in non-AR mode (gyro provides only rotation; no spatial
+ *     anchor).
+ *   - **Buffer lifetime**: pixel data is valid only for the
+ *     duration of the worklet call.  Worklets that need to retain
+ *     data must `toArrayBuffer()` synchronously inside the
+ *     worklet body — returning a reference and reading it later
+ *     reads freed memory.
+ *
+ * ## Threading
+ *
+ * The worklet runs on the producer thread (vision-camera's
+ * runtime in non-AR mode; the AR-session callback thread under
+ * Phase 4b).  Worklets MUST NOT block the producer thread for
+ * more than a few ms — the next frame's processing is gated on
+ * the previous frame returning.  Long work belongs on a queue
+ * crossed via Reanimated / worklets-core's `runOnJS`.
+ *
+ * @param worklet  The host's frame processor function.  Must be
+ *                 `'worklet'`-prefixed at the call site.  TS
+ *                 cannot enforce the prefix; the runtime will
+ *                 throw at attempt to invoke a non-worklet
+ *                 function.
+ * @param deps     Standard React deps array.  When `deps` change,
+ *                 the previous registration is removed and the
+ *                 new worklet is registered.  Same semantics as
+ *                 vision-camera's `useFrameProcessor`.
+ *
+ * @returns A vision-camera frame-processor object that
+ *          `<Camera frameProcessor={...}>` accepts.  In non-AR
+ *          mode this is what drives the per-frame worklet
+ *          invocation; in AR mode it's currently a no-op (vc
+ *          isn't mounted in AR mode anyway).
+ */
+function useFrameProcessor(worklet, deps) {
+    // Non-AR path: delegate to vision-camera's hook.  The returned
+    // processor object is what `<Camera>` hands to vc.  Worklet
+    // fires on vc's producer-thread runtime.
+    //
+    // Cast rationale: vc's hook expects `(frame: Frame) => void`.
+    // Our worklet is typed `(frame: StitcherFrame) => void`.
+    // `StitcherFrame` is a structural superset of `Frame` (it adds
+    // required `source` + `pose` and the optional AR fields), so
+    // assigning a function that consumes `StitcherFrame` to a
+    // `Frame`-consuming slot is unsound at the type level — TS is
+    // right to reject the direct assignment.  At RUNTIME the worklet
+    // will see vc's raw `Frame`; the `source` / `pose` / AR fields
+    // are undefined (the hook's docstring above documents this and
+    // tells hosts to guard).  We double-cast through `unknown` to
+    // suppress, accepting the explicit type-system gap as the price
+    // of Phase 4a's pre-Phase-4b deferral on cross-runtime frame
+    // wrapping.
+    const vcProcessor = (0, react_native_vision_camera_1.useFrameProcessor)(worklet, deps);
+    // AR path: install into the native registry if available (iOS
+    // Phase 4b.i — and Android Phase 4b.ii once it lands).  Falls
+    // back to the JS-side `StitcherWorkletRegistry` when the native
+    // installer isn't present (Android in 4b.i; remote debug mode;
+    // unit tests).  The fallback path matches Phase 4a's
+    // register-but-not-invoke semantics.
+    //
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    (0, react_1.useEffect)(() => {
+        const nativeReady = (0, ensureStitcherProxyInstalled_1.ensureStitcherProxyInstalled)();
+        if (nativeReady &&
+            typeof globalThis.__stitcherProxy !== 'undefined') {
+            // Native path — install through the JSI proxy.  Errors here
+            // most commonly mean the worklet doesn't have the
+            // `'worklet'` directive at the call site (the worklets-core
+            // babel plugin didn't transform it).  Surface them via the
+            // proxy's own throw with a host-side log so the failure is
+            // obvious.
+            let id;
+            try {
+                id = globalThis.__stitcherProxy.install(worklet);
+            }
+            catch (err) {
+                // Guard `__DEV__` read so the hook works in any environment
+                // that imports it without defining the flag (jest, SSR,
+                // custom tooling).
+                if (typeof __DEV__ !== 'undefined' && __DEV__) {
+                    console.error('[react-native-image-stitcher] __stitcherProxy.install ' +
+                        'threw — is the worklet function decorated with ' +
+                        "`'worklet';` and processed by react-native-worklets-core's " +
+                        'babel plugin?  Original error: ' +
+                        String(err));
+                }
+                return; // No cleanup needed — nothing was installed.
+            }
+            return () => {
+                try {
+                    globalThis.__stitcherProxy.uninstall(id);
+                }
+                catch {
+                    // Uninstall is best-effort; an exception here means the
+                    // proxy was already gone (e.g., app reload mid-cleanup).
+                }
+            };
+        }
+        // Fallback — JS-side registry.  Same as Phase 4a.
+        const jsId = StitcherWorkletRegistry_1.StitcherWorkletRegistry.register({
+            worklet,
+            isFirstParty: false,
+        });
+        return () => StitcherWorkletRegistry_1.StitcherWorkletRegistry.unregister(jsId);
+    }, deps);
+    return vcProcessor;
+}
+//# sourceMappingURL=useFrameProcessor.js.map

package/ios/Sources/RNImageStitcher/RNSARSession.swift

@@ -484,10 +484,41 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
                Int32(config.videoFormat.framesPerSecond))
         isRunning = true
         currentTrackingState = .initialising
+
+        // v0.8.0 Phase 3c — install the worklet runtime + register
+        // the first-party stitching callback.  The delegate's
+        // per-frame ingest now routes through
+        // `RNSARWorkletRuntime.dispatchFrame` (see
+        // `session(_:didUpdate:)` below) which invokes this
+        // callback synchronously.  Net behavior is byte-identical
+        // to the pre-Phase-3c direct `consumer.consumeFrame(...)`
+        // call.  The indirection sets up the seam where Phase 4
+        // will fan out to host worklets without touching this
+        // first-party path.
+        RNSARWorkletRuntime.shared().installIfNeeded()
+        RNSARWorkletRuntime.shared().setFirstPartyCallback {
+            [weak self] arFrame, pose in
+            // ARKit pool reuse contract: must consume the pixel
+            // buffer before returning.  The consumer's
+            // `consumeFrame` does that synchronously inside the
+            // call (NV12 → cv::Mat sync, then heavy work on its
+            // own queue).  We're on the same thread as the
+            // delegate (ARSession.delegateQueue), so the contract
+            // holds end-to-end.
+            guard let self = self else { return }
+            guard let consumer = self.incrementalConsumer else { return }
+            consumer.consumeFrame(pixelBuffer: arFrame.capturedImage,
+                                  pose: pose)
+        }
     }
 
     @objc public func stop() {
         guard isRunning else { return }
+        // v0.8.0 Phase 3c — drop the first-party callback so the
+        // closure's `[weak self]` reference can be released
+        // immediately + no in-flight delegate frame re-enters the
+        // engine after stop.  Idempotent.
+        RNSARWorkletRuntime.shared().setFirstPartyCallback(nil)
         arSession.pause()
         isRunning = false
         currentTrackingState = .notAvailable
@@ -561,16 +592,21 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
             }
         }
 
-        // Deliver this frame to the live incremental-stitching
-        // consumer if one is registered.  The consumer MUST consume
-        // the pixel buffer before returning (Apple's ARKit pool
-        // reuse contract — same constraint as the recording-append
-        // path below) — `IncrementalStitcher` does this by
-        // converting NV12 → cv::Mat synchronously inside the call,
-        // then doing the heavy work on its own queue.
-        if let consumer = self.incrementalConsumer {
-            consumer.consumeFrame(pixelBuffer: frame.capturedImage, pose: pose)
-        }
+        // v0.8.0 Phase 3c — route the per-frame ingest through the
+        // worklet runtime instead of calling the consumer directly.
+        // The first-party callback (installed in `start()` above)
+        // wraps the same `consumer.consumeFrame(pixelBuffer:pose:)`
+        // call path, so net behavior is byte-identical to v0.7.x.
+        // The indirection sets up the seam where Phase 4 will fan
+        // out to host worklets (registered via the v0.8.0
+        // `useFrameProcessor` TS hook + a JSI plugin entry point)
+        // without changing this first-party path.
+        //
+        // ARKit pool reuse contract: still satisfied — the runtime
+        // invokes the callback synchronously on the delegate
+        // thread, and the callback's `consumer.consumeFrame(...)`
+        // does the same NV12 → cv::Mat sync conversion as before.
+        RNSARWorkletRuntime.shared().dispatchFrame(frame, pose: pose)
 
         // If recording is in flight, append this frame to the
         // asset writer DIRECTLY — no queue hop.

package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// RNSARWorkletRuntime.h — Obj-C facade for the v0.8.0 AR-mode
+// worklet runtime.  Wraps `react-native-worklets-core`'s
+// `RNWorklet::JsiWorkletContext` (the same primitive vision-camera
+// uses for its Frame Processor runtime) so the lib can dispatch
+// per-ARFrame worklets on a thread we own — rather than ARKit's
+// delegate queue, where doing significant work would block the
+// AR session's update loop.
+//
+// ## Phase 3b scope (this commit)
+//
+// Owns:
+//   - The dispatch queue the worklet runtime pins to.
+//   - The underlying `JsiWorkletContext` (constructed lazily on
+//     `installIfNeeded`, lives for the singleton's lifetime).
+//
+// Exposes:
+//   - `+ shared` singleton accessor.
+//   - `- installIfNeeded` (idempotent runtime construction).
+//   - `- isInstalled` for diagnostics + tests.
+//   - `- dispatchFrame:pose:` — currently a no-op stub; Phase 3c
+//     fills in the actual host-object construction + worklet
+//     invocation + first-party stitching dispatch.
+//
+// Host-worklet registry is intentionally NOT in Phase 3b — Phase 4
+// lands the JSI plugin + TS-side hook that defines the storage
+// shape (NSMutableArray of boxed shared_ptrs vs a C++ vector ivar
+// vs something else).  Pre-committing the storage type here would
+// risk rework.  See
+// `docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md`
+// Phase 4 section for the planned API.
+//
+// ## Why Obj-C facade with `.mm` implementation
+//
+// The implementation needs to hold `std::shared_ptr<JsiWorkletContext>`
+// + run JSI value construction, which can't live in pure Swift.  Same
+// pattern as `KeyframeGateBridge.{h,mm}` + `StitcherFrameHostObject.{h,mm}`:
+// keep the header umbrella-safe (no JSI imports), put the C++ glue in
+// the .mm.
+//
+// ## Header umbrella safety
+//
+// This .h imports only Foundation + ARKit (both system frameworks).
+// Worklets-core types are confined to the .mm.
+
+#pragma once
+
+#import <Foundation/Foundation.h>
+#import <ARKit/ARKit.h>
+
+@class RNSARFramePose;
+
+NS_ASSUME_NONNULL_BEGIN
+
+NS_SWIFT_NAME(RNSARWorkletRuntime)
+@interface RNSARWorkletRuntime : NSObject
+
+/// Singleton accessor.  One AR worklet runtime per process; multiple
+/// `<Camera>` mounts share it.  Construction is cheap (just an Obj-C
+/// alloc + an `NSMutableArray`); the heavy JSI work happens in
+/// `-installIfNeeded`.
++ (instancetype)shared;
+
+/// Construct the underlying `JsiWorkletContext` if not yet
+/// installed.  Idempotent — repeated calls are no-ops.  Called from
+/// `RNSARSession` at AR-mode start time (Phase 3c will wire this
+/// up; Phase 3b ships the method but no one calls it yet).
+///
+/// Threading: safe to call from any thread; internally serialised.
+/// The runtime's own dispatch queue starts running once installed.
+- (void)installIfNeeded;
+
+/// Diagnostics + tests.  Returns `YES` after a successful
+/// `-installIfNeeded`.
+- (BOOL)isInstalled;
+
+/// Phase 3c — type of the first-party stitching callback.  Invoked
+/// synchronously on the caller thread (`ARSession.delegateQueue` —
+/// typically main queue today) per AR frame.  Block must consume
+/// the pixel buffer before returning (ARKit pool reuse contract).
+typedef void (^RNSARFirstPartyCallback)(ARFrame *arFrame,
+                                         RNSARFramePose *pose);
+
+/// Phase 3c — install the closure that takes ownership of the
+/// per-frame first-party stitching dispatch.  Called from
+/// `RNSARSession.start()` after the incremental consumer is set;
+/// the block then routes `dispatchFrame:pose:` calls through to
+/// the existing `incrementalConsumer.consumeFrame(...)` path.
+///
+/// Pre-Phase-3c the delegate called the consumer directly.  After
+/// Phase 3c the delegate calls `dispatchFrame:pose:` (this class)
+/// which invokes the callback.  Net behavior is byte-identical;
+/// the indirection sets up the seam where Phase 4 will fan out to
+/// host worklets without changing the first-party path.
+///
+/// Pass `nil` to clear (e.g. on `RNSARSession.stop()`).  Idempotent.
+- (void)setFirstPartyCallback:(nullable RNSARFirstPartyCallback)callback;
+
+/// Dispatch one AR frame through the registered worklets.  Called
+/// per `ARFrame` by `RNSARSession.delegate` once Phase 3c lands the
+/// migration (Phase 3b ships this method as a no-op stub so the
+/// runtime can be built + linked + the API surface fixed).
+///
+/// The Phase 3c implementation will:
+///   1. Build a `StitcherFrameHostObject` from `arFrame` + `pose`.
+///   2. Run the first-party stitching synchronously on the caller
+///      thread (preserves today's `ingestFromARCameraView` cost
+///      envelope at the producer site).
+///   3. If any host worklets are registered, dispatch the host
+///      object onto the worklet runtime's thread + invoke each
+///      worklet via `RNWorklet::WorkletInvoker::call`.
+///   4. Invalidate the host object after all worklets return.
+///
+/// Phase 3c gate: install/idempotence tests + this method's
+/// integration test required before merge.  See
+/// `docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md`
+/// Phase 3c gate criteria.
+///
+/// Threading: typically called from `ARSession.delegateQueue` (main
+/// queue by default; Phase 3c will pin it explicitly to a
+/// dedicated queue).
+- (void)dispatchFrame:(ARFrame *)arFrame pose:(RNSARFramePose *)pose
+    NS_SWIFT_NAME(dispatchFrame(_:pose:));
+
+@end
+
+NS_ASSUME_NONNULL_END