react-native-image-stitcher 0.18.0 → 0.19.0

package/ios/Sources/RNImageStitcher/RNSARSession.swift

@@ -492,6 +492,24 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
     private var lastArFrameMetaEmit: TimeInterval = 0
     private let arFrameMetaLock = NSLock()
 
+    // ──────────────────────────────────────────────────────────────
+    // v0.19.0 — native AR plugin framework (RNISARPluginRegistry)
+    // ──────────────────────────────────────────────────────────────
+    //
+    // While the plugin registry is NON-EMPTY, the per-frame path builds a
+    // `RNISARFrameContext` once and calls each registered plugin's
+    // `process(_:)` on the AR (delegate) thread (see `invokeArPlugins`).
+    // Non-nil SYNC results are cached here so the throttled `onArFrame`
+    // meta build can fold them in under `plugins: { [name]: result }`
+    // without re-running plugins.  Zero-plugin apps skip the whole path
+    // (the registry's `isEmpty` gate), so they pay nothing.
+    //
+    // Written on the AR thread (per-frame) and read on the same thread
+    // (the meta build runs inline in `session(_:didUpdate:)`), but guarded
+    // anyway for defensiveness against any future off-thread reader.
+    private var latestPluginSyncResults: [String: Any] = [:]
+    private let pluginSyncResultsLock = NSLock()
+
     private override init() {
         super.init()
         arSession.delegate = self
@@ -594,6 +612,12 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         detectedPlaneTransformInternal = nil
         bestRejectedAlignment = -1.0
         planeLatchLock.unlock()
+        // v0.19.0 — drop any cached SYNC plugin results so the next
+        // capture's `onArFrame` meta doesn't surface stale plugin output
+        // before the first frame of the new session runs the plugins.
+        pluginSyncResultsLock.lock()
+        latestPluginSyncResults = [:]
+        pluginSyncResultsLock.unlock()
     }
 
     /// Build the `ARWorldTrackingConfiguration` shared by `start()` and
@@ -846,6 +870,16 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         // double-consuming would ingest each frame twice.
         RNSARWorkletRuntime.shared().dispatchFrame(frame, pose: pose)
 
+        // v0.19.0 — native AR plugin framework.  When the registry is
+        // non-empty, build the per-frame `RNISARFrameContext` once and run
+        // every registered plugin's `process(_:)` SYNCHRONOUSLY on this AR
+        // thread (so the live pixel/depth buffers are valid for the call).
+        // Caches non-nil SYNC results for the meta build below.  Cheap
+        // no-op when no plugins are registered (the common case).  Runs
+        // BEFORE `maybeEmitArFrameMeta` so the throttled `onArFrame` meta
+        // can fold in this frame's freshest plugin results.
+        invokeArPlugins(frame, pose: pose)
+
         // v0.18.0 — `onArFrame` LIGHT-metadata channel.  Gated +
         // throttled; builds the ARFrameMeta dictionary and posts it for
         // the bridge to re-emit.  Cheap no-op when disabled (the common
@@ -1433,13 +1467,105 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         // Obj-C++ extraction helpers + the shared C++ extraction-config
         // gating (depth/anchors/mesh ⇐ enableDepth/enableAnchors/enableMesh).
         let meta = CameraFrameHostObject.lightArFrameMeta(from: frame, pose: pose)
+
+        // v0.19.0 — fold in any SYNC plugin results captured by
+        // `invokeArPlugins` for the freshest frames.  Only attach the
+        // `plugins` key when there's at least one result, so the common
+        // (no-plugin) meta shape is unchanged.  Snapshot under the lock,
+        // then bridge into a fresh dictionary copy.
+        pluginSyncResultsLock.lock()
+        let pluginResults = latestPluginSyncResults
+        pluginSyncResultsLock.unlock()
+        let userInfo: [AnyHashable: Any]
+        if pluginResults.isEmpty {
+            userInfo = meta
+        } else {
+            var withPlugins = meta
+            withPlugins["plugins"] = pluginResults
+            userInfo = withPlugins
+        }
+
         NotificationCenter.default.post(
             name: .retailensARFrameMeta,
             object: nil,
-            userInfo: meta
+            userInfo: userInfo
         )
     }
 
+    /// v0.19.0 — run all registered native AR plugins for this frame.
+    /// Gated on the registry being NON-EMPTY (the cheap `isEmpty` check) so
+    /// zero-plugin apps skip the context build entirely.  When plugins are
+    /// present, builds ONE `RNISARFrameContext` (zero-copy view of the
+    /// frame's live buffers + the already-built anchor dicts) and calls
+    /// each plugin's `process(_:)` SYNCHRONOUSLY on this AR (delegate)
+    /// thread — so the live `pixelBuffer` / `depthBuffer` are valid for the
+    /// call (the plugin must copy before offloading; see the protocol
+    /// docstring).  Non-nil SYNC results are cached in
+    /// `latestPluginSyncResults` for the throttled `onArFrame` meta to fold
+    /// in; ASYNC results arrive later via `RNISARPluginRegistry.emit`.
+    private func invokeArPlugins(_ frame: ARFrame, pose: RNSARFramePose) {
+        let registry = RNISARPluginRegistry.shared
+        guard !registry.isEmpty else { return }
+        let plugins = registry.plugins()
+        guard !plugins.isEmpty else { return }
+
+        // depthBuffer: expose the live sceneDepth map ONLY when the
+        // `<Camera enableDepth>` prop is on (gating read in Obj-C++ so the
+        // C++ extraction-config header stays out of Swift).  Prefer
+        // `sceneDepth`, fall back to `smoothedSceneDepth` — same precedence
+        // as the full extraction path.
+        var depthBuffer: CVPixelBuffer? = nil
+        if CameraFrameHostObject.arExtractionDepthEnabled() {
+            if let dd = frame.sceneDepth ?? frame.smoothedSceneDepth {
+                depthBuffer = dd.depthMap
+            }
+        }
+
+        // anchors: reuse the EXACT light dicts the `onArFrame` meta builds
+        // (gated on `enableAnchors`; empty otherwise) — DRY single source.
+        let anchorDicts = CameraFrameHostObject.arAnchorDicts(from: frame)
+        let anchors = anchorDicts as? [[String: Any]] ?? []
+
+        let context = RNISARFrameContext(
+            pixelBuffer: frame.capturedImage,
+            timestampNs: frame.timestamp * 1e9,
+            fx: pose.fx, fy: pose.fy, cx: pose.cx, cy: pose.cy,
+            imageWidth: pose.imageWidth, imageHeight: pose.imageHeight,
+            poseRotation: [pose.qx, pose.qy, pose.qz, pose.qw],
+            poseTranslation: [pose.tx, pose.ty, pose.tz],
+            trackingState: Self.trackingStateString(pose.trackingState),
+            depthBuffer: depthBuffer,
+            anchors: anchors
+        )
+
+        var syncResults: [String: Any] = [:]
+        for plugin in plugins {
+            // Defensive: a plugin throwing/crashing in `process` would take
+            // down the AR thread, but Swift has no try/catch for non-Error
+            // crashes — the contract is that plugins are well-behaved.  We
+            // simply collect non-nil results keyed by the plugin's name.
+            if let result = plugin.process(context) {
+                syncResults[plugin.name()] = result
+            }
+        }
+
+        pluginSyncResultsLock.lock()
+        latestPluginSyncResults = syncResults
+        pluginSyncResultsLock.unlock()
+    }
+
+    /// Map the SDK's `RNSARTrackingState` to the same string the
+    /// `onArFrame` meta + `CameraFrame.trackingState` use, so plugins see a
+    /// consistent vocabulary.
+    private static func trackingStateString(_ s: RNSARTrackingState) -> String {
+        switch s {
+        case .tracking:     return "normal"
+        case .limited:      return "limited"
+        case .initialising: return "limited"
+        case .notAvailable: return "notAvailable"
+        }
+    }
+
     private func makePose(from frame: ARFrame) -> RNSARFramePose {
         // ARKit's transform is a 4x4 matrix; extract translation
         // (last column) and rotation (top-left 3x3 → quaternion).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-image-stitcher",
-  "version": "0.18.0",
+  "version": "0.19.0",
   "description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/camera/ARCameraView.tsx

@@ -44,7 +44,7 @@ import {
 
 import { ensureStitcherProxyInstalled } from '../stitching/ensureStitcherProxyInstalled';
 import type { CameraFrameProcessor } from '../stitching/CameraFrame';
-import type { ARFrameMeta } from '../stitching/ARFrameMeta';
+import type { ARFrameMeta, ARPluginResult } from '../stitching/ARFrameMeta';
 
 
 // React Native looks up the component by its NATIVE name.
@@ -145,6 +145,28 @@ export interface ARCameraViewProps {
    * (≈ 10 Hz).  No effect unless `onArFrame` is provided.
    */
   arFrameMetaInterval?: number;
+
+  /**
+   * v0.19.0 — ASYNCHRONOUS AR-plugin result callback, invoked on the JS MAIN
+   * thread (NOT a worklet).  Part of the AR plugin framework: host-registered
+   * native plugins (see `RNISARPluginRegistry` / `RNSARPluginRegistry`) can
+   * offload heavy per-frame work to their own queue and later push a result
+   * via `registry.emit(name, result)`.  The SDK routes that to JS as a
+   * `RNImageStitcherARPluginResult` device event; when this prop is provided,
+   * this component subscribes and invokes the handler with
+   * `{ plugin, result }`.
+   *
+   * SYNCHRONOUS plugin results (computed inline on the AR thread) instead ride
+   * the throttled {@link onArFrame} event on {@link ARFrameMeta.plugins} —
+   * read them there.  This callback is ONLY for the out-of-band async channel.
+   *
+   * The subscription is independent of {@link onArFrame}: a host can read
+   * sync results via `onArFrame` and async results via `onArPluginResult`,
+   * either, or both.  Wiring mirrors `onArFrame` exactly (latest handler held
+   * in a ref so the subscription effect depends only on whether a handler is
+   * present; cleanup on unmount / when the handler is removed).
+   */
+  onArPluginResult?: (e: ARPluginResult) => void;
 }
 
 
@@ -237,6 +259,7 @@ export const ARCameraView = forwardRef<ARCameraViewHandle, ARCameraViewProps>(
       planeDetection,
       onArFrame,
       arFrameMetaInterval,
+      onArPluginResult,
     },
     ref,
   ): React.JSX.Element {
@@ -362,6 +385,48 @@ export const ARCameraView = forwardRef<ARCameraViewHandle, ARCameraViewProps>(
       };
     }, [arFrameEnabled, arFrameMetaInterval]);
 
+    // v0.19.0 — onArPluginResult device-event wiring (worklet-free, main
+    // thread).  Mirrors the onArFrame subscription above: the latest handler
+    // is held in a ref so the subscription effect depends only on WHETHER a
+    // handler is present, not its (per-render-changing) identity — so the
+    // native event subscription isn't torn down + re-established every render.
+    //
+    // This is a PURELY-JS subscription: unlike onArFrame there's no native
+    // "enable" toggle to flip.  Native emits `RNImageStitcherARPluginResult`
+    // whenever a registered plugin calls `registry.emit(...)`; the registry is
+    // empty unless the host registered plugins, so an app with no plugins
+    // never sees an event even if this prop is wired.
+    const onArPluginResultRef = useRef<
+      ((e: ARPluginResult) => void) | undefined
+    >(onArPluginResult);
+    useEffect(() => {
+      onArPluginResultRef.current = onArPluginResult;
+    }, [onArPluginResult]);
+
+    const arPluginResultEnabled = onArPluginResult != null;
+    useEffect(() => {
+      if (!arPluginResultEnabled) {
+        return undefined;
+      }
+      const native = (NativeModules as Record<string, unknown>)
+        .RNSARSession;
+      if (native == null) {
+        // Native module unavailable (e.g. web, or a native build predating
+        // the plugin event channel): no-op, no crash.
+        return undefined;
+      }
+      const emitter = new NativeEventEmitter(native as never);
+      const sub = emitter.addListener(
+        'RNImageStitcherARPluginResult',
+        (e: ARPluginResult) => {
+          onArPluginResultRef.current?.(e);
+        },
+      );
+      return () => {
+        sub.remove();
+      };
+    }, [arPluginResultEnabled]);
+
     useImperativeHandle(ref, () => ({
       takePhoto: async (options = {}) => {
         const native: any =

package/src/camera/Camera.tsx

@@ -67,7 +67,7 @@ import type {
 
 import { useARSession } from '../ar/useARSession';
 import type { CameraFrameProcessor } from '../stitching/CameraFrame';
-import type { ARFrameMeta } from '../stitching/ARFrameMeta';
+import type { ARFrameMeta, ARPluginResult } from '../stitching/ARFrameMeta';
 import { ARCameraView, type ARCameraViewHandle } from './ARCameraView';
 import { CameraShutter } from './CameraShutter';
 import { CameraView } from './CameraView';
@@ -814,6 +814,26 @@ export interface CameraProps {
    */
   arFrameMetaInterval?: number;
 
+  /**
+   * v0.19.0 — ASYNCHRONOUS AR-plugin result callback (the AR plugin
+   * framework), invoked on the JS MAIN thread (NOT a worklet).  Only fires in
+   * AR capture (`captureSource === 'ar'`).  Host-registered native plugins
+   * (see `RNISARPluginRegistry` / `RNSARPluginRegistry`) that offload heavy
+   * per-frame work to their own queue push results via
+   * `registry.emit(name, result)`; `<Camera>` threads this handler to
+   * `<ARCameraView>`, which subscribes to the `RNImageStitcherARPluginResult`
+   * device event and invokes it with `{ plugin, result }`.
+   *
+   * SYNCHRONOUS plugin results (computed inline on the AR thread) instead ride
+   * the throttled {@link onArFrame} event on {@link ARFrameMeta.plugins}.
+   * Use `onArFrame` for the in-band sync channel and `onArPluginResult` for
+   * the out-of-band async channel — a host can wire either or both.
+   *
+   * The SDK ships ONLY the generic plugin framework; there are no built-in
+   * plugins, so this never fires unless the host registers native plugins.
+   */
+  onArPluginResult?: (e: ARPluginResult) => void;
+
   // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────────
   /**
    * Which device holds the non-AR panorama capture accepts.
@@ -1227,6 +1247,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
     planeDetection,
     onArFrame,
     arFrameMetaInterval,
+    onArPluginResult,
     engine = 'batch-keyframe',
     // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
     panMode = 'vertical',
@@ -2497,6 +2518,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
           planeDetection={planeDetection}
           onArFrame={onArFrame}
           arFrameMetaInterval={arFrameMetaInterval}
+          onArPluginResult={onArPluginResult}
         />
       ) : (
         <CameraView

package/src/index.ts

@@ -265,8 +265,13 @@ export type {
 } from './stitching/CameraFrame';
 // v0.18.0 — LIGHT per-frame AR metadata delivered via the `onArFrame`
 // callback (main-thread, worklet-free).  See the type's docstring for why
-// it bypasses the worklet path.
+// it bypasses the worklet path.  v0.19.0 adds `plugins` (sync results from
+// host-registered AR plugins ride this same throttled event).
 export type { ARFrameMeta } from './stitching/ARFrameMeta';
+// v0.19.0 — the AR plugin framework's ASYNC result type, delivered via the
+// `onArPluginResult` callback (a plugin's out-of-band `registry.emit(...)`
+// result).  The SDK ships only the generic framework — no built-in plugins.
+export type { ARPluginResult } from './stitching/ARFrameMeta';
 // NOTE: the host-worklet / frame-stream hooks `useFrameProcessor`,
 // `useThrottledFrameProcessor` and `useFrameStream` (v0.8–v0.9) were
 // archived in the batch-keyframe cleanup — they drove the third-party

package/src/stitching/ARFrameMeta.ts

@@ -104,4 +104,54 @@ export interface ARFrameMeta {
     vertexCount: number;
     faceCount: number;
   } | null;
+
+  /**
+   * v0.19.0 — SYNCHRONOUS results from host-registered AR frame plugins
+   * (the AR plugin framework).  Keyed by each plugin's `name()`; the value
+   * is the light JSON result the plugin's `process(ctx)` returned on the AR
+   * thread (`nil`/`null`-returning plugins are omitted).  Only present when
+   * the native plugin registry is non-empty AND at least one plugin returned
+   * a sync result for this frame; otherwise omitted entirely (zero-plugin
+   * apps pay nothing — native skips building the context).
+   *
+   * The SDK ships ONLY the generic framework — there are no built-in
+   * plugins.  Hosts register native plugins via `RNISARPluginRegistry`
+   * (iOS) / `RNSARPluginRegistry` (Android) at startup; each plugin is
+   * called once per AR frame while the registry is non-empty.  Result
+   * values are `unknown` because each plugin defines its own shape — cast
+   * after reading the entry you care about (e.g.
+   * `meta.plugins?.brightness as number`).
+   *
+   * ## Sync vs async results
+   *
+   * This field carries only the LIGHT, in-band SYNC results (computed fast
+   * enough to ride the throttled `onArFrame` event).  Plugins that offload
+   * heavy work to their own queue deliver results out-of-band via
+   * `registry.emit(name, result)`, which surfaces through the separate
+   * `onArPluginResult` callback (the `RNImageStitcherARPluginResult`
+   * event) — NOT here.
+   */
+  plugins?: { [name: string]: unknown };
+}
+
+
+/**
+ * v0.19.0 — an ASYNCHRONOUS result from a host-registered AR frame plugin,
+ * delivered via the `onArPluginResult` callback.
+ *
+ * Unlike the in-band SYNC results carried on {@link ARFrameMeta.plugins}
+ * (which ride the throttled `onArFrame` event), a plugin produces an async
+ * result by offloading heavy work to its own queue and later calling
+ * `registry.emit(name, result)` on the native side.  The SDK routes that to
+ * JS as a `RNImageStitcherARPluginResult` device event; `<ARCameraView>`
+ * subscribes and invokes `onArPluginResult` on the JS MAIN thread.
+ *
+ * `result` is `unknown` because each plugin defines its own result shape —
+ * branch on `plugin` (the emitting plugin's `name()`) and cast accordingly.
+ */
+export interface ARPluginResult {
+  /** The `name()` of the plugin that emitted this result. */
+  plugin: string;
+  /** The plugin-defined result payload (cast after branching on `plugin`). */
+  result: unknown;
 }