react-native-image-stitcher 0.19.0 → 0.20.1

package/src/camera/Camera.tsx

@@ -41,8 +41,10 @@
  */
 
 import React, {
+  forwardRef,
   useCallback,
   useEffect,
+  useImperativeHandle,
   useMemo,
   useRef,
   useState,
@@ -68,6 +70,8 @@ import type {
 import { useARSession } from '../ar/useARSession';
 import type { CameraFrameProcessor } from '../stitching/CameraFrame';
 import type { ARFrameMeta, ARPluginResult } from '../stitching/ARFrameMeta';
+import type { AROverlay } from '../stitching/AROverlay';
+import type { AROverlayMethods } from './arOverlayController';
 import { ARCameraView, type ARCameraViewHandle } from './ARCameraView';
 import { CameraShutter } from './CameraShutter';
 import { CameraView } from './CameraView';
@@ -834,6 +838,26 @@ export interface CameraProps {
    */
   onArPluginResult?: (e: ARPluginResult) => void;
 
+  /**
+   * v0.20.0 — AR OVERLAY / ANNOTATION renderer.  A declarative array of 2D
+   * shapes drawn ON TOP of the AR camera preview, each anchored to WORLD
+   * positions and REPROJECTED to screen on every AR frame from the current
+   * camera pose + intrinsics (smooth display-rate tracking, no 3D engine).
+   * Only meaningful in AR capture (`captureSource === 'ar'`); `<Camera>`
+   * threads this straight through to the underlying `<ARCameraView>`.
+   *
+   * State-driven: pass a React-state array and update it as your world points
+   * change (e.g. from {@link CameraProps.onArFrame} plane anchors).  The set is
+   * diffed against the current overlays BY `id`.  For zero-render-latency
+   * mutations use the imperative ref methods on the `<Camera>` handle instead
+   * ({@link CameraHandle}: `setOverlays` / `addOverlay` / `updateOverlay` /
+   * `removeOverlay` / `clearOverlays`) — both paths funnel through the same
+   * native channel.  JS-set overlays merge on the native side with overlays a
+   * registered AR plugin placed directly (namespaced so neither clobbers the
+   * other).  See {@link AROverlay} for the shape.
+   */
+  overlays?: AROverlay[];
+
   // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────────
   /**
    * Which device holds the non-AR panorama capture accepts.
@@ -922,6 +946,26 @@ export interface CameraProps {
 }
 
 
+/**
+ * v0.20.0 — imperative handle exposed via the `<Camera>` ref.
+ *
+ * Currently scoped to the AR-overlay methods ({@link AROverlayMethods}:
+ * `setOverlays` / `addOverlay` / `updateOverlay` / `removeOverlay` /
+ * `clearOverlays`), which forward to the underlying `<ARCameraView>`'s overlay
+ * channel when AR mode is mounted.  They are no-ops while the camera is in
+ * non-AR mode (no `<ARCameraView>` is mounted, and overlays only render over
+ * the AR preview) — use the declarative {@link CameraProps.overlays} prop for
+ * a set that survives AR↔non-AR transitions, since it re-applies automatically
+ * whenever `<ARCameraView>` (re)mounts.
+ *
+ * The shape is identical to {@link ARCameraViewHandle}'s overlay subset so a
+ * host can use either component with the same overlay code.  Photo / panorama
+ * capture remain driven by the built-in shutter (no imperative capture methods
+ * on this handle — see the component docstring's scope note).
+ */
+export interface CameraHandle extends AROverlayMethods {}
+
+
 // ─── Sub-components ─────────────────────────────────────────────────
 
 /**
@@ -1207,8 +1251,15 @@ function extractPanoramaOverrides(props: CameraProps): PanoramaPropOverrides {
 
 /**
  * The public `<Camera>` component.
+ *
+ * v0.20.0 — now a `forwardRef`.  The ref exposes {@link CameraHandle} (the AR
+ * overlay methods); existing callers that don't pass a ref are unaffected
+ * (`forwardRef` makes the ref optional).
  */
-export function Camera(props: CameraProps): React.JSX.Element {
+export const Camera = forwardRef<CameraHandle, CameraProps>(function Camera(
+  props: CameraProps,
+  ref,
+): React.JSX.Element {
   const {
     defaultCaptureSource = 'non-ar',
     defaultLens = '1x',
@@ -1248,6 +1299,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
     onArFrame,
     arFrameMetaInterval,
     onArPluginResult,
+    overlays,
     engine = 'batch-keyframe',
     // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
     panMode = 'vertical',
@@ -1511,6 +1563,22 @@ export function Camera(props: CameraProps): React.JSX.Element {
   const visionCameraRef = useRef<VisionCamera | null>(null);
   const arViewRef = useRef<ARCameraViewHandle | null>(null);
 
+  // v0.20.0 — AR overlay imperative handle.  `<Camera>` itself renders no
+  // overlay layer; the overlay methods forward to the mounted
+  // `<ARCameraView>`'s handle (which owns the controller + native dispatch).
+  // No-op when AR mode isn't mounted (`arViewRef.current === null`), matching
+  // the CameraHandle docstring — the declarative `overlays` prop is the path
+  // that survives AR↔non-AR transitions.  The `overlays` prop is also threaded
+  // straight to `<ARCameraView>` below, so a host can use either API.
+  useImperativeHandle(ref, (): CameraHandle => ({
+    setOverlays: (o) => arViewRef.current?.setOverlays(o),
+    addOverlay: (o) => arViewRef.current?.addOverlay(o),
+    updateOverlay: (id, patch) => arViewRef.current?.updateOverlay(id, patch),
+    removeOverlay: (id) => arViewRef.current?.removeOverlay(id),
+    clearOverlays: () => arViewRef.current?.clearOverlays(),
+    raycast: () => arViewRef.current?.raycast() ?? Promise.resolve(null),
+  }), []);
+
   // Effect that does the async transition work whenever the settled
   // refs disagree with the current isAR/lens.  Order matters:
   //   1. Set the cameraTransitioning state so the gate stays closed
@@ -2519,6 +2587,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
           onArFrame={onArFrame}
           arFrameMetaInterval={arFrameMetaInterval}
           onArPluginResult={onArPluginResult}
+          overlays={overlays}
         />
       ) : (
         <CameraView
@@ -3015,7 +3084,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
       />
     </View>
   );
-}
+});
 
 
 function noop(): void {

package/src/camera/arOverlayController.ts

@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * v0.20.0 — shared JS→native plumbing for the AR overlay renderer.
+ *
+ * `<ARCameraView>` and `<Camera>` expose an IDENTICAL imperative overlay API
+ * (`setOverlays` / `addOverlay` / `updateOverlay` / `removeOverlay` /
+ * `clearOverlays`) plus a declarative `overlays` prop.  Rather than duplicate
+ * the diff + native-dispatch logic in each component, both build their handle
+ * from {@link createAROverlayController} — DRY, single source of truth for the
+ * wire format and the merge-by-id semantics.
+ *
+ * ## Native mechanism (agreed cross-platform contract)
+ *
+ * Every AR-session setting in this SDK already flows through the
+ * `RNSARSession` native-module singleton (`setPlaneDetection`,
+ * `setArFrameMetaEnabled`, `setSceneReconstructionEnabled`) because
+ * `RNSARSession.shared` drives the single mounted `RNSARCameraView`.  Overlays
+ * follow the same pattern: a single `setOverlays(overlays)` method on
+ * `RNSARSession` carries the FULL current JS-set overlay array each time it
+ * changes.  Native replaces its JS-set overlay collection wholesale (the merge
+ * with the namespaced native-plugin set happens on the native side) and the
+ * overlay layer redraws every AR frame.
+ *
+ * The declarative `overlays` prop and the imperative methods both ultimately
+ * call this same `setOverlays` with the resolved array, so the two APIs are
+ * interchangeable and can't diverge.
+ *
+ * Why a module method (not a UIManager view command)?  It matches every other
+ * AR setting in this codebase and there is only ever ONE `RNSARCameraView`
+ * mounted (ARKit/ARCore can't share the camera), so there's nothing to key by
+ * view tag.  The equivalent UIManager view-command name, for native sides that
+ * prefer per-view dispatch, is documented as `RNSARCameraViewOverlays`.
+ */
+
+import { NativeModules } from 'react-native';
+
+import type { AROverlay } from '../stitching/AROverlay';
+
+/**
+ * The imperative overlay methods exposed on both `<ARCameraView>` and
+ * `<Camera>` refs.  Identical shape on both so a host can swap components
+ * without rewriting overlay code.
+ */
+export interface AROverlayMethods {
+  /** Replace the entire JS-set overlay collection. */
+  setOverlays: (overlays: AROverlay[]) => void;
+  /** Add one overlay (replaces any existing overlay with the same `id`). */
+  addOverlay: (overlay: AROverlay) => void;
+  /**
+   * Shallow-merge a patch into the overlay with `id`.  No-op if no overlay
+   * with that `id` is currently set.
+   */
+  updateOverlay: (id: string, patch: Partial<AROverlay>) => void;
+  /** Remove the overlay with `id` (no-op if absent). */
+  removeOverlay: (id: string) => void;
+  /** Remove all JS-set overlays. */
+  clearOverlays: () => void;
+  /**
+   * Raycast from the screen centre (the crosshair) to the first real-world
+   * surface and resolve its world position `[x, y, z]` in metres (ARKit/ARCore
+   * world frame), or `null` when nothing is hit (e.g. a featureless wall before
+   * any plane is detected).  Use it to place an overlay ON the aimed surface at
+   * the real distance — pass the result as a `worldPosition` to
+   * {@link setOverlays} / {@link addOverlay} — instead of guessing a distance.
+   * Resolves `null` (never throws) when the native module / method is absent.
+   */
+  raycast: () => Promise<[number, number, number] | null>;
+}
+
+interface RNSARSessionOverlayModule {
+  // On iOS the native `setOverlays` is a Promise method (resolver/rejecter
+  // RN-injected); on Android it's `void`.  We call it fire-and-forget but
+  // type it as possibly-thenable so the defensive `.catch` below compiles.
+  setOverlays?: (overlays: AROverlay[]) => void | Promise<unknown>;
+  // Raycast resolves `{ worldPosition: [x,y,z] }` on a hit, or `null`.
+  raycast?: () => Promise<{ worldPosition?: number[] } | null>;
+}
+
+/** The `RNSARSession` native-module method name overlays dispatch through. */
+export const AR_OVERLAY_SET_METHOD = 'setOverlays' as const;
+
+/**
+ * The agreed UIManager view-command name for native sides that drive overlays
+ * via per-view command dispatch instead of the module method.  The JS layer
+ * dispatches through the module method (there's only one AR view), but the
+ * name is pinned here so the native side can match if it chooses commands.
+ */
+export const AR_OVERLAY_VIEW_COMMAND = 'RNSARCameraViewOverlays' as const;
+
+/**
+ * Build an overlay controller backed by an in-memory ordered set keyed by
+ * `id`.  Every mutating call resolves the new full array and pushes it to
+ * native via `RNSARSession.setOverlays`.  The controller is the single source
+ * of truth for BOTH the imperative ref methods and the declarative `overlays`
+ * prop (the prop's effect calls `setOverlays` with the prop value).
+ */
+export function createAROverlayController(): AROverlayMethods & {
+  /** Current JS-set overlays in insertion order (used by tests / diffing). */
+  getOverlays: () => AROverlay[];
+} {
+  // Insertion-ordered map: preserves the order overlays were added so the
+  // native render order is stable + predictable.
+  const overlaysById = new Map<string, AROverlay>();
+
+  const flush = (): void => {
+    const native = (NativeModules as Record<string, unknown>)
+      .RNSARSession as RNSARSessionOverlayModule | undefined;
+    // Native module / method unavailable (web, or a native build predating the
+    // overlay channel): no-op, no crash — mirrors the other AR setters.
+    const ret = native?.setOverlays?.(Array.from(overlaysById.values()));
+    // iOS returns a Promise (the native method is Promise-typed); swallow any
+    // rejection so a transient native error never surfaces as an unhandled
+    // rejection.  Android returns void — the optional chain skips the catch.
+    (ret as Promise<unknown> | undefined)?.catch?.(() => undefined);
+  };
+
+  return {
+    getOverlays: () => Array.from(overlaysById.values()),
+
+    setOverlays: (overlays: AROverlay[]) => {
+      overlaysById.clear();
+      for (const o of overlays) {
+        // Last-writer-wins on duplicate ids in the incoming array.
+        overlaysById.set(o.id, o);
+      }
+      flush();
+    },
+
+    addOverlay: (overlay: AROverlay) => {
+      // Re-set to move an existing id to the end? No — preserve original slot
+      // by deleting first only when absent.  Map#set keeps the existing slot
+      // when the key already exists, so a plain set is the right "replace in
+      // place" behaviour.
+      overlaysById.set(overlay.id, overlay);
+      flush();
+    },
+
+    updateOverlay: (id: string, patch: Partial<AROverlay>) => {
+      const existing = overlaysById.get(id);
+      if (existing == null) {
+        return;
+      }
+      // Shallow-merge; `id` is preserved from the existing overlay regardless
+      // of what the patch carries (the map key must stay consistent).
+      overlaysById.set(id, { ...existing, ...patch, id });
+      flush();
+    },
+
+    removeOverlay: (id: string) => {
+      if (overlaysById.delete(id)) {
+        flush();
+      }
+    },
+
+    clearOverlays: () => {
+      if (overlaysById.size > 0) {
+        overlaysById.clear();
+        flush();
+      }
+    },
+
+    raycast: async (): Promise<[number, number, number] | null> => {
+      const native = (NativeModules as Record<string, unknown>)
+        .RNSARSession as RNSARSessionOverlayModule | undefined;
+      const fn = native?.raycast;
+      // Native module / method unavailable (web, or a native build predating
+      // the raycast channel): resolve null — the caller falls back.
+      if (typeof fn !== 'function') {
+        return null;
+      }
+      try {
+        const res = await fn();
+        const wp = res?.worldPosition;
+        if (Array.isArray(wp) && wp.length >= 3) {
+          return [Number(wp[0]), Number(wp[1]), Number(wp[2])];
+        }
+        return null;
+      } catch {
+        return null;
+      }
+    },
+  };
+}

package/src/index.ts

@@ -27,6 +27,7 @@
 export { Camera, CameraError } from './camera/Camera';
 export type {
   CameraProps,
+  CameraHandle,
   CameraCaptureResult,
   PanoramaCaptureResult,
   CameraErrorCode,
@@ -272,6 +273,20 @@ export type { ARFrameMeta } from './stitching/ARFrameMeta';
 // `onArPluginResult` callback (a plugin's out-of-band `registry.emit(...)`
 // result).  The SDK ships only the generic framework — no built-in plugins.
 export type { ARPluginResult } from './stitching/ARFrameMeta';
+// v0.20.0 — AR OVERLAY / ANNOTATION renderer data model.  A 2D shape anchored
+// to a world point (or world quad) and reprojected to screen every AR frame.
+// Drive it via the declarative `overlays` prop or the imperative ref methods
+// (`setOverlays` / `addOverlay` / `updateOverlay` / `removeOverlay` /
+// `clearOverlays`) on both `<Camera>` and `<ARCameraView>`.
+export type { AROverlay } from './stitching/AROverlay';
+// The shared imperative-overlay method signatures (the `<Camera>` /
+// `<ARCameraView>` ref handles extend this).  Plus the agreed native channel
+// names, for hosts / native plugins matching the wire contract.
+export type { AROverlayMethods } from './camera/arOverlayController';
+export {
+  AR_OVERLAY_SET_METHOD,
+  AR_OVERLAY_VIEW_COMMAND,
+} from './camera/arOverlayController';
 // NOTE: the host-worklet / frame-stream hooks `useFrameProcessor`,
 // `useThrottledFrameProcessor` and `useFrameStream` (v0.8–v0.9) were
 // archived in the batch-keyframe cleanup — they drove the third-party

package/src/stitching/AROverlay.ts

@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * v0.20.0 — the AR OVERLAY / ANNOTATION renderer's data model.
+ *
+ * An {@link AROverlay} describes a 2D shape (a billboard marker/box or a
+ * world-anchored quad) that the native overlay layer draws ON TOP of the AR
+ * camera preview (`RNSARCameraView`).  Each overlay is anchored to WORLD
+ * positions and REPROJECTED to screen on EVERY AR frame from the current
+ * camera pose + intrinsics — so it tracks the scene at display rate with no
+ * 3D-engine dependency.
+ *
+ * ## Two ways to anchor an overlay
+ *
+ *   1. **A single world point** (`worldPosition`) — drawn as a billboard
+ *      marker/box facing the camera, sized by `sizeMeters` (default a small
+ *      marker).  Use this for a pin on a detected plane anchor, a label on a
+ *      point of interest, etc.
+ *   2. **Explicit world corners** (`worldQuad`, 3–4 points) — drawn as the
+ *      outline/box connecting the projected corners.  Use this for a detected
+ *      quad (a shelf face, a packet, a door) whose real-world shape you
+ *      already know.
+ *
+ * Provide ONE of the two anchor forms.  If both are present `worldQuad` wins
+ * (it's the more specific description); the native renderers read `worldQuad`
+ * first and fall back to `worldPosition` + `sizeMeters`.
+ *
+ * ## Rendering / reprojection (native)
+ *
+ * The native side reprojects each overlay's world point(s) to screen with the
+ * AR framework's BUILT-IN, correct projection — iOS
+ * `ARFrame.camera.projectPoint(_:orientation:viewportSize:)`, Android
+ * `viewMatrix · projectionMatrix` → clip → NDC → screen.  Points behind the
+ * camera or off-screen are hidden.  The layer redraws every frame so the
+ * outline/box + label stay pinned to the world as the camera moves.
+ *
+ * ## Where overlays come from
+ *
+ * Overlays reach the native renderer through two INDEPENDENT, merged sets:
+ *
+ *   - **JS-set** — the declarative `overlays` prop or the imperative ref
+ *     methods (`setOverlays` / `addOverlay` / `updateOverlay` / `removeOverlay`
+ *     / `clearOverlays`) on `<Camera>` and `<ARCameraView>`.
+ *   - **Native-plugin-set** — a registered AR plugin places overlays directly
+ *     via the 0.19 registry (`RNISARPluginRegistry.setOverlays(...)` on iOS /
+ *     `RNSARPluginRegistry.setOverlays(...)` on Android), with zero JS latency.
+ *
+ * The native renderer draws the UNION of both sets; the plugin set is
+ * namespaced so a JS `setOverlays(...)` never clobbers plugin overlays.
+ */
+export interface AROverlay {
+  /**
+   * Stable identifier.  The declarative `overlays` prop diffs the incoming
+   * array against the current set BY `id` (add / update / remove); the
+   * imperative `updateOverlay` / `removeOverlay` methods key off it too.  Must
+   * be unique within a set.
+   */
+  id: string;
+
+  /**
+   * Anchor form 1 — a single world point in METRES (world space `[x, y, z]`).
+   * Drawn as a billboard marker/box of `sizeMeters` extent facing the camera.
+   * Ignored when `worldQuad` is provided.
+   */
+  worldPosition?: [number, number, number];
+
+  /**
+   * Box extent in METRES `[width, height]` at `worldPosition`.  Only meaningful
+   * with `worldPosition`.  Defaults to a small marker on the native side when
+   * omitted.
+   */
+  sizeMeters?: [number, number];
+
+  /**
+   * Anchor form 2 — 3 or 4 explicit world corners in METRES (e.g. a detected
+   * quad).  Each corner is `[x, y, z]` in world space.  Drawn as the
+   * outline/box connecting the projected corners.  Takes precedence over
+   * `worldPosition` + `sizeMeters` when both are present.
+   */
+  worldQuad?: Array<[number, number, number]>;
+
+  /**
+   * Draw style.  Default `'outline'` (stroked edges).  `'box'` is a filled /
+   * boxed marker.  Both render in 2D this release.
+   */
+  shape?: 'box' | 'outline';
+
+  /** Optional text label drawn at the overlay's anchor point. */
+  label?: string;
+
+  /**
+   * Stroke / fill colour as a hex string (e.g. `'#00E5FF'`).  Defaults to a
+   * theme colour on the native side when omitted.
+   */
+  color?: string;
+
+  /**
+   * Render mode.  Default `'2d'` — a flat shape reprojected to screen.
+   * `'3d'` is SCAFFOLD ONLY this release: the data-model field exists and the
+   * native renderers leave a marked hook for a future SceneKit (iOS) / Android
+   * 3D renderer, but v1 treats `'3d'` as `'2d'` (with a one-time native log
+   * warning).  Document-only forward compatibility.
+   */
+  mode?: '2d' | '3d';
+}