react-native-image-stitcher 0.11.0 → 0.12.0

package/src/camera/useOrientationDrift.ts

@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * useOrientationDrift — detects mid-capture device rotation.
+ *
+ * Pairs with `useDeviceOrientation()` to surface the case where the
+ * user rotates the device *during* an active capture.  The
+ * incremental stitching engine supports both portrait (Mode B,
+ * horizontal pan) and landscape (Mode A, vertical pan) capture
+ * modes as first-class — but mixing them mid-capture produces
+ * malformed output ("cross-mode capture is best-effort," per
+ * `incremental.ts:373-403`).  Hosts that want to protect against
+ * this use this hook + `OrientationDriftModal` together: the
+ * `<Camera>` flagship component auto-abandons capture the instant
+ * `drifted === true` (PR-2 wiring); the modal surfaces an
+ * explanatory popup to the user.
+ *
+ * ## API contract
+ *
+ * Pass `active` true while a capture is in flight, false otherwise.
+ * Returns:
+ *
+ *   - `captureOrientation` — the orientation snapshotted at the
+ *     moment `active` transitioned false → true.  `undefined` when
+ *     `active` is false.
+ *   - `currentOrientation` — live orientation from
+ *     `useDeviceOrientation()`.  Always defined (defaults to
+ *     `'portrait'` until the accelerometer's first sample).
+ *   - `drifted` — `true` IFF `active` is currently true AND
+ *     `currentOrientation !== captureOrientation` at some point
+ *     since the snapshot.  **Latching** — once true, stays true
+ *     until `active` flips back to false.  This is intentional:
+ *     after detection, callers should auto-abandon the capture
+ *     (engine `stop()`); allowing the flag to clear before then
+ *     would mask the drift if the user rotated back to the
+ *     original orientation between the detection tick and the
+ *     callers' abandonment effect.
+ *
+ * ## Semantics by transition
+ *
+ *   - `active` false → true:  snapshot `currentOrientation`;
+ *     reset `drifted` to false.
+ *   - `active` true (steady):  if `currentOrientation !==
+ *     captureOrientation` at any point, latch `drifted = true`.
+ *   - `active` true → false:  clear snapshot; reset `drifted`.
+ *
+ * ## Why a separate hook (rather than inlining in `<Camera>`)
+ *
+ * Hosts using the Layer-2 building blocks (`CameraView` directly,
+ * custom capture UX) can reuse this hook without mounting the
+ * full `<Camera>` flagship.  Same composition pattern as
+ * `useIMUTranslationGate` and `useKeyframeStream`.
+ *
+ * ## Testing
+ *
+ * The pure state-transition function `_computeDriftStateForTests`
+ * is exported separately so jest can exercise all 5 transition
+ * cases without booting a React render.  The hook itself is a
+ * thin wrapper around it (verified via on-device manual flow in
+ * the v0.12 verification checklist).
+ */
+
+import { useEffect, useState } from 'react';
+
+import {
+  useDeviceOrientation,
+  type DeviceOrientation,
+} from './useDeviceOrientation';
+
+
+export interface UseOrientationDriftReturn {
+  /**
+   * `true` IFF a capture is active and the device has rotated since
+   * the snapshot taken at capture start.  Latching: once true, stays
+   * true until `active` flips false.
+   */
+  drifted: boolean;
+
+  /**
+   * Snapshot of `currentOrientation` at the moment `active`
+   * transitioned false → true.  `undefined` when `active` is false.
+   */
+  captureOrientation: DeviceOrientation | undefined;
+
+  /**
+   * Live device orientation from `useDeviceOrientation()`.  Always
+   * defined.  Exposed so callers (e.g. the drift modal) can show
+   * "captured in PORTRAIT, now LANDSCAPE-LEFT" copy without
+   * mounting `useDeviceOrientation()` themselves.
+   */
+  currentOrientation: DeviceOrientation;
+}
+
+
+/**
+ * Internal state of the drift detector.  Two scalar pieces: the
+ * snapshotted capture orientation (undefined when inactive) + the
+ * latched drift flag.
+ */
+interface DriftState {
+  captureOrientation: DeviceOrientation | undefined;
+  drifted: boolean;
+}
+
+
+const INITIAL_STATE: DriftState = {
+  captureOrientation: undefined,
+  drifted: false,
+};
+
+
+/**
+ * Pure state-transition function for the drift detector.  Exported
+ * with a `_` prefix to signal "internal — not part of the public
+ * API."  Jest uses this directly so tests don't need a React
+ * renderer (the lib's jest config is pure-data / no RN preset).
+ *
+ * Given the previous state + the current `active` flag + the
+ * current device orientation, returns the new state.  Idempotent
+ * when nothing changed (returns the same object reference) so
+ * downstream `useState(setState)` calls become no-ops.
+ */
+export function _computeDriftStateForTests(
+  prev: DriftState,
+  active: boolean,
+  currentOrientation: DeviceOrientation,
+): DriftState {
+  if (!active) {
+    // active is false (or just transitioned to false).  Clear the
+    // snapshot + drift flag.  Idempotent when already cleared.
+    if (prev.captureOrientation === undefined && !prev.drifted) {
+      return prev;
+    }
+    return INITIAL_STATE;
+  }
+
+  // active is true.
+  if (prev.captureOrientation === undefined) {
+    // false → true transition.  Snapshot the current orientation.
+    // drifted starts false because, by definition, the current
+    // orientation matches itself.
+    return { captureOrientation: currentOrientation, drifted: false };
+  }
+
+  // active is steady true.  Check for drift.  Latching: once
+  // drifted is true, never set it back to false until active
+  // flips (handled above).
+  if (!prev.drifted && currentOrientation !== prev.captureOrientation) {
+    return { captureOrientation: prev.captureOrientation, drifted: true };
+  }
+
+  // No transition + no new drift.  Return prev to avoid an
+  // unnecessary state update + re-render.
+  return prev;
+}
+
+
+export function useOrientationDrift(
+  active: boolean,
+): UseOrientationDriftReturn {
+  const currentOrientation = useDeviceOrientation();
+  const [state, setState] = useState<DriftState>(INITIAL_STATE);
+
+  useEffect(() => {
+    setState((prev) => _computeDriftStateForTests(prev, active, currentOrientation));
+  }, [active, currentOrientation]);
+
+  return {
+    drifted: state.drifted,
+    captureOrientation: state.captureOrientation,
+    currentOrientation,
+  };
+}

package/src/index.ts

@@ -162,6 +162,19 @@ export { useCapture } from './camera/useCapture';
 export type { TakePhotoCallOptions } from './camera/useCapture';
 export { useVideoCapture } from './camera/useVideoCapture';
 export { useDeviceOrientation } from './camera/useDeviceOrientation';
+export type { DeviceOrientation } from './camera/useDeviceOrientation';
+
+// v0.12.0 — orientation-aware Camera (R2-lite).  `useOrientationDrift`
+// snapshots the device orientation at capture start and latches a
+// `drifted` flag if the user rotates mid-capture.  Pairs with
+// `OrientationDriftModal` for the auto-abandon UX flow.  The
+// flagship `<Camera>` component wires both internally (PR-2);
+// Layer-2 hosts using `CameraView` directly can compose the pair
+// manually (see the modal's docstring for the integration pattern).
+export { useOrientationDrift } from './camera/useOrientationDrift';
+export type { UseOrientationDriftReturn } from './camera/useOrientationDrift';
+export { OrientationDriftModal } from './camera/OrientationDriftModal';
+export type { OrientationDriftModalProps } from './camera/OrientationDriftModal';
 
 // ── Incremental stitching engine ──────────────────────────────────────
 // JS bindings around the native `IncrementalStitcher` module.  Use

package/src/stitching/__tests__/useStitcherWorklet.test.ts

@@ -0,0 +1,202 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Unit tests for `useStitcherWorklet`.
+ *
+ * Coverage focus (v0.11.1):
+ *
+ *   - **AR-source short-circuit.**  The hook's docstring promises
+ *     that AR-mode hosts can call `stitcher.call(frame)` from a
+ *     single composed worklet body without per-mode branching; AR
+ *     stitching runs natively via the AR-side dispatcher.  Pre-
+ *     v0.11.1 the code didn't enforce that — `stitcher.call` would
+ *     invoke the vc Frame Processor plugin even on AR-source
+ *     frames, which throws `getPropertyAsObject: property '__frame'
+ *     is undefined` because AR frames are `StitcherFrameHostObject`
+ *     instances and don't carry vc's JSI `Frame` proxy marker.  The
+ *     throw was caught silently by the per-worklet error handler in
+ *     `RNSARWorkletRuntime.mm`, surfacing only as an `os_log` entry
+ *     — invisible to JS, which is why composed hosts saw their
+ *     post-`stitcher.call` lines (`fireFrameProcessorLog`,
+ *     `runOnJS` callbacks) silently never execute in AR mode.  Test
+ *     2 of `docs/v0.11.0-manual-verification-checklist.md`
+ *     reproduced this on Ram's iPhone.  This test pins the fix.
+ *
+ *   - **vc-source happy path.**  vc-source frames (and frames whose
+ *     `source` is `undefined` — which is what vc's raw `Frame`
+ *     looks like; the lib doesn't wrap vc frames in Phase 4a) MUST
+ *     still invoke the plugin.
+ *
+ * ## Why mock React's hooks directly
+ *
+ * The hook owns state via `useState` (the JSI plugin handle) and
+ * side effects via `useEffect` (plugin acquisition retry loop + gyro
+ * subscription).  The existing test pattern in this directory (see
+ * `useThrottledFrameProcessor.test.ts`) doesn't use a React renderer
+ * — instead it mocks the hooks the SUT calls so the SUT can be
+ * executed as a plain function.  Same approach here: we mock
+ * `useState` to return a pre-resolved plugin, `useCallback` to
+ * return the function as-is, `useEffect` as a no-op (we don't need
+ * the plugin-acquisition retry or gyro for the call-routing test).
+ */
+
+import type { StitcherFrame } from '../StitcherFrame';
+
+// ─── Mock vision-camera ──────────────────────────────────────────
+const pluginCallSpy = jest.fn();
+const fakePlugin = { call: pluginCallSpy } as unknown as object;
+
+jest.mock('react-native-vision-camera', () => ({
+  VisionCameraProxy: {
+    initFrameProcessorPlugin: jest.fn(() => fakePlugin),
+  },
+}));
+
+// ─── Mock react-native-worklets-core ─────────────────────────────
+jest.mock('react-native-worklets-core', () => ({
+  useSharedValue: (initial: number) => ({ value: initial }),
+}));
+
+// ─── Mock react-native-sensors ───────────────────────────────────
+jest.mock('react-native-sensors', () => ({
+  gyroscope: { subscribe: jest.fn(() => ({ unsubscribe: jest.fn() })) },
+  setUpdateIntervalForType: jest.fn(),
+  SensorTypes: { gyroscope: 'gyroscope' },
+}));
+
+// ─── Mock React's hooks so the SUT runs as a plain function ──────
+//
+// `useState` returns the plugin pre-resolved.  `useCallback` returns
+// the function identity (deps array ignored — we're not testing
+// re-render semantics).  `useEffect` is a no-op (no plugin retry,
+// no gyro subscription).  This lets us call the hook synchronously
+// and exercise the worklet body via the returned `call` function.
+jest.mock('react', () => {
+  const actual = jest.requireActual('react');
+  return {
+    ...actual,
+    useState: <T,>(initial: T): [T, (next: T) => void] => {
+      // For the [plugin, setPlugin] tuple: return the fake plugin
+      // immediately rather than starting at `null`.  This skips the
+      // plugin-acquisition retry path and lets `call` actually
+      // invoke `plugin.call(...)`.
+      const resolved = (initial === null ? fakePlugin : initial) as T;
+      return [resolved, () => {}];
+    },
+    useEffect: () => {},
+    useCallback: <T,>(fn: T): T => fn,
+  };
+});
+
+// SUT — imported AFTER mocks so the hook sees them.
+// eslint-disable-next-line import/first
+import { useStitcherWorklet } from '../useStitcherWorklet';
+
+describe('useStitcherWorklet', () => {
+  beforeEach(() => {
+    pluginCallSpy.mockReset();
+  });
+
+  describe('AR-source short-circuit (v0.11.1 fix)', () => {
+    it('does NOT invoke the vc plugin for AR-source frames', () => {
+      const { call } = useStitcherWorklet();
+      const arFrame: StitcherFrame = {
+        width: 1920,
+        height: 1080,
+        pixelFormat: 'yuv',
+        orientation: 'landscape-right',
+        timestamp: 0,
+        toArrayBuffer: () => new ArrayBuffer(0),
+        source: 'ar',
+        pose: { rotation: [0, 0, 0, 1] },
+      };
+      call(arFrame);
+      expect(pluginCallSpy).not.toHaveBeenCalled();
+    });
+
+    it('does NOT invoke the vc plugin for AR-source frames even when called repeatedly', () => {
+      const { call } = useStitcherWorklet();
+      const arFrame: StitcherFrame = {
+        width: 1920,
+        height: 1080,
+        pixelFormat: 'yuv',
+        orientation: 'landscape-right',
+        timestamp: 0,
+        toArrayBuffer: () => new ArrayBuffer(0),
+        source: 'ar',
+        pose: { rotation: [0, 0, 0, 1] },
+      };
+      for (let i = 0; i < 30; i++) call(arFrame);
+      expect(pluginCallSpy).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('vc-source happy path', () => {
+    it('invokes the vc plugin for vc-source frames', () => {
+      const { call } = useStitcherWorklet();
+      const vcFrame: StitcherFrame = {
+        width: 1920,
+        height: 1080,
+        pixelFormat: 'yuv',
+        orientation: 'landscape-right',
+        timestamp: 0,
+        toArrayBuffer: () => new ArrayBuffer(0),
+        source: 'vc',
+        pose: { rotation: [0, 0, 0, 1] },
+      };
+      call(vcFrame);
+      expect(pluginCallSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it('invokes the vc plugin for frames with undefined source (raw vc Frame)', () => {
+      // vc's raw `Frame` doesn't carry the `source` field — the lib's
+      // Phase 4a deferral means we don't wrap vc frames into
+      // `StitcherFrame`.  The AR-source check must treat undefined
+      // as "not AR" to preserve the non-AR worklet path.
+      const { call } = useStitcherWorklet();
+      const rawVcFrame = {
+        width: 1920,
+        height: 1080,
+        pixelFormat: 'yuv',
+        orientation: 'landscape-right',
+        timestamp: 0,
+        toArrayBuffer: () => new ArrayBuffer(0),
+        // `source` intentionally absent
+      } as unknown as StitcherFrame;
+      call(rawVcFrame);
+      expect(pluginCallSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('plugin.call payload shape', () => {
+    it('passes the frame + a numeric-intrinsics params object', () => {
+      const { call } = useStitcherWorklet();
+      const vcFrame: StitcherFrame = {
+        width: 1920,
+        height: 1080,
+        pixelFormat: 'yuv',
+        orientation: 'landscape-right',
+        timestamp: 0,
+        toArrayBuffer: () => new ArrayBuffer(0),
+        source: 'vc',
+        pose: { rotation: [0, 0, 0, 1] },
+      };
+      call(vcFrame);
+      expect(pluginCallSpy).toHaveBeenCalledWith(
+        vcFrame,
+        expect.objectContaining({
+          tx: 0, ty: 0, tz: 0,
+          qx: expect.any(Number),
+          qy: expect.any(Number),
+          qz: expect.any(Number),
+          qw: expect.any(Number),
+          fx: expect.any(Number),
+          fy: expect.any(Number),
+          cx: 960,
+          cy: 540,
+          imageWidth: 1920,
+          imageHeight: 1080,
+        }),
+      );
+    });
+  });
+});

package/src/stitching/incremental.ts

@@ -139,9 +139,11 @@ export interface IncrementalState {
    * at the FIRST-FRAME determination thereafter.
    *
    * **This is the single source of truth for orientation across
-   * the SDK + host.**  JS-side hooks (e.g. `useDeviceOrientation`,
-   * `useWindowDimensions`) are unreliable when iOS interface-
-   * orientation lock is on; pose-derived detection is.  UI
+   * the SDK + host.**  Pose-derived detection is preferred over
+   * JS-side hooks because it works identically regardless of host
+   * configuration — `useWindowDimensions` reports JS-portrait when
+   * the host is portrait-locked (even with the device in landscape),
+   * while pose data reflects what the camera actually saw.  UI
    * components that need to know orientation (band overlay, dim
    * bars, pan guide) MUST consume `state.isLandscape` rather
    * than re-detecting.

package/src/stitching/useStitcherWorklet.ts

@@ -336,6 +336,31 @@ export function useStitcherWorklet(
     'worklet';
     if (plugin == null) return;
 
+    // v0.11.1 — AR-source frames are stitched natively by the AR-
+    // side dispatcher (`RNSARSession.swift:510-511` → the first-
+    // party callback installed in `RNSARWorkletRuntime`).  Calling
+    // the vc Frame Processor plugin here would throw
+    // `getPropertyAsObject: property '__frame' is undefined`
+    // because AR frames are `StitcherFrameHostObject` instances
+    // and don't carry the vc `Frame` proxy's JSI marker.  The
+    // throw is caught silently by the per-worklet error handler
+    // (`RNSARWorkletRuntime.mm:284-301`) and bubbles up only to
+    // `os_log` — invisible to JS, which is why pre-v0.11.1
+    // composed hosts saw their post-`stitcher.call` lines
+    // (`fireFrameProcessorLog`, `runOnJS` callbacks) silently
+    // never execute in AR mode.  Silent no-op here matches the
+    // module-header promise that AR mode is "unaffected" by this
+    // hook (the AR-side stitching path runs natively, independent
+    // of the composed worklet body).
+    //
+    // The `(frame as StitcherFrame).source` cast is safe: vc
+    // `Frame` doesn't carry a `source` property so the check
+    // returns `undefined !== 'ar'` → `true`, and the worklet
+    // proceeds normally.  Only frames that explicitly tag
+    // themselves as AR-source (which our native AR dispatcher
+    // does — see `StitcherFrameHostObject.mm`) get short-circuited.
+    if ((frame as StitcherFrame).source === 'ar') return;
+
     // Throttle (verbatim from useFrameProcessorDriver).
     sharedFrameCounter.value += 1;
     const N = sharedEvalEveryN.value;