react-native-image-stitcher 0.7.1 → 0.9.0

package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts

@@ -0,0 +1,178 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Unit tests for the v0.9.0 Layer 2 `useThrottledFrameProcessor` hook.
+ *
+ * The worklet runtime can't run in jest (no JSI, no worklets-core).
+ * What we CAN test:
+ *
+ *   - The `sampleHz` clamping (`[0.5, 30]`)
+ *   - `minIntervalMs` math (1000 / sampleHz)
+ *   - The deps propagation (host's deps → useFrameProcessor's deps)
+ *   - The throttle gate logic (extracted as a pure function for
+ *     isolated verification — see `_throttleGateForTests`).
+ *
+ * The hook itself is tested via a thin React-renderer-free harness:
+ * we mock `useFrameProcessor` + `useSharedValue` so we can verify
+ * the call shape without booting the worklet runtime.
+ */
+
+import { useThrottledFrameProcessor } from '../useThrottledFrameProcessor';
+
+// ─── Mock vision-camera + worklets-core ─────────────────────────────
+// These are minimal-shim mocks — enough surface for the hook to call
+// `useFrameProcessor(workletBody, deps)` and `useSharedValue(0)`.
+
+const useFrameProcessorMock = jest.fn();
+const useSharedValueMock = jest.fn();
+
+jest.mock('../useFrameProcessor', () => ({
+  useFrameProcessor: (...args: unknown[]) => useFrameProcessorMock(...args),
+}));
+
+jest.mock('react-native-worklets-core', () => ({
+  useSharedValue: (initial: number) => useSharedValueMock(initial),
+}));
+
+describe('useThrottledFrameProcessor', () => {
+  beforeEach(() => {
+    useFrameProcessorMock.mockReset();
+    useSharedValueMock.mockReset();
+    // Default behaviour for useSharedValue: return an object with a
+    // mutable `.value` field (mirrors worklets-core's API).
+    useSharedValueMock.mockImplementation((initial: number) => ({
+      value: initial,
+    }));
+  });
+
+  describe('sampleHz clamping', () => {
+    it('clamps below 0.5 to 0.5', () => {
+      const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
+      useThrottledFrameProcessor(noop, { sampleHz: 0.1 }, []);
+      // useFrameProcessor receives the wrapped worklet; the deps
+      // array's first entry is `minIntervalMs`.  For sampleHz=0.5,
+      // minIntervalMs = 2000.
+      const [, deps] = useFrameProcessorMock.mock.calls[0]!;
+      expect(deps[0]).toBeCloseTo(2000);
+    });
+
+    it('clamps above 30 to 30', () => {
+      const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
+      useThrottledFrameProcessor(noop, { sampleHz: 999 }, []);
+      const [, deps] = useFrameProcessorMock.mock.calls[0]!;
+      // sampleHz=30 → minIntervalMs = 33.333...
+      expect(deps[0]).toBeCloseTo(1000 / 30);
+    });
+
+    it('passes through in-range sampleHz unchanged', () => {
+      const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
+      useThrottledFrameProcessor(noop, { sampleHz: 2 }, []);
+      const [, deps] = useFrameProcessorMock.mock.calls[0]!;
+      expect(deps[0]).toBeCloseTo(500);
+    });
+
+    it('accepts boundary values exactly', () => {
+      const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
+      useThrottledFrameProcessor(noop, { sampleHz: 0.5 }, []);
+      let deps = useFrameProcessorMock.mock.calls[0]![1];
+      expect(deps[0]).toBeCloseTo(2000);
+
+      useFrameProcessorMock.mockClear();
+      useThrottledFrameProcessor(noop, { sampleHz: 30 }, []);
+      deps = useFrameProcessorMock.mock.calls[0]![1];
+      expect(deps[0]).toBeCloseTo(1000 / 30);
+    });
+  });
+
+  describe('deps propagation', () => {
+    it('appends host deps after the internal interval + worklet deps', () => {
+      const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
+      const hostDep1 = { id: 'a' };
+      const hostDep2 = 42;
+      useThrottledFrameProcessor(noop, { sampleHz: 2 }, [hostDep1, hostDep2]);
+      const [, deps] = useFrameProcessorMock.mock.calls[0]!;
+      // Expected shape: [minIntervalMs, worklet, ...hostDeps]
+      expect(deps).toHaveLength(4);
+      expect(deps[0]).toBeCloseTo(500);
+      expect(deps[1]).toBe(noop);
+      expect(deps[2]).toBe(hostDep1);
+      expect(deps[3]).toBe(hostDep2);
+    });
+
+    it('with empty host deps: deps = [minIntervalMs, worklet]', () => {
+      const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
+      useThrottledFrameProcessor(noop, { sampleHz: 2 }, []);
+      const [, deps] = useFrameProcessorMock.mock.calls[0]!;
+      expect(deps).toHaveLength(2);
+      expect(deps[0]).toBeCloseTo(500);
+      expect(deps[1]).toBe(noop);
+    });
+  });
+
+  describe('throttle gate', () => {
+    // The throttle logic lives INSIDE the wrapped worklet body, which
+    // jest can't execute directly (it's a `'worklet'`-prefixed
+    // function).  But the wrapped function IS just a plain JS
+    // function until the worklets-core babel plugin transforms it,
+    // so we can call it manually with mock frames + a mock
+    // shared-value gate.
+    //
+    // The body's logic:
+    //   if (frame.timestamp - lastSampleMs.value < minIntervalMs) return;
+    //   lastSampleMs.value = frame.timestamp;
+    //   worklet(frame);
+
+    it('fires the worklet on the first frame regardless of timestamp', () => {
+      const hostWorklet = jest.fn();
+      useThrottledFrameProcessor(hostWorklet, { sampleHz: 2 }, []);
+      const [wrappedBody] = useFrameProcessorMock.mock.calls[0]!;
+
+      const frame = { timestamp: 12345 } as Parameters<typeof hostWorklet>[0];
+      wrappedBody(frame);
+
+      expect(hostWorklet).toHaveBeenCalledTimes(1);
+      expect(hostWorklet).toHaveBeenCalledWith(frame);
+    });
+
+    it('skips a frame too close to the previous sample', () => {
+      const hostWorklet = jest.fn();
+      useThrottledFrameProcessor(hostWorklet, { sampleHz: 2 }, []); // 500ms interval
+      const [wrappedBody] = useFrameProcessorMock.mock.calls[0]!;
+
+      wrappedBody({ timestamp: 1000 } as never);
+      wrappedBody({ timestamp: 1100 } as never); // 100ms after — too soon
+      wrappedBody({ timestamp: 1200 } as never); // 200ms after — too soon
+
+      expect(hostWorklet).toHaveBeenCalledTimes(1);
+    });
+
+    it('fires again exactly at the interval boundary', () => {
+      const hostWorklet = jest.fn();
+      useThrottledFrameProcessor(hostWorklet, { sampleHz: 2 }, []); // 500ms
+      const [wrappedBody] = useFrameProcessorMock.mock.calls[0]!;
+
+      wrappedBody({ timestamp: 1000 } as never);
+      wrappedBody({ timestamp: 1500 } as never); // exactly at boundary
+
+      expect(hostWorklet).toHaveBeenCalledTimes(2);
+    });
+
+    it('fires again past the interval boundary', () => {
+      const hostWorklet = jest.fn();
+      useThrottledFrameProcessor(hostWorklet, { sampleHz: 2 }, []); // 500ms
+      const [wrappedBody] = useFrameProcessorMock.mock.calls[0]!;
+
+      wrappedBody({ timestamp: 1000 } as never);
+      wrappedBody({ timestamp: 1600 } as never); // 600ms after
+
+      expect(hostWorklet).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('shared value lifecycle', () => {
+    it('initializes lastSampleMs to 0', () => {
+      const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
+      useThrottledFrameProcessor(noop, { sampleHz: 2 }, []);
+      expect(useSharedValueMock).toHaveBeenCalledWith(0);
+    });
+  });
+});

package/src/stitching/ensureStitcherProxyInstalled.ts

@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { NativeModules } from 'react-native';
+
+/**
+ * v0.8.0 Phase 4b — one-shot installer that asks the native side
+ * to install `globalThis.__stitcherProxy` on the main JS runtime.
+ *
+ * ## When this runs
+ *
+ * The first call to `useFrameProcessor` triggers this.  Idempotent:
+ * once the global is installed, subsequent calls short-circuit.
+ *
+ * ## What it does
+ *
+ * Calls into the platform-native `StitcherJsiInstaller` RN module
+ * which is registered with a `RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(install)`
+ * on iOS (see `ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm`)
+ * and — Phase 4b.ii — an analogous Kotlin TurboModule on Android.
+ *
+ * The native module reaches into the main JS runtime via
+ * `RCTCxxBridge.runtime` (iOS) / the equivalent Android JSI access
+ * pattern and installs a host object on `globalThis.__stitcherProxy`
+ * exposing `install(workletFn)` / `uninstall(id)` / `count()`.
+ *
+ * ## Failure modes (and what happens then)
+ *
+ * 1. **Module not registered** (Android in Phase 4b.i; old iOS
+ *    builds without the new pod files).  `NativeModules
+ *    .StitcherJsiInstaller` is `undefined`.  This function returns
+ *    `false` and the hook falls back to the JS-side
+ *    `StitcherWorkletRegistry` — host worklets are registered
+ *    on the JS side but never fan out to AR mode.  No crash, no
+ *    regression vs. Phase 4a.
+ *
+ * 2. **JSI runtime unreachable** (e.g., remote debug mode).  The
+ *    sync method returns `false`.  Same JS-side-registry fallback.
+ *
+ * 3. **Native install succeeds but global not yet visible.**
+ *    The native call is SYNCHRONOUS (`BLOCKING_SYNCHRONOUS_METHOD`),
+ *    so by the time the function returns the global is installed.
+ *    No race here.
+ *
+ * ## Why a separate module
+ *
+ * The install method is a one-time runtime bootstrap, not a
+ * per-call API.  Putting it on its own RN module (vs. on the
+ * existing `StitcherBridge` / `IncrementalStitcherBridge`) keeps
+ * the responsibility surface narrow and the failure mode easy
+ * to diagnose ("`__stitcherProxy` not installed" → check
+ * `StitcherJsiInstaller` module registration first).
+ */
+
+interface StitcherJsiInstallerModule {
+  install(): boolean;
+}
+
+/**
+ * `__DEV__` is RN's global dev-flag.  Guard the read with `typeof`
+ * so the helper works in any environment that imports it without
+ * defining __DEV__ (jest, SSR, custom tooling).  Same pattern RN's
+ * own debug code uses.
+ */
+function isDev(): boolean {
+  return typeof __DEV__ !== 'undefined' && __DEV__;
+}
+
+let installed = false;
+
+export function ensureStitcherProxyInstalled(): boolean {
+  if (installed) return true;
+  // Already installed by an earlier hook mount.  Cheap fast-path.
+  if (typeof (globalThis as { __stitcherProxy?: unknown }).__stitcherProxy !== 'undefined') {
+    installed = true;
+    return true;
+  }
+
+  const mod = (NativeModules as { StitcherJsiInstaller?: StitcherJsiInstallerModule })
+    .StitcherJsiInstaller;
+  if (mod == null || typeof mod.install !== 'function') {
+    // Module not present — Android until Phase 4b.ii lands, or
+    // an old iOS build.  Surface this once at debug-info level so
+    // the host can see "your worklets are JS-registered only" in
+    // logcat / Console.app without a noisy per-frame warning.
+    if (isDev() && !warnedAboutMissingModule) {
+      warnedAboutMissingModule = true;
+      console.info(
+        '[react-native-image-stitcher] StitcherJsiInstaller native ' +
+          'module not found; host worklets registered in JS-side ' +
+          'registry only.  AR-mode dispatch requires the native install ' +
+          '(iOS Phase 4b.i — included in v0.8.0; Android Phase 4b.ii ' +
+          '— follow-up release).',
+      );
+    }
+    return false;
+  }
+
+  try {
+    const ok = mod.install();
+    if (!ok) {
+      // Native module ran but couldn't install (JSI runtime
+      // unreachable).  Same fallback as the missing-module case.
+      if (isDev() && !warnedAboutFailedInstall) {
+        warnedAboutFailedInstall = true;
+        console.info(
+          '[react-native-image-stitcher] StitcherJsiInstaller.install() ' +
+            'returned false (JSI runtime unreachable — remote debug ' +
+            'mode?).  Falling back to JS-side host worklet registry.',
+        );
+      }
+      return false;
+    }
+    installed = true;
+    return true;
+  } catch (err) {
+    if (isDev() && !warnedAboutFailedInstall) {
+      warnedAboutFailedInstall = true;
+      console.info(
+        '[react-native-image-stitcher] StitcherJsiInstaller.install() ' +
+          'threw: ' +
+          String(err) +
+          '.  Falling back to JS-side host worklet registry.',
+      );
+    }
+    return false;
+  }
+}
+
+let warnedAboutMissingModule = false;
+let warnedAboutFailedInstall = false;
+
+/**
+ * Test-only — reset module-internal state.  Used by jest to allow
+ * multiple test cases to re-trigger the install path independently.
+ * NOT exported from `src/index.ts`.
+ */
+export function _resetStitcherProxyInstallStateForTests(): void {
+  installed = false;
+  warnedAboutMissingModule = false;
+  warnedAboutFailedInstall = false;
+}

package/src/stitching/useFrameProcessor.ts

@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { useEffect, type DependencyList } from 'react';
+import {
+  useFrameProcessor as visionCameraUseFrameProcessor,
+  type DrawableFrameProcessor,
+  type Frame,
+  type ReadonlyFrameProcessor,
+} from 'react-native-vision-camera';
+
+import { ensureStitcherProxyInstalled } from './ensureStitcherProxyInstalled';
+import { StitcherWorkletRegistry } from './StitcherWorkletRegistry';
+import type { StitcherFrameProcessor } from './StitcherFrame';
+
+/**
+ * Shape of the native-installed `globalThis.__stitcherProxy` host
+ * object (iOS Phase 4b.i; Android Phase 4b.ii).  When present, the
+ * hook prefers the native registry over the JS-side mirror — the
+ * native AR worklet runtime reads from the native side directly.
+ */
+interface StitcherProxy {
+  install(workletFn: StitcherFrameProcessor): string;
+  uninstall(id: string): void;
+  count(): number;
+}
+
+/**
+ * v0.8.0 Phase 4a — public hook for hosts to attach a per-frame
+ * worklet that runs in BOTH AR and non-AR capture modes.
+ *
+ * ## Quick start
+ *
+ * ```tsx
+ * import { useFrameProcessor, type StitcherFrame } from 'react-native-image-stitcher';
+ *
+ * function MyOcrOverlay() {
+ *   const processor = useFrameProcessor((frame: StitcherFrame) => {
+ *     'worklet';
+ *     // Pixel data is in `frame.toArrayBuffer()`.
+ *     // AR-only fields: `frame.arDepth`, `frame.arAnchors`, `frame.arTrackingState`.
+ *     // Discriminate via `frame.source === 'ar'` / `'vc'`.
+ *   }, []);
+ *   return <Camera frameProcessor={processor} ... />;
+ * }
+ * ```
+ *
+ * ## Two behaviours, depending on mode
+ *
+ * **Non-AR mode (today, fully working):** the worklet runs on
+ * vision-camera's Frame Processor runtime.  Same thread + same
+ * cost envelope as a plain `useFrameProcessor` from
+ * `react-native-vision-camera`.  The lib's own first-party
+ * stitching plugin runs alongside on the same producer-thread
+ * runtime (composition is handled by vision-camera's own dispatch
+ * order).
+ *
+ * Your worklet receives whatever vision-camera delivers — vc's raw
+ * `Frame`.  This is a structural subset of `StitcherFrame`: the
+ * vc-shaped fields (`width`, `height`, `pixelFormat`, `orientation`,
+ * `timestamp`, `toArrayBuffer`) are guaranteed; the
+ * `StitcherFrame`-only fields (`source`, `pose`, `arDepth`,
+ * `arAnchors`, `arTrackingState`) are **undefined** at runtime
+ * because the lib does NOT wrap or augment vc's `Frame` in Phase 4a
+ * (cross-worklet-boundary field injection is Phase 4b work).
+ * Worklets that need to read `source` / `pose` MUST guard for
+ * `undefined`:
+ *
+ * ```ts
+ * if (frame.source === 'ar') { ... }   // false in non-AR mode
+ * if (frame.pose) { ... }              // skipped in non-AR mode
+ * ```
+ *
+ * **AR mode — iOS Phase 4b.i (this release):** the worklet is
+ * installed into the native registry via
+ * `globalThis.__stitcherProxy.install(workletFn)`, where
+ * `__stitcherProxy` is a JSI host object installed at lib
+ * bootstrap by the native `StitcherJsiInstaller` module.  The
+ * AR worklet runtime (`RNSARWorkletRuntime`) reads from the
+ * native registry on each `dispatchFrame:pose:` call and fans
+ * out invocations — your worklet fires alongside the lib's
+ * first-party stitching path.
+ *
+ * **AR mode — Android Phase 4b.ii (deferred):** the native
+ * installer + JNI bridge from `StitcherWorkletRuntime.kt`'s
+ * `runFirstParty {...}` path to a parallel C++ registry land in
+ * a follow-up release.  Until then, on Android the hook falls
+ * back to the JS-side `StitcherWorkletRegistry`; AR-mode host
+ * worklets register but do not invoke.  No regression vs.
+ * Phase 4a; iOS gets the API first.
+ *
+ * ### When Phase 4b.ii lands (Android)
+ *
+ * The hook's call signature does NOT change.  Android hosts that
+ * write code today against this API will see their worklets
+ * start firing in AR mode automatically when Phase 4b.ii is
+ * merged.  No migration required.
+ *
+ * ## Frame contract
+ *
+ * The worklet receives a {@link StitcherFrame} (see
+ * `src/stitching/StitcherFrame.ts` for the full contract +
+ * lifecycle).  Highlights:
+ *
+ *   - **`source`** discriminator: `'vc'` or `'ar'`.  Branch on this
+ *     before reading `arDepth` / `arAnchors` / `arTrackingState`
+ *     so non-AR captures don't break.
+ *   - **`pose`** always present.  `pose.translation` is `undefined`
+ *     in non-AR mode (gyro provides only rotation; no spatial
+ *     anchor).
+ *   - **Buffer lifetime**: pixel data is valid only for the
+ *     duration of the worklet call.  Worklets that need to retain
+ *     data must `toArrayBuffer()` synchronously inside the
+ *     worklet body — returning a reference and reading it later
+ *     reads freed memory.
+ *
+ * ## Threading
+ *
+ * The worklet runs on the producer thread (vision-camera's
+ * runtime in non-AR mode; the AR-session callback thread under
+ * Phase 4b).  Worklets MUST NOT block the producer thread for
+ * more than a few ms — the next frame's processing is gated on
+ * the previous frame returning.  Long work belongs on a queue
+ * crossed via Reanimated / worklets-core's `runOnJS`.
+ *
+ * @param worklet  The host's frame processor function.  Must be
+ *                 `'worklet'`-prefixed at the call site.  TS
+ *                 cannot enforce the prefix; the runtime will
+ *                 throw at attempt to invoke a non-worklet
+ *                 function.
+ * @param deps     Standard React deps array.  When `deps` change,
+ *                 the previous registration is removed and the
+ *                 new worklet is registered.  Same semantics as
+ *                 vision-camera's `useFrameProcessor`.
+ *
+ * @returns A vision-camera frame-processor object that
+ *          `<Camera frameProcessor={...}>` accepts.  In non-AR
+ *          mode this is what drives the per-frame worklet
+ *          invocation; in AR mode it's currently a no-op (vc
+ *          isn't mounted in AR mode anyway).
+ */
+export function useFrameProcessor(
+  worklet: StitcherFrameProcessor,
+  deps: DependencyList,
+): ReadonlyFrameProcessor | DrawableFrameProcessor {
+  // Non-AR path: delegate to vision-camera's hook.  The returned
+  // processor object is what `<Camera>` hands to vc.  Worklet
+  // fires on vc's producer-thread runtime.
+  //
+  // Cast rationale: vc's hook expects `(frame: Frame) => void`.
+  // Our worklet is typed `(frame: StitcherFrame) => void`.
+  // `StitcherFrame` is a structural superset of `Frame` (it adds
+  // required `source` + `pose` and the optional AR fields), so
+  // assigning a function that consumes `StitcherFrame` to a
+  // `Frame`-consuming slot is unsound at the type level — TS is
+  // right to reject the direct assignment.  At RUNTIME the worklet
+  // will see vc's raw `Frame`; the `source` / `pose` / AR fields
+  // are undefined (the hook's docstring above documents this and
+  // tells hosts to guard).  We double-cast through `unknown` to
+  // suppress, accepting the explicit type-system gap as the price
+  // of Phase 4a's pre-Phase-4b deferral on cross-runtime frame
+  // wrapping.
+  const vcProcessor = visionCameraUseFrameProcessor(
+    worklet as unknown as (frame: Frame) => void,
+    deps,
+  );
+
+  // AR path: install into the native registry if available (iOS
+  // Phase 4b.i — and Android Phase 4b.ii once it lands).  Falls
+  // back to the JS-side `StitcherWorkletRegistry` when the native
+  // installer isn't present (Android in 4b.i; remote debug mode;
+  // unit tests).  The fallback path matches Phase 4a's
+  // register-but-not-invoke semantics.
+  //
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  useEffect(() => {
+    const nativeReady = ensureStitcherProxyInstalled();
+    if (
+      nativeReady &&
+      typeof (globalThis as { __stitcherProxy?: StitcherProxy }).__stitcherProxy !== 'undefined'
+    ) {
+      // Native path — install through the JSI proxy.  Errors here
+      // most commonly mean the worklet doesn't have the
+      // `'worklet'` directive at the call site (the worklets-core
+      // babel plugin didn't transform it).  Surface them via the
+      // proxy's own throw with a host-side log so the failure is
+      // obvious.
+      let id: string | undefined;
+      try {
+        id = (globalThis as unknown as { __stitcherProxy: StitcherProxy }).__stitcherProxy.install(
+          worklet,
+        );
+      } catch (err) {
+        // Guard `__DEV__` read so the hook works in any environment
+        // that imports it without defining the flag (jest, SSR,
+        // custom tooling).
+        if (typeof __DEV__ !== 'undefined' && __DEV__) {
+          console.error(
+            '[react-native-image-stitcher] __stitcherProxy.install ' +
+              'threw — is the worklet function decorated with ' +
+              "`'worklet';` and processed by react-native-worklets-core's " +
+              'babel plugin?  Original error: ' +
+              String(err),
+          );
+        }
+        return; // No cleanup needed — nothing was installed.
+      }
+      return () => {
+        try {
+          (globalThis as unknown as { __stitcherProxy: StitcherProxy }).__stitcherProxy.uninstall(id!);
+        } catch {
+          // Uninstall is best-effort; an exception here means the
+          // proxy was already gone (e.g., app reload mid-cleanup).
+        }
+      };
+    }
+
+    // Fallback — JS-side registry.  Same as Phase 4a.
+    const jsId = StitcherWorkletRegistry.register({
+      worklet,
+      isFirstParty: false,
+    });
+    return () => StitcherWorkletRegistry.unregister(jsId);
+  }, deps);
+
+  return vcProcessor;
+}