react-native-image-stitcher 0.7.1 → 0.9.0

package/src/stitching/useFrameStream.ts

@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// v0.9.0 Layer 3 — JS-thread sampled-frame stream over Layer 1 +
+// Layer 2.
+//
+// ## What this is
+//
+// A hook that:
+//   1. Throttles a worklet via `useThrottledFrameProcessor` (Layer 2)
+//      to fire at `sampleHz` Hz.
+//   2. Inside the worklet, calls the `save_frame_as_jpeg` vc Frame
+//      Processor plugin (Layer 1) to JPEG-encode the frame to a
+//      bounded-rotation slot on disk.
+//   3. Bridges the resulting `SampledFrame` (file path + pose +
+//      dims) to a JS-thread callback via `runOnJS`.
+//
+// The host gets a per-sample callback on the JS thread with a file
+// path they can pass to `<Image>`, an OCR RN module, a cloud-upload
+// library, etc.  Zero worklet boilerplate.
+//
+// ## When to use this (vs alternatives)
+//
+//   - **`useFrameStream`** (this hook) — JS-thread consumers.  File-
+//     path OCR libraries, cloud upload, thumbnail UI, sampled
+//     server-side analysis.
+//   - **`useThrottledFrameProcessor`** (Layer 2) — worklet-native
+//     consumers.  Native OCR (Vision.framework / ML Kit) wrapped as
+//     vc plugins, TFLite ML inference, LiDAR depth processing.
+//     Lower latency; no JPEG roundtrip.
+//   - **`useFrameProcessor`** — every camera frame; full control.
+//
+// ## Slot reuse / disk usage
+//
+// JPEG files are written to `<outputDir>/stream-<N>.jpg` where N
+// cycles 0..3 based on `frame.timestamp / 1000`.  At most 4 stale
+// JPEGs ever exist on disk; the same file is rewritten on each
+// rotation, so disk usage is bounded.
+//
+// Hosts that need long-term retention (e.g., archive each sample
+// for later upload) MUST copy the file synchronously inside the
+// handler — the slot may be overwritten by the next sample.
+//
+// ## Backpressure
+//
+// If the JS handler returns slower than `1/sampleHz`, subsequent
+// ticks DO still fire (the throttle is time-based, not handler-
+// completion-based).  This means multiple handler invocations can
+// be in flight simultaneously.  For most use cases that's fine
+// (the handlers are pure or commute).  Hosts that need serialised
+// handling should track in-flight state themselves and early-return.
+//
+// ## AR vs non-AR
+//
+// Works in both modes because it composes over
+// `useThrottledFrameProcessor` → `useFrameProcessor`.  In AR mode
+// the worklet auto-registers via `__stitcherProxy` (v0.8.0 Phase
+// 4b.i/iii); in non-AR mode the returned processor object is
+// passed to `<Camera frameProcessor={...}>`.  The hook returns
+// the processor object so hosts can wire it up either way.
+
+import { useCallback, useEffect, useMemo, useRef } from 'react';
+import { Platform } from 'react-native';
+import {
+  VisionCameraProxy,
+  type Frame,
+  type FrameProcessorPlugin,
+} from 'react-native-vision-camera';
+import { Worklets } from 'react-native-worklets-core';
+
+import { useThrottledFrameProcessor } from './useThrottledFrameProcessor';
+import type { StitcherFrame } from './StitcherFrame';
+import type {
+  FrameStreamOptions,
+  SampledFrame,
+} from '../types';
+
+/**
+ * `useFrameStream` — Layer 3.  See module docstring for the full
+ * design + use-case mapping.  Quick start:
+ *
+ * ```tsx
+ * import { Camera, useFrameStream } from 'react-native-image-stitcher';
+ *
+ * function MyScreen() {
+ *   const fp = useFrameStream(
+ *     { sampleHz: 2, quality: 75 },
+ *     (sample) => {
+ *       setThumbnail(sample.jpegPath);
+ *     },
+ *   );
+ *   return <Camera frameProcessor={fp} ... />;
+ * }
+ * ```
+ *
+ * @param options  `{ sampleHz, quality?, outputDir? }`.  `sampleHz`
+ *                 clamped to `[0.5, 10]`.
+ * @param handler  JS-thread callback fired per sample.  Receives a
+ *                 `SampledFrame`.  May return a Promise; rejections
+ *                 are caught + logged (not re-thrown) so one
+ *                 misbehaving handler doesn't break the stream.
+ *
+ * @returns A `useFrameProcessor`-shaped processor object — pass to
+ *          `<Camera frameProcessor={...}>` for non-AR mode wiring.
+ *          (AR mode auto-registration via `__stitcherProxy` is
+ *          handled inside `useFrameProcessor`.)
+ */
+export function useFrameStream(
+  options: FrameStreamOptions,
+  handler: (sample: SampledFrame) => void | Promise<void>,
+): ReturnType<typeof useThrottledFrameProcessor> {
+  const sampleHz = Math.max(0.5, Math.min(10, options.sampleHz));
+  const quality = options.quality ?? 75;
+
+  // Default output dir: a per-app cache subdirectory.  Hosts that
+  // want a known path supply their own via `options.outputDir`.
+  // `Platform.OS`-specific cache paths are read once at hook mount.
+  const outputDir = useMemo(() => {
+    if (options.outputDir != null) return options.outputDir;
+    // Both platforms expose a cache directory at a predictable path
+    // via React Native APIs; we use a small inline computation to
+    // avoid pulling `react-native-fs` as a hard dep.  The lib's
+    // existing JPEG encode targets the app's data dir via similar
+    // logic in `RNSARCameraView.kt` / `IncrementalStitcher.swift`.
+    //
+    // We just generate a relative-ish path under /tmp/ for cross-
+    // platform simplicity; the native plugin writes wherever it's
+    // told to (absolute path), so as long as the directory exists
+    // the encode succeeds.  Hosts that care about file lifecycle
+    // should supply `outputDir` explicitly.
+    return Platform.OS === 'ios'
+      ? '/tmp/rnis-frame-stream'
+      : '/data/local/tmp/rnis-frame-stream';
+  }, [options.outputDir]);
+
+  // Ensure outputDir exists on the native side.  We could use
+  // react-native-fs but to keep the dep surface minimal, we just
+  // attempt to create via a tiny native call — or, simpler, accept
+  // that the plugin's write call will fail if the dir doesn't
+  // exist + log a clear error.  For v0.9.0 baseline we defer
+  // mkdir to the host (document it in the option's JSDoc) OR fall
+  // back to the platform's tmpdir which already exists.
+  //
+  // The tmpdir defaults above always exist on iOS + Android, so
+  // the common case "host doesn't supply outputDir" Just Works.
+
+  // Stable JS-side handler reference for `runOnJS`.  The hook re-
+  // captures `handler` on every render but the ref keeps the
+  // worklet closure pointing at the latest callback (avoid stale
+  // captures).
+  const handlerRef = useRef(handler);
+  handlerRef.current = handler;
+
+  const onSampleJS = useCallback((sample: SampledFrame) => {
+    const result = handlerRef.current(sample);
+    if (
+      result != null &&
+      typeof (result as Promise<void>).catch === 'function'
+    ) {
+      (result as Promise<void>).catch((err) => {
+        // eslint-disable-next-line no-console
+        console.error('[useFrameStream] handler threw:', err);
+      });
+    }
+  }, []);
+
+  const onSampleOnJS = useMemo(
+    () => Worklets.createRunOnJS(onSampleJS),
+    [onSampleJS],
+  );
+
+  // ── Plugin acquisition (Layer 1) ─────────────────────────────────
+  //
+  // `initFrameProcessorPlugin` can return `undefined` if the native
+  // registry hasn't initialised yet (rare race on app start).  We
+  // retry every 16ms (one display frame) until success — matches
+  // the pattern in `useFrameProcessorDriver`.
+  const pluginRef = useRef<FrameProcessorPlugin | null>(null);
+  useEffect(() => {
+    let cancelled = false;
+    let timerId: ReturnType<typeof setTimeout> | null = null;
+    const tryAcquire = () => {
+      if (cancelled) return;
+      const p = VisionCameraProxy.initFrameProcessorPlugin(
+        'save_frame_as_jpeg',
+        {},
+      );
+      if (p != null) {
+        pluginRef.current = p;
+        return;
+      }
+      timerId = setTimeout(tryAcquire, 16);
+    };
+    tryAcquire();
+    return () => {
+      cancelled = true;
+      if (timerId != null) clearTimeout(timerId);
+    };
+  }, []);
+
+  // The worklet body — fires at sampleHz, calls the JPEG plugin,
+  // bridges the result to JS.  Note we read `pluginRef.current`
+  // inside the worklet via the captured `plugin` value below;
+  // worklets-core handles the JS↔worklet reference.
+  const plugin = pluginRef.current;
+
+  return useThrottledFrameProcessor(
+    (frame: StitcherFrame) => {
+      'worklet';
+      if (plugin == null) return;
+
+      // Slot rotation: compute slot from frame timestamp.  At
+      // sampleHz=2 (500ms interval), the slot index changes every
+      // ~1s, giving each slot ~2 samples before being overwritten.
+      // That's overkill for the "stream-of-samples" use case but
+      // matches the docstring's "at most 4 stale JPEGs" guarantee.
+      const slot = Math.floor(frame.timestamp / 1000) % 4;
+      const path = `${outputDir}/stream-${slot}.jpg`;
+
+      // vc's `FrameProcessorPlugin.call` expects vc's `Frame` type.
+      // `StitcherFrame` is structurally a superset (it adds `source`,
+      // `pose`, AR-only fields).  Cast through `unknown` — same
+      // pattern v0.8.0's `useFrameProcessor` uses when handing a
+      // StitcherFrame-typed worklet to vc.
+      const result = plugin.call(frame as unknown as Frame, {
+        path,
+        quality,
+      });
+      if (
+        result == null ||
+        (result as { ok?: boolean }).ok !== true
+      ) {
+        // Native side reported an error (path not writable, format
+        // wrong, etc.).  Silently skip this sample — the next tick
+        // will retry.  The plugin already logs the specific reason
+        // on the native side.
+        return;
+      }
+      const r = result as {
+        path: string;
+        width: number;
+        height: number;
+      };
+
+      onSampleOnJS({
+        jpegPath: r.path,
+        pose: frame.pose,
+        timestamp: frame.timestamp,
+        width: r.width,
+        height: r.height,
+      });
+    },
+    { sampleHz },
+    [plugin, outputDir, quality, onSampleOnJS],
+  );
+}

package/src/stitching/useThrottledFrameProcessor.ts

@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// v0.9.0 Layer 2 — throttle gate over v0.8.0's `useFrameProcessor`.
+//
+// ## What this is
+//
+// A thin wrapper around `useFrameProcessor` that enforces a maximum
+// invocation rate (`sampleHz`) at the worklet layer.  The host's
+// worklet fires up to `sampleHz` times per second; ticks too close
+// together are dropped via a `useSharedValue<number>` monotonic-time
+// gate inside the worklet body.
+//
+// ## When to use this (vs alternatives)
+//
+//   - **`useFrameProcessor` directly** — every camera frame (~30-60 Hz).
+//     Use for true-realtime processing that wants to see every frame.
+//   - **`useThrottledFrameProcessor`** (this hook) — sub-frame-rate
+//     worklet-native processing.  The worklet runtime has direct
+//     access to `frame.toArrayBuffer()`, `frame.arDepth`,
+//     `frame.arAnchors`, and can call other vc Frame Processor plugins
+//     (native OCR libraries, TFLite ML inference, etc.).  Results
+//     bridged to JS via `runOnJS`.
+//   - **`useFrameStream`** (Layer 3, also in this directory) —
+//     sub-frame-rate JS-thread consumer.  The lib JPEG-encodes each
+//     sample on the producer thread and delivers a `SampledFrame`
+//     (file path + pose + dims) to a JS-thread callback.  Use for
+//     file-path OCR libraries (RN modules wrapping ML Kit etc.),
+//     cloud upload, thumbnail UI.
+//
+// ## Use-case mapping (canonical)
+//
+// | Use case                              | Layer | Why                              |
+// |---------------------------------------|-------|----------------------------------|
+// | OCR via Vision.framework / ML Kit     | **2** | native libs, bbox in frame coords|
+// | TFLite ML detection (via vc plugin)   | **2** | same shape as OCR                |
+// | LiDAR depth → 3D reconstruction       | **2** | depth too large to bridge        |
+// | Pose-only telemetry                   | **2** | tiny payload, no encoding needed |
+// | File-path OCR (RN module)             |   3   | host wants a JPEG, not pixels    |
+// | Cloud upload (sampled JPEG feed)      |   3   | JPEG IS the payload              |
+// | Live thumbnail preview UI             |   3   | `<Image source={{uri: ...}}>`    |
+//
+// See `docs/host-app-integration.md` § "Tier 2 + 3" for recipes.
+//
+// ## Threading
+//
+// The wrapped worklet fires on whatever runtime `useFrameProcessor`
+// dispatches on:
+//   - **Non-AR mode**: vision-camera's Frame Processor runtime
+//     (producer thread).
+//   - **AR mode**: the lib's `RNSARWorkletRuntime` (iOS) /
+//     worklets-core default context (Android) — fired by the AR
+//     session's per-frame dispatch.  See v0.8.0 Phase 4b.i / 4b.iii.
+//
+// Either way, the worklet MUST NOT block — the next frame's
+// processing is gated on this one returning.  Long work belongs
+// behind `runOnJS` / a separate worklet runtime.
+//
+// ## Behaviour at the throttle boundary
+//
+// The hook tracks a monotonic-time shared value of "last sample time".
+// Each tick checks if `frame.timestamp - lastSampleMs.value >=
+// (1000 / sampleHz)`.  If yes, the worklet body runs and the value
+// updates; if no, the worklet returns silently.
+//
+// Edge cases:
+//   - First-ever tick: `lastSampleMs.value` starts at 0; first frame's
+//     timestamp will be >> 0 → first tick always fires.  Subsequent
+//     ticks throttle as expected.
+//   - vc v4 timestamp semantics: per the project's worklet-throttle
+//     gotcha note, `frame.timestamp` is NOT reliably nanoseconds in
+//     vc v4.  The hook treats `frame.timestamp` as ALREADY in
+//     milliseconds (which is what vc v4 actually delivers; the
+//     v0.8.0 StitcherFrame contract documents this).  If a future
+//     vc version changes the unit, the throttle math here needs
+//     re-checking.
+
+import type { DependencyList } from 'react';
+import { useSharedValue } from 'react-native-worklets-core';
+
+import { useFrameProcessor } from './useFrameProcessor';
+import type {
+  StitcherFrame,
+  StitcherFrameProcessor,
+} from './StitcherFrame';
+import type { ThrottledFrameProcessorOptions } from '../types';
+
+/**
+ * Throttled variant of `useFrameProcessor`.  See the module
+ * docstring for the full use-case mapping; quick version:
+ *
+ * ```tsx
+ * const fp = useThrottledFrameProcessor(
+ *   (frame) => {
+ *     'worklet';
+ *     // worklet-native OCR / ML / depth processing here
+ *   },
+ *   { sampleHz: 2 },
+ *   [],
+ * );
+ * return <Camera frameProcessor={fp} ... />;
+ * ```
+ *
+ * @param worklet  Host's frame-processor worklet.  Must be
+ *                 `'worklet'`-prefixed.  Runs at most `sampleHz`
+ *                 times per second.
+ * @param options  `{ sampleHz }` — clamped to `[0.5, 30]`.
+ * @param deps     Standard React deps array.  Treated the same as
+ *                 `useFrameProcessor`'s deps — when they change the
+ *                 inner worklet is re-bound.
+ *
+ * @returns A `useFrameProcessor`-shaped processor object, pass it
+ *          to `<Camera frameProcessor={...}>`.
+ */
+export function useThrottledFrameProcessor(
+  worklet: StitcherFrameProcessor,
+  options: ThrottledFrameProcessorOptions,
+  deps: DependencyList,
+): ReturnType<typeof useFrameProcessor> {
+  // Clamp + derive interval.  Done outside the worklet so the
+  // useSharedValue / useFrameProcessor hooks see stable values.
+  const sampleHz = Math.max(0.5, Math.min(30, options.sampleHz));
+  const minIntervalMs = 1000 / sampleHz;
+
+  // Monotonic-time gate.  Initialised to 0 → first tick always
+  // fires (frame.timestamp >> 0).
+  const lastSampleMs = useSharedValue(0);
+
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  return useFrameProcessor(
+    (frame: StitcherFrame) => {
+      'worklet';
+      const now = frame.timestamp;
+      if (now - lastSampleMs.value < minIntervalMs) {
+        return;
+      }
+      lastSampleMs.value = now;
+      worklet(frame);
+    },
+    // The throttle interval is captured in the worklet closure; if
+    // it changes we need to re-bind the worklet so the new
+    // `minIntervalMs` takes effect.  Same for the host's worklet
+    // identity (so deps changes on the host side re-bind too).
+    [minIntervalMs, worklet, ...deps],
+  );
+}

package/src/types.ts

@@ -56,6 +56,101 @@ export interface DeviceMetadata {
 // `Camera.tsx` adapts this into the public `CameraCaptureResult` (a
 // discriminated union of photo + panorama) before emitting `onCapture`.
 
+/**
+ * v0.9.0 Layer 3 — one sampled frame delivered by `useFrameStream`
+ * to the JS-thread handler.
+ *
+ * The JPEG file at `jpegPath` is the stream's own copy.  Hosts that
+ * need long-term retention MUST copy the file synchronously inside
+ * the handler — the same path may be overwritten by a subsequent
+ * sample (slot reuse — see the hook's docstring for the rotation
+ * policy).
+ */
+export interface SampledFrame {
+  /** Absolute filesystem path to the JPEG.  No `file://` prefix. */
+  jpegPath: string;
+
+  /**
+   * Pose at sample time.  `translation` is `undefined` in non-AR
+   * mode (gyro provides rotation only; no spatial anchor).
+   */
+  pose: {
+    rotation: [number, number, number, number];
+    translation?: [number, number, number];
+  };
+
+  /** Frame timestamp (ms; per the v0.8.0 StitcherFrame contract). */
+  timestamp: number;
+
+  /** JPEG width / height in pixels. */
+  width: number;
+  height: number;
+}
+
+/**
+ * v0.9.0 Layer 3 — options for `useFrameStream`.
+ *
+ * For worklet-native processing without JPEG roundtrip (OCR via
+ * Vision/ML Kit, TFLite ML, LiDAR depth), use
+ * `useThrottledFrameProcessor` (Layer 2) instead.
+ */
+export interface FrameStreamOptions {
+  /**
+   * Target sampling rate in Hertz.  Clamped to `[0.5, 10]`.  The
+   * Layer 2 throttle gate enforces the rate inside the worklet;
+   * ticks too close together are dropped silently.
+   *
+   * Clamp upper bound (10 Hz) is intentionally lower than Layer 2's
+   * (30 Hz) — beyond 10 Hz the per-frame JPEG encode + JS-bridge
+   * cost dominates the wall-clock budget.  Hosts that need higher
+   * rates should be on Layer 2 with their own JPEG encoder call
+   * (or no JPEG at all).
+   */
+  sampleHz: number;
+
+  /**
+   * JPEG quality (0-100).  Default 75.  Clamped silently to
+   * `[1, 100]` by the underlying `save_frame_as_jpeg` native plugin.
+   */
+  quality?: number;
+
+  /**
+   * Directory to write JPEG files into.  Defaults to a per-app
+   * `<cache>/rnis-frame-stream/` subdirectory.  The directory is
+   * `mkdir -p`'d on first use; hosts that supply an existing
+   * absolute path are responsible for its lifecycle.
+   */
+  outputDir?: string;
+}
+
+/**
+ * v0.9.0 Layer 2 — options for `useThrottledFrameProcessor`.
+ *
+ * Wraps v0.8.0's `useFrameProcessor` with a monotonic-time throttle
+ * gate so the supplied worklet fires at most `sampleHz` times per
+ * second.  Use for sub-frame-rate worklet-native processing — native
+ * OCR (Vision.framework / ML Kit), TFLite ML detection, LiDAR depth
+ * processing — where the bbox / depth payloads are small enough to
+ * bridge to JS via `runOnJS`.
+ *
+ * For JS-thread JPEG consumers (file-path OCR libraries, cloud
+ * upload, thumbnail UI), use `useFrameStream` (Layer 3) instead.
+ */
+export interface ThrottledFrameProcessorOptions {
+  /**
+   * Target sampling rate in Hertz.  Clamped to `[0.5, 30]`.  Inside
+   * the worklet a monotonic-time gate enforces the rate; ticks too
+   * close together are silently dropped.
+   *
+   * The clamp upper bound (30 Hz) sits at typical AR rates on
+   * mid-range Android devices — beyond that, the host should just
+   * use `useFrameProcessor` directly (no throttle).  The clamp
+   * lower bound (0.5 Hz) prevents accidentally-zero-divide values
+   * + matches `useFrameStream`'s convention.
+   */
+  sampleHz: number;
+}
+
 export interface CaptureResult {
   /** Unique device-generated UUID */
   deviceUuid: string;