react-native-image-stitcher 0.1.0

package/src/stitching/stitchFrames.ts

@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * stitchFrames — video / frame stitching API.
+ *
+ * Implementation status (Phase 2 of #8):
+ *   - iOS: Swift native module that vendors upstream OpenCV's iOS
+ *     framework and calls `cv::Stitcher::SCANS` mode (designed for
+ *     translational shelf captures).  Lives in
+ *     `retailens-capture-sdk/ios/Sources/RNImageStitcher/`.
+ *   - Android: deferred to Phase 3 — same OpenCV surface, different
+ *     build (NDK + Gradle).  Until that lands, Android calls hit the
+ *     `StitchNotImplementedError` path below.
+ *
+ * Why fail loudly instead of falling back to JS?
+ *   The cloud-sync pipeline depends on a stitched panorama being
+ *   present.  Silently producing a broken or single-frame "panorama"
+ *   would corrupt downstream SOS computation.  Hard-failing here lets
+ *   the host app surface the unsupported-platform error to the user
+ *   immediately rather than discovering it on the server hours later.
+ */
+
+import { NativeModules, Platform } from 'react-native';
+
+
+export interface StitchFramesOptions {
+  /**
+   * Absolute paths to the input image files in capture order.
+   * Must share a camera + focal length (we don't blend across sources).
+   */
+  framePaths: string[];
+  /**
+   * Output path for the stitched image (JPEG).  Host app chooses the
+   * location (tmp vs. cache vs. Documents).
+   */
+  outputPath: string;
+  /** JPEG quality [0-100].  Default 85. */
+  quality?: number;
+}
+
+
+export interface StitchFramesResult {
+  /** Absolute path to the stitched image on disk. */
+  outputPath: string;
+  /** Pixel dimensions of the stitched image. */
+  width: number;
+  height: number;
+  /** Wall-clock ms the stitcher took (host apps log this for perf tracking). */
+  durationMs: number;
+}
+
+
+/**
+ * Stitch ``framePaths`` into a single panoramic image.
+ *
+ * Throws ``StitchNotImplementedError`` until the native module ships.
+ * Callers should catch that specific error and fall back gracefully
+ * (e.g. surface a "single-frame mode only" banner in the UI).
+ */
+export async function stitchFrames(
+  options: StitchFramesOptions,
+): Promise<StitchFramesResult> {
+  // Look for the native module by its canonical name so we can flip
+  // this function to "implemented" simply by registering the module
+  // in AppDelegate / MainApplication.
+  const native: unknown =
+    (NativeModules as Record<string, unknown>)['BatchStitcher'];
+  if (native && typeof native === 'object' && 'stitch' in (native as object)) {
+    const fn = (native as { stitch: (o: StitchFramesOptions) => Promise<StitchFramesResult> }).stitch;
+    return fn(options);
+  }
+
+  throw new StitchNotImplementedError(
+    `stitchFrames is not yet implemented on ${Platform.OS}. `
+    + 'The react-native-image-stitcher native stitcher module is expected '
+    + 'but not registered — the JS shim is throwing by design so the '
+    + 'host app can fall back to single-frame mode rather than ship '
+    + 'broken panoramas.',
+  );
+}
+
+
+export class StitchNotImplementedError extends Error {
+  public readonly code = 'STITCH_NOT_IMPLEMENTED';
+  constructor(message: string) {
+    super(message);
+    this.name = 'StitchNotImplementedError';
+  }
+}

package/src/stitching/stitchVideo.ts

@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * stitchVideo — end-to-end "video → panorama" API.
+ *
+ * The host app records video while the user holds the shutter; on
+ * release we hand the video file path to this function and it
+ * returns a single stitched panoramic JPEG.  Internally:
+ *
+ *   1. Native side extracts N evenly-spaced frames from the video
+ *      (AVAssetImageGenerator on iOS, MediaMetadataRetriever on
+ *      Android once Phase 3 lands).
+ *   2. Native side feeds the frames into `cv::Stitcher::SCANS`.
+ *   3. Frames are deleted; only the final panorama remains.
+ *
+ * The single bridge call keeps the JS thread out of the
+ * extract→stitch dance entirely; nothing has to round-trip frame
+ * paths back to JS just to hand them to a sibling native call.
+ */
+
+import { NativeModules, Platform } from 'react-native';
+
+import type { StitchFramesResult } from './stitchFrames';
+import { StitchNotImplementedError } from './stitchFrames';
+
+
+export interface StitchVideoOptions {
+  /**
+   * Absolute path to the recorded video file.  Accepts paths with
+   * or without the `file://` prefix — the native module strips it.
+   */
+  videoPath: string;
+  /**
+   * Where the resulting panoramic JPEG should be written.
+   */
+  outputPath: string;
+  /**
+   * Number of frames to sample from the video.  Default 10 — the
+   * empirical sweet spot for shelf scans on iPhone hardware: enough
+   * overlap that homography stays robust, few enough that stitching
+   * stays under 4 seconds.  Increase only if the user pans further
+   * than ~1 m of shelf in a single hold.
+   */
+  maxFrames?: number;
+  /**
+   * JPEG quality [0..100].  Applied to BOTH the intermediate frames
+   * (extracted from video) AND the final panorama.  Default 85.
+   */
+  quality?: number;
+  /**
+   * Projection used during the warp step.  See native side for
+   * details.  Default `'plane'` — straight verticals/horizontals,
+   * good for shelf scans + close-up subjects.  Hourglass shape on
+   * partial arcs is handled by the rectangular-crop step.
+   *   - `'plane'`: flat projection.  Best for ≤30° pans.
+   *   - `'cylindrical'`: handles rotational mid-arc pans.
+   *   - `'spherical'`: full 90°+ panoramic captures.
+   */
+  warperType?: 'plane' | 'cylindrical' | 'spherical';
+  /**
+   * Blending strategy across stitched seams.
+   *   - `'multiband'` (default): high-quality seams when alignment
+   *     is good; can produce visible halos when adjacent frames
+   *     have inconsistent exposure.
+   *   - `'feather'`: simple alpha-weighted blend; faster + no
+   *     halo artifacts; slightly more visible seam edge.
+   */
+  blenderType?: 'multiband' | 'feather';
+  /**
+   * Seam finding strategy.
+   *   - `'graphcut'` (default): cv::detail::GraphCutSeamFinder
+   *     finds optimal cuts before blending — pairs beautifully
+   *     with `multiband`, but holds all warped frames in memory
+   *     simultaneously (higher peak).
+   *   - `'skip'`: streams warp+feed in a single pass, never
+   *     holding more than one warped frame.  Lower peak memory.
+   *     Right choice on low-RAM devices or with `feather`.
+   */
+  seamFinderType?: 'graphcut' | 'skip';
+  /**
+   * Phase 5: pose-driven stitching.  When present and non-empty,
+   * the native stitcher skips features → matching → BundleAdjuster
+   * and builds cv::detail::CameraParams directly from each pose's
+   * intrinsics + quaternion.  Each entry has the shape returned
+   * by `NativeModules.RNSARSession.snapshotPoseLog()`:
+   *
+   *   { tx, ty, tz, qx, qy, qz, qw,
+   *     fx, fy, cx, cy,
+   *     imageWidth, imageHeight,
+   *     timestampMs, trackingState }
+   *
+   * Frames whose closest pose is beyond a 100 ms tolerance are
+   * dropped before stitching; if fewer than 2 remain the call
+   * rejects with `opencv-failed-1032` so the host can fall back
+   * to the feature-matched path (re-call `stitchVideo` without
+   * `poses`).
+   */
+  poses?: Array<{
+    tx: number; ty: number; tz: number;
+    qx: number; qy: number; qz: number; qw: number;
+    fx: number; fy: number; cx: number; cy: number;
+    imageWidth: number; imageHeight: number;
+    timestampMs: number;
+    trackingState: number;
+  }>;
+}
+
+
+/**
+ * Stitch a recorded video file into a single panoramic JPEG.
+ *
+ * Throws `StitchNotImplementedError` on platforms where the native
+ * stitcher hasn't shipped yet (Android until Phase 3).  Throws an
+ * Error with a code-like message on stitcher failures the JS layer
+ * may want to recover from (see `StitcherError` cases on the native
+ * side: `insufficient-frames`, `read-failed`, `write-failed`,
+ * `opencv-failed-<code>`).
+ */
+export async function stitchVideo(
+  options: StitchVideoOptions,
+): Promise<StitchFramesResult> {
+  const native: unknown =
+    (NativeModules as Record<string, unknown>)['BatchStitcher'];
+  if (
+    native
+    && typeof native === 'object'
+    && typeof (native as { stitchVideo?: unknown }).stitchVideo === 'function'
+  ) {
+    const fn = (native as {
+      stitchVideo: (o: StitchVideoOptions) => Promise<StitchFramesResult>;
+    }).stitchVideo;
+    // 60-second hard deadline.  cv::Stitcher::stitch can occasionally
+    // grind for minutes inside bundle adjustment on hard inputs (low
+    // texture, extreme parallax).  A timeout converts those into a
+    // clear UI error rather than letting the stitching banner sit
+    // forever.  Real stitches finish in 2-8 seconds on iPhone with
+    // the tuned PANORAMA settings; 60 s is comfortably above that.
+    return Promise.race([
+      fn(options),
+      new Promise<StitchFramesResult>((_, reject) => {
+        setTimeout(
+          () => reject(new Error('stitch-timeout: stitching took longer than 30 s')),
+          30_000,
+        );
+      }),
+    ]);
+  }
+
+  throw new StitchNotImplementedError(
+    `stitchVideo is not yet implemented on ${Platform.OS}. `
+    + 'The react-native-image-stitcher native stitcher module is expected '
+    + 'but not registered (or the build predates Phase 2.5 of #8).',
+  );
+}

package/src/stitching/useIncrementalJSDriver.ts

@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * useIncrementalJSDriver — vision-camera + gyro frame driver for
+ * the incremental panorama engine, used in non-AR captures on both
+ * iOS and Android.
+ *
+ * History: previously called `useIncrementalAndroidDriver` because
+ * it was Android-only.  As of 2026-05-17 (Issue #2), the native
+ * `processFrameAtPath` entry point exists on both platforms and the
+ * hook drives non-AR on iOS too; renamed 2026-05-19 to reflect
+ * that.
+ *
+ * Why this exists
+ *   In AR captures the engine consumes frames from the ARSession
+ *   stream natively (60 Hz pose + image delivery, zero JS
+ *   involvement once started).  In NON-AR captures there is no AR
+ *   session — vision-camera owns the camera — so the engine needs
+ *   another frame source.  This hook fills the gap:
+ *
+ *     - vision-camera keeps the camera viewport
+ *     - `takeSnapshot()` runs at ~250 ms intervals during press-hold
+ *     - `react-native-sensors` gyroscope is integrated to estimate
+ *       cumulative yaw/pitch (drives the FoV-overlap gate)
+ *     - Each snapshot path + integrated pose is fed to
+ *       `IncrementalStitcher.processFrameAtPath()`
+ *
+ * Trade-off vs the AR path
+ *   Gyro integration drifts ~1–2° per minute.  Acceptable for the
+ *   typical 5–15 s shelf pan; not great for ambitious 360° captures.
+ *   Snapshot rate is ~4 Hz (vs 60 Hz in AR mode).  Pose drives
+ *   frame-selection only — the actual image alignment is feature-
+ *   matched + RANSAC-fit, so quality of the panorama itself isn't
+ *   bounded by gyro accuracy.
+ *
+ * Lifecycle
+ *   `start({ cameraRef })` enables the loop; `stop()` tears down.
+ *   Both should be called by the host's hold-start / hold-complete
+ *   handlers.  Safe to call on either platform; the hook only
+ *   activates inside the start/stop block.
+ */
+
+import { useCallback, useRef } from 'react';
+import { NativeModules } from 'react-native';
+import {
+  gyroscope,
+  setUpdateIntervalForType,
+  SensorTypes,
+} from 'react-native-sensors';
+import type { Subscription } from 'rxjs';
+import type { Camera } from 'react-native-vision-camera';
+
+
+export interface UseIncrementalJSDriverOptions {
+  /**
+   * Snapshot interval in ms.  Default 250 (≈ 4 Hz).  Lower = more
+   * candidate frames + more disk I/O.  Don't go below 200 — vision-
+   * camera's snapshot pipeline can't keep up reliably below that.
+   */
+  snapshotIntervalMs?: number;
+  /**
+   * Gyro sample rate in ms (~30 Hz default matches the existing
+   * `PanoramaGuidance` cadence).  Used for pose integration only —
+   * not the snapshot rate.
+   */
+  gyroIntervalMs?: number;
+  /**
+   * Approximate horizontal FoV of the device camera.  Drives the
+   * overlap-percent calculation in the native engine.  Default 65°
+   * is a reasonable mid-tier smartphone average.
+   */
+  fovHorizDegrees?: number;
+  /**
+   * Approximate vertical FoV of the device camera.  Default 50° for
+   * typical 4:3 phone cameras.  When ARCore-driven path is in use
+   * the engine receives both FoVs straight from intrinsics; the
+   * gyro driver is a fallback so the defaults are good enough.
+   */
+  fovVertDegrees?: number;
+}
+
+
+export interface IncrementalJSDriverHandle {
+  start: (cameraRef: React.RefObject<Camera | null>) => void;
+  stop: () => void;
+  isRunning: boolean;
+}
+
+
+interface NativeProcessFrame {
+  processFrameAtPath(options: {
+    path: string;
+    yaw: number;
+    pitch: number;
+    fovHorizDegrees: number;
+    fovVertDegrees: number;
+    trackingPoor: boolean;
+    /** Quaternion (x, y, z, w) — pose-driven path. */
+    qx: number; qy: number; qz: number; qw: number;
+    /** Sensor-resolution intrinsics. */
+    fx: number; fy: number; cx: number; cy: number;
+    imageWidth: number;
+    imageHeight: number;
+  }): Promise<unknown>;
+}
+
+
+function getNativeIncremental(): NativeProcessFrame | null {
+  const m = (NativeModules as Record<string, unknown>)['IncrementalStitcher'];
+  if (!m || typeof m !== 'object') return null;
+  return m as NativeProcessFrame;
+}
+
+
+export function useIncrementalJSDriver(
+  options: UseIncrementalJSDriverOptions = {},
+): IncrementalJSDriverHandle {
+  const {
+    snapshotIntervalMs = 250,
+    gyroIntervalMs = 33,
+    fovHorizDegrees = 65,
+    fovVertDegrees = 50,
+  } = options;
+
+  const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const gyroSubRef = useRef<Subscription | null>(null);
+  const cameraRef = useRef<React.RefObject<Camera | null> | null>(null);
+  // Integrated pose accumulators, in radians.  Reset on each
+  // start() call.  Y-axis = horizontal pan (yaw), X-axis = vertical
+  // pan (pitch).  Sign convention matches ARKit: counter-clockwise
+  // from above is positive yaw.
+  const yawRef = useRef(0);
+  const pitchRef = useRef(0);
+  const lastGyroAtRef = useRef<number | null>(null);
+  // Single in-flight guard so we don't pile up overlapping snapshot
+  // promises on slow devices — if last snapshot hasn't finished
+  // when the next interval fires, skip.
+  const snapshotInFlightRef = useRef(false);
+  // Module-level "is the driver active right now" — exposed to the
+  // host because the hook itself doesn't trigger re-renders.
+  const isRunningRef = useRef(false);
+
+  const stop = useCallback(() => {
+    if (intervalRef.current) {
+      clearInterval(intervalRef.current);
+      intervalRef.current = null;
+    }
+    if (gyroSubRef.current) {
+      gyroSubRef.current.unsubscribe();
+      gyroSubRef.current = null;
+    }
+    cameraRef.current = null;
+    isRunningRef.current = false;
+  }, []);
+
+  const start = useCallback(
+    (cameraRefArg: React.RefObject<Camera | null>) => {
+      // 2026-05-17 (Issue #2) — removed the Android-only platform
+      // guard.  iOS now also exposes `processFrameAtPath` (see the
+      // Swift bridge), so the same driver feeds both platforms in
+      // non-AR mode.
+      if (isRunningRef.current) return;
+      const native = getNativeIncremental();
+      if (!native) return;
+
+      cameraRef.current = cameraRefArg;
+      yawRef.current = 0;
+      pitchRef.current = 0;
+      lastGyroAtRef.current = null;
+      snapshotInFlightRef.current = false;
+      isRunningRef.current = true;
+
+      // Gyro integration.  Each sample carries angular velocity in
+      // rad/s; multiply by elapsed time to accumulate angular
+      // displacement.  Note: the gyro axes are device-local; we use
+      // y for yaw and x for pitch on a device held in portrait.
+      // Landscape would swap, but the FoV-overlap gate is dominant-
+      // axis based on the .mm side, so the convention matters less
+      // than consistency.
+      setUpdateIntervalForType(SensorTypes.gyroscope, gyroIntervalMs);
+      gyroSubRef.current = gyroscope.subscribe({
+        next: ({ x, y }) => {
+          const now = Date.now();
+          if (lastGyroAtRef.current === null) {
+            lastGyroAtRef.current = now;
+            return;
+          }
+          const dt = (now - lastGyroAtRef.current) / 1000.0;
+          lastGyroAtRef.current = now;
+          yawRef.current += y * dt;
+          pitchRef.current += x * dt;
+        },
+        error: (err) => {
+          // eslint-disable-next-line no-console
+          console.warn('[useIncrementalJSDriver] gyro error', err);
+        },
+      });
+
+      // Snapshot loop.
+      const tick = async () => {
+        if (snapshotInFlightRef.current) return;
+        const cam = cameraRef.current?.current;
+        if (!cam) return;
+        snapshotInFlightRef.current = true;
+        try {
+          const snap = await cam.takeSnapshot({ quality: 70 });
+          if (!snap?.path) return;
+          // Synthesise a quaternion from integrated yaw + pitch.
+          // Yaw rotates about world Y (gravity), pitch about world X
+          // (perpendicular to gravity in the device's frame).
+          // Combined as q = q_yaw · q_pitch.
+          const halfYaw = yawRef.current / 2;
+          const halfPitch = pitchRef.current / 2;
+          const cy_ = Math.cos(halfYaw);
+          const sy_ = Math.sin(halfYaw);
+          const cp = Math.cos(halfPitch);
+          const sp = Math.sin(halfPitch);
+          // q_yaw   = (0, sy, 0, cy)
+          // q_pitch = (sp, 0, 0, cp)
+          // q = q_yaw * q_pitch:
+          const qx = cy_ * sp;
+          const qy = sy_ * cp;
+          const qz = -sy_ * sp;
+          const qw = cy_ * cp;
+
+          // Vision-camera v4 doesn't expose camera intrinsics on
+          // Android, so we estimate fx/fy from the snapshot's pixel
+          // dimensions + assumed FoV.  cx/cy at image centre.  This
+          // is approximate; the proper Android live path is the
+          // ARCameraView, where ARCore gives us the real intrinsics.
+          const w = snap.width ?? 1920;
+          const h = snap.height ?? 1440;
+          const fx = w / (2.0 * Math.tan(((fovHorizDegrees * Math.PI) / 180) / 2));
+          const fy = h / (2.0 * Math.tan(((fovVertDegrees * Math.PI) / 180) / 2));
+          const cx = w / 2;
+          const cy = h / 2;
+
+          await native.processFrameAtPath({
+            path: snap.path,
+            yaw: yawRef.current,
+            pitch: pitchRef.current,
+            fovHorizDegrees,
+            fovVertDegrees,
+            trackingPoor: false,
+            qx, qy, qz, qw,
+            fx, fy, cx, cy,
+            imageWidth: w, imageHeight: h,
+          });
+        } catch (err) {
+          // Swallow per-frame errors so the loop keeps running.
+          // eslint-disable-next-line no-console
+          console.warn(
+            '[useIncrementalJSDriver] processFrame failed', err,
+          );
+        } finally {
+          snapshotInFlightRef.current = false;
+        }
+      };
+      // Kick off an immediate first frame so the engine doesn't sit
+      // idle for the first interval period.
+      tick();
+      intervalRef.current = setInterval(tick, snapshotIntervalMs);
+    },
+    [snapshotIntervalMs, gyroIntervalMs, fovHorizDegrees, fovVertDegrees],
+  );
+
+  return {
+    start,
+    stop,
+    get isRunning(): boolean {
+      return isRunningRef.current;
+    },
+  };
+}