react-native-image-stitcher 0.4.1 → 0.5.1

package/ios/Sources/RNImageStitcher/KeyframeGateFrameProcessor.mm

@@ -0,0 +1,196 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// KeyframeGateFrameProcessor.mm — F8.3 vision-camera Frame Processor
+// plugin: a thin pose-injector that hands every producer-thread frame
+// to `IncrementalStitcher.consumeFrameFromPlugin`.
+//
+// JS-side usage (from a worklet):
+//
+//     import { VisionCameraProxy, useFrameProcessor } from
+//       'react-native-vision-camera';
+//
+//     const plugin = VisionCameraProxy.initFrameProcessorPlugin(
+//       'cv_flow_gate_process_frame', {},
+//     );
+//
+//     const fp = useFrameProcessor((frame) => {
+//       'worklet';
+//       if (plugin == null) return;
+//       plugin.call(frame, {
+//         qx, qy, qz, qw,         // gyro-integrated quaternion
+//         fx, fy, cx, cy,         // synthesised intrinsics
+//         imageWidth, imageHeight,
+//         // tx/ty/tz default to 0 (no AR translation in non-AR mode)
+//         // trackingStateRaw default = 2 (= .tracking)
+//       });
+//     }, [plugin]);
+//
+// F8.3 SCOPE — the plugin owns NO gate state and NO per-frame
+// decision logic.  It just:
+//   1. Extracts `CVPixelBuffer` from the vision-camera frame.
+//   2. Builds a pose from the worklet's `arguments` dict (with
+//      defaults safe for non-AR mode).
+//   3. Calls `[IncrementalStitcher.shared consumeFrameFromPlugin:…]`
+//      which routes into the SAME entry point AR mode uses
+//      (`consumeFrame(pixelBuffer:pose:)`).
+//
+// The KeyframeGate evaluation, work-queue dispatch, deep-copy, and
+// engine ingest all happen INSIDE `consumeFrame` — exactly as they
+// already do for AR mode.  Single source of truth, no duplication.
+//
+// CONDITIONAL COMPILATION — this file imports vision-camera headers.
+// The SDK's podspec does NOT declare a Pod dependency on VisionCamera
+// because we don't want non-camera-using consumers to be forced to
+// pull it.  The `__has_include` guard means: if the consumer's pod
+// install pulled vision-camera (which it will, since `<Camera>`
+// requires it as a peer dep), this plugin compiles in.  Otherwise the
+// file is a no-op translation unit.
+
+#import <Foundation/Foundation.h>
+
+#if __has_include(<VisionCamera/FrameProcessorPlugin.h>)
+
+#import <VisionCamera/Frame.h>
+#import <VisionCamera/FrameProcessorPlugin.h>
+#import <VisionCamera/FrameProcessorPluginRegistry.h>
+#import <VisionCamera/VisionCameraProxyHolder.h>
+#import <CoreVideo/CoreVideo.h>
+
+// Forward-declare only the Swift APIs we use here.  Importing the
+// full `RNImageStitcher-Swift.h` would force this TU to also import
+// React (`RCTEventEmitter`, `RCTViewManager`) and ARKit
+// (`ARSessionDelegate`), because the generated header exposes every
+// `@objc` symbol in the module.  We don't need any of those.
+//
+// Risk: this declaration must stay in sync with the Swift extension
+// at the bottom of `IncrementalStitcher.swift`.  Both files are
+// committed together; signature drift would be caught at link time
+// (unresolved selector) and at the next build.
+@class IncrementalStitcher;
+@interface IncrementalStitcher : NSObject
++ (IncrementalStitcher * _Nonnull)shared;
+- (void)consumeFrameFromPluginWithPixelBuffer:(CVPixelBufferRef _Nonnull)pixelBuffer
+                                           tx:(double)tx
+                                           ty:(double)ty
+                                           tz:(double)tz
+                                           qx:(double)qx
+                                           qy:(double)qy
+                                           qz:(double)qz
+                                           qw:(double)qw
+                                           fx:(double)fx
+                                           fy:(double)fy
+                                           cx:(double)cx
+                                           cy:(double)cy
+                                   imageWidth:(NSInteger)imageWidth
+                                  imageHeight:(NSInteger)imageHeight
+                                  timestampMs:(double)timestampMs
+                             trackingStateRaw:(NSInteger)trackingStateRaw;
+@end
+
+// Read a Double from the per-call `arguments` dict with a default.
+// Used to extract pose params; tolerant of missing keys (non-AR mode
+// may send only the rotation fields, not translation/intrinsics).
+static double kg_argDouble(NSDictionary* args, NSString* key, double defaultValue) {
+  if (args == nil) return defaultValue;
+  NSNumber* n = args[key];
+  return [n isKindOfClass:[NSNumber class]] ? n.doubleValue : defaultValue;
+}
+static NSInteger kg_argInt(NSDictionary* args, NSString* key, NSInteger defaultValue) {
+  if (args == nil) return defaultValue;
+  NSNumber* n = args[key];
+  return [n isKindOfClass:[NSNumber class]] ? n.integerValue : defaultValue;
+}
+
+@interface KeyframeGateFrameProcessor : FrameProcessorPlugin
+@end
+
+@implementation KeyframeGateFrameProcessor
+
+- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy
+                  withOptions:(NSDictionary* _Nullable)options {
+  // No per-instance setup.  All gate tunables (overlapThreshold,
+  // maxCount, flow params, strategy, ...) live on
+  // `IncrementalStitcher` and are configured at its `start()` time
+  // from the host-app settings.  The plugin is a stateless
+  // pose-injector.
+  return [super initWithProxy:proxy withOptions:options];
+}
+
+- (id)callback:(Frame*)frame withArguments:(NSDictionary* _Nullable)arguments {
+  CMSampleBufferRef sampleBuffer = frame.buffer;
+  if (sampleBuffer == NULL) {
+    return @{@"submitted": @NO, @"error": @"no sample buffer"};
+  }
+
+  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+  if (pixelBuffer == NULL) {
+    return @{@"submitted": @NO, @"error": @"no pixel buffer"};
+  }
+
+  // Frame dims for the pose.  Read from plane 0 if planar (YUV) else
+  // whole buffer; this is the dimensionality the stitcher expects.
+  size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
+  NSInteger width  = (NSInteger)(planeCount >= 1
+      ? CVPixelBufferGetWidthOfPlane(pixelBuffer, 0)
+      : CVPixelBufferGetWidth(pixelBuffer));
+  NSInteger height = (NSInteger)(planeCount >= 1
+      ? CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)
+      : CVPixelBufferGetHeight(pixelBuffer));
+
+  // Pose from worklet args.  Defaults are safe non-AR values:
+  //   * tx/ty/tz = 0 (no translation in non-AR; gyro only gives rot)
+  //   * qw = 1 (identity quaternion if JS hasn't supplied rotation)
+  //   * fx/fy/cx/cy = 0 → JS-driver caller MUST supply these (the
+  //     engine derives FoV from intrinsics; 0 would yield NaN FoV).
+  //     We default the principal point to image centre as a safer
+  //     fallback if only fx/fy are missing.
+  //   * trackingStateRaw = 2 → `.tracking` (non-AR captures don't
+  //     have a real tracking-quality signal; engine's `trackingPoor`
+  //     path stays inactive, matching legacy `useIncrementalJSDriver`).
+  double tx = kg_argDouble(arguments, @"tx", 0.0);
+  double ty = kg_argDouble(arguments, @"ty", 0.0);
+  double tz = kg_argDouble(arguments, @"tz", 0.0);
+  double qx = kg_argDouble(arguments, @"qx", 0.0);
+  double qy = kg_argDouble(arguments, @"qy", 0.0);
+  double qz = kg_argDouble(arguments, @"qz", 0.0);
+  double qw = kg_argDouble(arguments, @"qw", 1.0);
+  double fx = kg_argDouble(arguments, @"fx", 0.0);
+  double fy = kg_argDouble(arguments, @"fy", 0.0);
+  double cx = kg_argDouble(arguments, @"cx", (double)width  / 2.0);
+  double cy = kg_argDouble(arguments, @"cy", (double)height / 2.0);
+  double timestampMs = kg_argDouble(arguments, @"timestampMs", 0.0);
+  NSInteger trackingState = kg_argInt(arguments, @"trackingStateRaw", 2);
+
+  // Submit.  consumeFrame internally early-returns if isRunning ==
+  // false, so it's safe to call every producer-thread frame whether
+  // or not a capture is in progress.  ~1-2 µs of overhead per
+  // "stitcher not running" frame; negligible at 30 fps.
+  [IncrementalStitcher.shared
+      consumeFrameFromPluginWithPixelBuffer:pixelBuffer
+                                         tx:tx ty:ty tz:tz
+                                         qx:qx qy:qy qz:qz qw:qw
+                                         fx:fx fy:fy cx:cx cy:cy
+                                 imageWidth:width
+                                imageHeight:height
+                                timestampMs:timestampMs
+                           trackingStateRaw:trackingState];
+
+  return @{@"submitted": @YES};
+}
+
+// Auto-register the plugin at class-load time.  Name must match what
+// JS passes to `VisionCameraProxy.initFrameProcessorPlugin(...)`.
++ (void)load {
+  [FrameProcessorPluginRegistry
+    addFrameProcessorPlugin:@"cv_flow_gate_process_frame"
+            withInitializer:^FrameProcessorPlugin* _Nonnull(
+                VisionCameraProxyHolder* proxy,
+                NSDictionary* _Nullable options) {
+              return [[KeyframeGateFrameProcessor alloc]
+                  initWithProxy:proxy withOptions:options];
+            }];
+}
+
+@end
+
+#endif // __has_include(<VisionCamera/FrameProcessorPlugin.h>)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-image-stitcher",
-  "version": "0.4.1",
+  "version": "0.5.1",
   "description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -61,6 +61,7 @@
     "react-native-safe-area-context": "^4.0.0",
     "react-native-sensors": "^7.0.0",
     "react-native-vision-camera": "^4.0.0",
+    "react-native-worklets-core": "^1.3.0",
     "rxjs": "^7.0.0",
     "ts-jest": "^29.1.0",
     "typescript": "^5.5.0"
@@ -69,6 +70,7 @@
     "react": ">=18.0.0",
     "react-native": ">=0.72.0",
     "react-native-vision-camera": ">=4.7.0",
+    "react-native-worklets-core": ">=1.3.0",
     "react-native-sensors": ">=7.0.0",
     "react-native-safe-area-context": ">=4.0.0"
   }

package/src/camera/Camera.tsx

@@ -56,7 +56,11 @@ import {
   type ViewStyle,
 } from 'react-native';
 import { useSafeAreaInsets } from 'react-native-safe-area-context';
-import type { Camera as VisionCamera } from 'react-native-vision-camera';
+import type {
+  Camera as VisionCamera,
+  DrawableFrameProcessor,
+  ReadonlyFrameProcessor,
+} from 'react-native-vision-camera';
 
 import { useARSession } from '../ar/useARSession';
 import { ARCameraView, type ARCameraViewHandle } from './ARCameraView';
@@ -86,6 +90,7 @@ import {
   type IncrementalState,
 } from '../stitching/incremental';
 import { useIncrementalJSDriver } from '../stitching/useIncrementalJSDriver';
+import { useFrameProcessorDriver } from '../stitching/useFrameProcessorDriver';
 import { useIncrementalStitcher } from '../stitching/useIncrementalStitcher';
 import { useIMUTranslationGate } from '../sensors/useIMUTranslationGate';
 import { toBareFilePath, toFileUri } from '../utils/paths';
@@ -164,6 +169,15 @@ export type CameraErrorCode =
   | 'STITCH_CAMERA_PARAMS_FAIL'
   | 'STITCH_OOM'
   | 'OUTPUT_WRITE_FAILED'
+  /**
+   * Vision-camera surfaced a runtime error that isn't a known
+   * transient lifecycle event (those are swallowed inside the SDK's
+   * `<CameraView>`).  Examples that DO reach the host as this code:
+   * `format/invalid-format`, `capture/recording-canceled`,
+   * `device/microphone-permission-denied`, ...  The full error
+   * object is on `.cause` for inspection.
+   */
+  | 'VISION_CAMERA_RUNTIME'
   | 'UNKNOWN';
 
 
@@ -221,6 +235,29 @@ export interface CameraProps {
   showSettingsButton?: boolean;
   style?: StyleProp<ViewStyle>;
 
+  /**
+   * Which incremental stitcher engine to drive.  Default
+   * `'batch-keyframe'` — collects accepted JPEGs and runs
+   * `cv::Stitcher` once at finalize time.  This is the v0.4+
+   * production default and what the v0.5 Frame Processor migration
+   * exercises.
+   *
+   * Switch to a live engine (`'firstwins-rectilinear'` or
+   * `'hybrid'`) for low-latency in-flight stitching.  Live engines
+   * exercise the F8.6 pixel-buffer ingest path (skipping the JPEG
+   * encode/decode round-trip; ~30–50 ms saved per accept) when the
+   * Frame Processor driver is active.
+   *
+   * See `docs/f8-frame-processor-plan.md` and the v0.5.0
+   * CHANGELOG for the trade-offs between batch-keyframe and live
+   * engines.
+   */
+  engine?: 'batch-keyframe'
+    | 'hybrid'
+    | 'slitscan-rotate' | 'slitscan-both'
+    | 'firstwins' | 'firstwins-zoomed' | 'firstwins-rectilinear'
+    | 'slitscan';
+
   /**
    * Optional destination directory for captures.  When set, the lib
    * lands tap-photos at `${outputDir}/photo-${ts}.jpg` and panoramas
@@ -256,6 +293,47 @@ export interface CameraProps {
   onLensChange?: (lens: CameraLens) => void;
   onFramesDropped?: (info: FramesDroppedInfo) => void;
   onError?: (err: CameraError) => void;
+
+  /**
+   * Optional vision-camera frame processor.  Only attached to the
+   * non-AR preview (AR mode uses ARCameraView, which doesn't expose
+   * a worklet seam).  Build the worklet on the host side with
+   * `useFrameProcessor` from `react-native-vision-camera`.
+   *
+   * Introduced for F8 (FrameProcessor port) — see
+   * `docs/f8-frame-processor-plan.md`.
+   *
+   * As of v0.5 (F8.3) this prop is **deprecated for the standard
+   * non-AR capture flow**: the SDK now installs its own frame
+   * processor via `useFrameProcessorDriver` that pipes pixel
+   * buffers into the incremental stitcher with synthesised pose.
+   * Setting this prop in the default mode will be IGNORED with a
+   * one-time console.warn — supplying your own worklet would race
+   * with the SDK's pixel-buffer feed.
+   *
+   * Three coexistence rules:
+   *   * Default (modern non-AR): SDK owns the worklet, this prop
+   *     is ignored.
+   *   * `legacyDriver={true}`: SDK uses the old `useIncrementalJSDriver`
+   *     (takeSnapshot path).  Honoured for diagnostics or as an
+   *     escape hatch.
+   *   * AR mode: vision-camera Camera isn't mounted, this prop is
+   *     irrelevant.
+   */
+  frameProcessor?: ReadonlyFrameProcessor | DrawableFrameProcessor;
+
+  /**
+   * Opt back into the legacy `useIncrementalJSDriver` for non-AR
+   * captures (the v0.4 path: `takeSnapshot` → JPEG → cache file →
+   * `IncrementalStitcher.processFrameAtPath`).
+   *
+   * Default `false` (use the new `useFrameProcessorDriver`, which
+   * runs the gate on the camera producer thread at native frame
+   * rate via a vision-camera Frame Processor plugin).  The legacy
+   * path will be removed in v0.6 — set this only if you hit a
+   * specific issue with the new driver and need to ship a fix.
+   */
+  legacyDriver?: boolean;
 }
 
 
@@ -530,6 +608,9 @@ export function Camera(props: CameraProps): React.JSX.Element {
     onLensChange,
     onFramesDropped,
     onError,
+    frameProcessor: hostFrameProcessor,
+    legacyDriver = false,
+    engine = 'batch-keyframe',
   } = props;
 
   const insets = useSafeAreaInsets();
@@ -727,10 +808,45 @@ export function Camera(props: CameraProps): React.JSX.Element {
   // imperative pattern (start on hold-start, stop on hold-end) avoids
   // the re-render churn entirely.
   const jsDriver = useIncrementalJSDriver();
-  // Safety: ensure the driver is stopped if the component unmounts
+  // F8.3 — vision-camera Frame Processor variant.  Always
+  // instantiated so we don't have conditional hook calls; only one
+  // of the two drivers actually .start()s per capture.  Stop() on
+  // an idle driver is a no-op.
+  const fpDriver = useFrameProcessorDriver();
+  // Safety: ensure both drivers are stopped if the component unmounts
   // mid-recording.  Empty deps so this only fires on unmount.
   // eslint-disable-next-line react-hooks/exhaustive-deps
-  useEffect(() => () => { jsDriver.stop(); }, []);
+  useEffect(() => () => { jsDriver.stop(); fpDriver.stop(); }, []);
+
+  // F8.3 — one-shot deprecation warning when the host supplies their
+  // own `frameProcessor` while running in the default (Frame
+  // Processor driver) mode.  Two worklets racing on the same
+  // producer thread would corrupt the engine's workQueue ordering;
+  // the SDK's own worklet wins and the host's is ignored.  Hosts
+  // that *need* a custom worklet must opt into `legacyDriver={true}`
+  // (which switches off the SDK's worklet entirely).
+  const hostFrameProcessorIgnoredWarnedRef = useRef(false);
+  if (
+    hostFrameProcessor != null
+    && !legacyDriver
+    && !hostFrameProcessorIgnoredWarnedRef.current
+  ) {
+    hostFrameProcessorIgnoredWarnedRef.current = true;
+    // eslint-disable-next-line no-console
+    console.warn(
+      '[react-native-image-stitcher] The `frameProcessor` prop on '
+      + '<Camera> is ignored when the default driver is active '
+      + '(legacyDriver=false).  Either remove the prop or set '
+      + 'legacyDriver={true} to opt into the legacy path.',
+    );
+  }
+  // The Frame Processor worklet actually bound to vision-camera's
+  // Camera.  Resolution order:
+  //   1. Legacy mode: honor the host's prop (or null).
+  //   2. Modern mode: SDK driver's worklet, regardless of host's prop.
+  const effectiveFrameProcessor = legacyDriver
+    ? (hostFrameProcessor ?? null)
+    : fpDriver.frameProcessor;
 
   // ── Subscribe to engine state for live keyframe thumbs ──────────
   useEffect(() => {
@@ -787,6 +903,17 @@ export function Camera(props: CameraProps): React.JSX.Element {
     const accepted = incrementalState?.acceptedCount ?? 0;
     if (accepted > lastAcceptedCountRef.current) {
       lastAcceptedCountRef.current = accepted;
+      // F8.3 review-of-review (M3 revert): originally gated this to
+      // `legacyDriver` because the Frame Processor driver doesn't
+      // consult `imuGate` for its own pose synthesis.  That ignored a
+      // load-bearing side effect: `imuGate.resetAnchor()` bounds the
+      // IIR-integrator drift window per-accept, and
+      // `imuGate.getTotalAbsMetres()` is read at finalize time
+      // (Camera.tsx:1097) as `imuTranslationMetres` into the native
+      // stitchMode auto-resolver (PANORAMA vs SCANS).  Without the
+      // per-accept reset, long FP-driver captures let IIR drift
+      // compound → inflated metres → biased toward SCANS.  Keep the
+      // reset firing for ALL non-AR modes.
       if (isNonAR) {
         imuGate.resetAnchor();
       }
@@ -917,26 +1044,43 @@ export function Camera(props: CameraProps): React.JSX.Element {
         snapshotEveryNAccepts: 1,
         frameRotationDegrees: orientationRotation,
         captureOrientation: deviceOrientation,
-        frameSourceMode: isNonAR ? 'jsDriver' : 'arSession',
+        // F8.3 — non-AR captures pick between the new Frame Processor
+        // driver (default) and the legacy JS-snapshot driver (opt-in
+        // via `legacyDriver={true}`).  AR captures always use the
+        // ARSession-driven path.
+        frameSourceMode: isNonAR
+          ? (legacyDriver ? 'jsDriver' : 'frameProcessor')
+          : 'arSession',
         composeWidth: 1920,
         composeHeight: 1080,
         canvasWidth: 5000,
         canvasHeight: 5000,
-        engine: 'batch-keyframe',
+        engine,
         config: panoramaSettingsToNativeConfig({
           ...settings,
           captureSource: effectiveCaptureSource,
         }),
       });
+      // F8.3 review-of-review (M3 revert): `imuGate.resetAnchor()`
+      // is load-bearing for the stitchMode auto-resolver (see the
+      // matching comment on the per-accept reset useEffect above).
+      // Keep firing it on every capture start, not just legacy mode.
       imuGate.resetAnchor();
-      // Start pumping vision-camera snapshots into the engine for
-      // non-AR captures.  AR mode feeds frames natively from the
-      // ARSession, so the JS driver stays idle in that path.  This
-      // mirrors AuditCaptureScreen.handleHoldStart's `androidDriver.start`
-      // imperative call — see the comment near `useIncrementalJSDriver`
-      // for why this is NOT done via useEffect.
+      // Start the non-AR frame source.  AR mode feeds natively from
+      // ARSession so both drivers stay idle in that path.
+      //   * Default: Frame Processor driver — worklet runs on the
+      //     producer thread, plugin calls `consumeFrameFromPlugin`
+      //     directly.  No camera ref needed (vision-camera owns it).
+      //   * Legacy: JS driver — `takeSnapshot` + `processFrameAtPath`
+      //     via the cameraRef.
+      // Imperative-pattern rationale: see the useIncrementalJSDriver
+      // comment above re. why this isn't a useEffect.
       if (isNonAR) {
-        jsDriver.start(visionCameraRef);
+        if (legacyDriver) {
+          jsDriver.start(visionCameraRef);
+        } else {
+          fpDriver.start();
+        }
       }
     } catch (err) {
       setStatusPhase('idle');
@@ -957,16 +1101,22 @@ export function Camera(props: CameraProps): React.JSX.Element {
     effectiveCaptureSource,
     imuGate,
     jsDriver,
+    fpDriver,
+    legacyDriver,
+    engine,
     onError,
   ]);
 
   const handleHoldEnd = useCallback(async () => {
     if (statusPhase !== 'recording') return;
     setStatusPhase('stitching');
-    // Stop pumping new snapshots before finalizing so the engine isn't
-    // racing the final cv::Stitcher pass against late-arriving keyframes.
-    // No-op in AR mode where jsDriver was never started.
+    // Stop pumping new frames before finalizing so the engine isn't
+    // racing the final cv::Stitcher pass against late-arriving
+    // keyframes.  Both stop() calls are no-ops when the
+    // corresponding driver wasn't started (AR mode, or the inactive
+    // driver in non-AR mode).
     jsDriver.stop();
+    fpDriver.stop();
     try {
       // Compose the panorama output path: host-controlled if
       // `outputDir` is set, else the lib's canonical capture dir
@@ -1044,6 +1194,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
     onError,
     recordingStartedAt,
     jsDriver,
+    fpDriver,
     // F10 Phase 2 review N1 — these four were missing pre-fix.  The
     // callback reads `settings.debug` (to gate the stitchToast),
     // `isNonAR` (to decide whether to read IMU totalAbs translation),
@@ -1101,6 +1252,30 @@ export function Camera(props: CameraProps): React.JSX.Element {
           video
           flash="off"
           style={StyleSheet.absoluteFill}
+          // F8 (FrameProcessor port) — host-supplied worklet runs on
+          // the camera producer thread for every frame.  Only wired
+          // in non-AR mode; AR mode uses ARCameraView which doesn't
+          // expose a frame-processor seam.  See
+          // docs/f8-frame-processor-plan.md.
+          cameraProps={effectiveFrameProcessor != null
+            ? { frameProcessor: effectiveFrameProcessor }
+            : undefined}
+          onError={(err) => {
+            // CameraView already filters known transient lifecycle
+            // errors (screen-lock, etc.) before invoking this.  What
+            // reaches here is a real vision-camera runtime issue:
+            // pull `code`/`message` defensively (the type is
+            // `unknown` from CameraView's perspective) and wrap in
+            // a SDK-typed `CameraError` so hosts get a stable shape.
+            const e = err as { code?: string; message?: string };
+            const codeStr = e?.code ?? 'unknown';
+            const msg = e?.message ?? String(err);
+            onError?.(new CameraError(
+              'VISION_CAMERA_RUNTIME',
+              `${codeStr}: ${msg}`,
+              err,
+            ));
+          }}
         />
       )}
 

package/src/camera/CameraView.tsx

@@ -60,6 +60,24 @@ export interface CameraViewProps {
    * preferences (focus-on-tap vs. tap-to-lock).
    */
   onPreviewTap?: (event: { x: number; y: number }) => void;
+
+  /**
+   * Forwarded from vision-camera's `<Camera onError>` AFTER lifecycle
+   * errors are filtered.  The SDK's built-in filter swallows:
+   *
+   *   * `system/camera-is-restricted` — screen-lock / DoNotDisturb
+   *     temporarily revokes camera access; vision-camera re-acquires
+   *     on resume.  Logged to console.warn, NOT surfaced.
+   *   * `system/camera-has-been-disconnected` — another app grabbed
+   *     the camera.  Same auto-recovery.
+   *   * `device/camera-already-in-use` — same class as above.
+   *
+   * Real errors (permission denials, hardware failures, malformed
+   * format requests) are forwarded.  Hosts can therefore safely
+   * pipe this to a redbox / Crashlytics without getting paged on
+   * routine screen-lock events.
+   */
+  onError?: (error: unknown) => void;
 }
 
 
@@ -68,6 +86,19 @@ export interface CameraViewProps {
  * to callers (so ``cameraRef.current.takePhoto()`` keeps working),
  * while presenting a smaller API on the outside.
  */
+// Error codes vision-camera reports for transient lifecycle events.
+// Filtered out of the SDK's onError forward (see `handleVcError` in
+// the body): the camera self-recovers when the device comes back into
+// the foreground / regains permission / the other app releases the
+// device.  Surfacing these as host errors causes spurious crash
+// reports during routine phone-lock / app-switch operations.
+const VC_LIFECYCLE_ERROR_CODES: ReadonlySet<string> = new Set([
+  'system/camera-is-restricted',         // screen lock, DoNotDisturb, MDM policy
+  'system/camera-has-been-disconnected', // another app grabbed the camera
+  'device/camera-already-in-use',        // same class as above
+]);
+
+
 export const CameraView = forwardRef<Camera | null, CameraViewProps>(function CameraView(
   {
     device,
@@ -77,9 +108,27 @@ export const CameraView = forwardRef<Camera | null, CameraViewProps>(function Ca
     guidance,
     style,
     cameraProps,
+    onError,
   },
   ref,
 ): React.JSX.Element {
+  // Error filter — see `VC_LIFECYCLE_ERROR_CODES` for the swallow
+  // list rationale.  `code` on vision-camera's `CameraRuntimeError`
+  // is typed as a string; treat any non-string defensively as a
+  // "forward it" so we don't accidentally swallow unknown errors.
+  const handleVcError = (err: unknown): void => {
+    const code = (err as { code?: unknown })?.code;
+    if (typeof code === 'string' && VC_LIFECYCLE_ERROR_CODES.has(code)) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        '[react-native-image-stitcher] vision-camera reported a '
+        + `transient lifecycle error (${code}); the camera will `
+        + 'auto-recover on resume.  Not forwarding to onError.',
+      );
+      return;
+    }
+    onError?.(err);
+  };
   // Internal ref so we can both attach to <Camera> and forward outward.
   const innerRef = useRef<Camera>(null);
   useImperativeHandle(ref, () => innerRef.current as Camera);
@@ -112,6 +161,7 @@ export const CameraView = forwardRef<Camera | null, CameraViewProps>(function Ca
         // "what you see is what was taken".
         outputOrientation="device"
         torch={flash === 'on' ? 'on' : 'off'}
+        onError={handleVcError}
         {...cameraProps}
       />
       {guidance ? (

package/src/index.ts

@@ -178,6 +178,18 @@ export {
 export type { IncrementalState } from './stitching/incremental';
 export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
 export { useIncrementalJSDriver } from './stitching/useIncrementalJSDriver';
+export type {
+  UseIncrementalJSDriverOptions,
+  IncrementalJSDriverHandle,
+} from './stitching/useIncrementalJSDriver';
+// F8.3 — vision-camera Frame Processor variant of the non-AR
+// driver.  Preferred over `useIncrementalJSDriver` in v0.5+; the
+// JS driver stays exported as a deprecated fallback until v0.6.
+export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
+export type {
+  UseFrameProcessorDriverOptions,
+  FrameProcessorDriverHandle,
+} from './stitching/useFrameProcessorDriver';
 
 // ── Batch stitching ───────────────────────────────────────────────────
 // Feed a video file straight to OpenCV's cv::Stitcher, bypassing the

package/src/stitching/incremental.ts

@@ -203,11 +203,20 @@ export interface IncrementalStartOptions {
    *   - 'jsDriver' — engine skips AR-session registration; JS
    *     feeds frames via `processFrameAtPath`.  Use in iOS non-AR
    *     captures (vision-camera + gyro).  No AR session required.
+   *     LEGACY; deprecated in v0.5, removed in v0.6.
    *
-   * Android ignores this option — its engine always accepts
-   * JS-driven frames.
+   *   - 'frameProcessor' (F8.3 iOS / F8.4 Android, v0.5+) — engine
+   *     flips on `frameProcessorIngestEnabled` so the vision-camera
+   *     Frame Processor plugin (`cv_flow_gate_process_frame`) can
+   *     feed pixel data directly into the engine's gate path.  iOS
+   *     passes the `CVPixelBuffer` straight to `consumeFrame`;
+   *     Android extracts the Y plane to a ByteArray and encodes
+   *     accepted frames to JPEG inline (the platform-specific
+   *     engine-input divergence is tracked as F8.6).  Use in non-AR
+   *     captures driven by `useFrameProcessorDriver`.  Pairs with
+   *     `Camera`'s default driver mode.
    */
-  frameSourceMode?: 'arSession' | 'jsDriver';
+  frameSourceMode?: 'arSession' | 'jsDriver' | 'frameProcessor';
   /** Compose-resolution width in pixels (default 720 for portrait, 960 for landscape). */
   composeWidth?: number;
   /** Compose-resolution height in pixels (default 960 for portrait, 720 for landscape). */