react-native-image-stitcher 0.8.0 → 0.10.0

package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift

@@ -1863,12 +1863,27 @@ public final class IncrementalStitcher: NSObject {
         config: [String: Any],
         completion: @escaping ([String: Any]?, NSError?) -> Void
     ) {
+        // v0.10.0 #15A — emit `validating` at the very top so JS sees
+        // refine activity even when validation fails fast.  Frames may
+        // be empty here; report whatever the caller asked for.
+        emitRefineProgress(
+            stage: "validating",
+            fraction: 0.05,
+            frames: framePaths.count,
+            errorMessage: nil
+        )
         guard framePaths.count >= 2 else {
+            let msg = "refinePanorama requires at least 2 framePaths (got \(framePaths.count))."
+            emitRefineProgress(
+                stage: "error",
+                fraction: 1.0,
+                frames: framePaths.count,
+                errorMessage: msg
+            )
             completion(nil, NSError(
                 domain: "RNImageStitcherIncremental",
                 code: 9101,
-                userInfo: [NSLocalizedDescriptionKey:
-                    "refinePanorama requires at least 2 framePaths (got \(framePaths.count))."]
+                userInfo: [NSLocalizedDescriptionKey: msg]
             ))
             return
         }
@@ -1876,11 +1891,17 @@ public final class IncrementalStitcher: NSObject {
         for p in framePaths {
             let cleaned = p.hasPrefix("file://") ? String(p.dropFirst(7)) : p
             if !fm.fileExists(atPath: cleaned) {
+                let msg = "refinePanorama: keyframe missing on disk — \(cleaned)"
+                emitRefineProgress(
+                    stage: "error",
+                    fraction: 1.0,
+                    frames: framePaths.count,
+                    errorMessage: msg
+                )
                 completion(nil, NSError(
                     domain: "RNImageStitcherIncremental",
                     code: 9102,
-                    userInfo: [NSLocalizedDescriptionKey:
-                        "refinePanorama: keyframe missing on disk — \(cleaned)"]
+                    userInfo: [NSLocalizedDescriptionKey: msg]
                 ))
                 return
             }
@@ -1912,7 +1933,18 @@ public final class IncrementalStitcher: NSObject {
                cleanedOutput,
                warper, blender, seam)
 
-        refineQueue.async {
+        // v0.10.0 #15A — capture for the inner closure; `self` is
+        // captured weakly so the progress emitter survives only as
+        // long as the IncrementalStitcher itself does.  An emitter
+        // call on a torn instance is a no-op via `?.`.
+        let frameCount = framePaths.count
+        refineQueue.async { [weak self] in
+            self?.emitRefineProgress(
+                stage: "stitching",
+                fraction: 0.1,
+                frames: frameCount,
+                errorMessage: nil
+            )
             // C2-style: closure captures only value-typed locals
             // (paths, output path, config strings).  No `self` access
             // is needed for the cv::Stitcher call — OpenCVStitcher is
@@ -1935,24 +1967,53 @@ public final class IncrementalStitcher: NSObject {
                 // OpenCVStitcher hit one of its six guarded failure
                 // returns; surface as a clean NSError.
                 if r.width == 0 && r.height == 0 {
+                    let msg = "refinePanorama: stitcher returned sentinel — see preceding [BatchStitcher] log for cause."
+                    self?.emitRefineProgress(
+                        stage: "error",
+                        fraction: 1.0,
+                        frames: frameCount,
+                        errorMessage: msg
+                    )
                     completion(nil, NSError(
                         domain: "RNImageStitcherIncremental",
                         code: 9107,
-                        userInfo: [NSLocalizedDescriptionKey:
-                            "refinePanorama: stitcher returned sentinel — see preceding [BatchStitcher] log for cause."]
+                        userInfo: [NSLocalizedDescriptionKey: msg]
                     ))
                     return
                 }
+                // Stitch succeeded — OpenCVStitcher writes the JPEG
+                // internally before returning, so "writing" really
+                // captures the final assembly + file I/O cost.  Emit
+                // here so JS can flip its label from "Stitching" to
+                // "Writing" before the done event fires.
+                self?.emitRefineProgress(
+                    stage: "writing",
+                    fraction: 0.9,
+                    frames: frameCount,
+                    errorMessage: nil
+                )
+                self?.emitRefineProgress(
+                    stage: "done",
+                    fraction: 1.0,
+                    frames: frameCount,
+                    errorMessage: nil
+                )
                 completion([
                     "panoramaPath": r.outputPath,
                     "width": Int(r.width),
                     "height": Int(r.height),
-                    "framesRequested": framePaths.count,
-                    "framesIncluded": framePaths.count,
+                    "framesRequested": frameCount,
+                    "framesIncluded": frameCount,
                     "framesDropped": 0,
                     "finalConfidenceThresh": -1.0,
                 ], nil)
             } catch let err as NSError {
+                self?.emitRefineProgress(
+                    stage: "error",
+                    fraction: 1.0,
+                    frames: frameCount,
+                    errorMessage: err.localizedDescription
+                )
                 completion(nil, err)
             }
         }
@@ -2092,6 +2153,74 @@ public final class IncrementalStitcher: NSObject {
         )
     }
 
+    /// v0.10.0 #15A — emit a refine-pipeline phase update on the same
+    /// `IncrementalStateUpdate` channel that carries `isRefining` /
+    /// `refinedPanoramaPath`.  Five `stage` values fire across the
+    /// lifetime of one `refinePanorama` call:
+    ///
+    ///   - `validating` (fraction 0.05) — synchronous input checks
+    ///   - `stitching`  (fraction 0.10) — start of the OpenCV stitch
+    ///   - `writing`    (fraction 0.90) — stitch returned, JPEG written
+    ///   - `done`       (fraction 1.00) — completion handler invoked
+    ///   - `error`      (fraction 1.00) — failure path (`errorMessage`
+    ///                                    is non-nil)
+    ///
+    /// `fraction` is intentionally coarse: OpenCV's `Stitcher` doesn't
+    /// expose stage-by-stage callbacks, so the 0.10 → 0.90 jump is a
+    /// single opaque step.  JS uses the `stage` string for the UI
+    /// label and `fraction` purely for spinner progress.
+    ///
+    /// Reuses the existing channel (rather than introducing a new
+    /// device-event name) so the JS subscriber doesn't need to wire
+    /// a second listener.  The payload carries the same skeleton
+    /// `emitRefinementState` emits (lastState fields preserved) so
+    /// `isRefining` / `refinedPanoramaPath` sticky-merge logic on the
+    /// JS side keeps working untouched.
+    private func emitRefineProgress(
+        stage: String,
+        fraction: Double,
+        frames: Int?,
+        errorMessage: String?
+    ) {
+        // Disk-trail breadcrumb — every refine emit lands here so a
+        // future regression can be diagnosed by pulling the bridge's
+        // debug file without needing live Console.app access.
+        IncrementalStitcher.fileLog(
+            "[refine.progress] stage=\(stage) frac=\(fraction) frames=\(frames ?? -1) hasError=\(errorMessage != nil)"
+        )
+        stateLock.lock()
+        let prev = self.lastState
+        stateLock.unlock()
+        let state = IncrementalStateObject(
+            panoramaPath: prev?.panoramaPath,
+            width: prev?.width ?? 0,
+            height: prev?.height ?? 0,
+            acceptedCount: prev?.acceptedCount ?? 0,
+            outcome: prev?.outcome ?? .acceptedHigh,
+            confidence: prev?.confidence ?? 1.0,
+            overlapPercent: prev?.overlapPercent ?? -1.0,
+            processingMs: 0,
+            isLandscape: prev?.isLandscape ?? false,
+            paintedExtent: prev?.paintedExtent ?? 0,
+            panExtent: prev?.panExtent ?? 0,
+            keyframeMax: prev?.keyframeMax ?? 0
+        )
+        var dict = state.asDictionary()
+        dict["refineStage"] = stage
+        dict["refineProgress"] = fraction
+        if let f = frames {
+            dict["refineFrames"] = f
+        }
+        if let e = errorMessage {
+            dict["refineError"] = e
+        }
+        NotificationCenter.default.post(
+            name: .retailensIncrementalStateUpdate,
+            object: nil,
+            userInfo: dict
+        )
+    }
+
     /// Cancel an in-progress capture without producing output.
     /// Same V12.1 synchronous-stop pattern as finalize.
     @objc public func cancel() {

package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift

@@ -33,6 +33,18 @@ public final class IncrementalStitcherBridge: RCTEventEmitter {
 
     public override init() {
         super.init()
+        // Under RN bridgeless interop the bridge's init() can be
+        // invoked twice on the same instance (observed via identical
+        // instance pointers firing the observer selector twice per
+        // notification).  Defensively remove any prior registration
+        // for this notification name before adding one, so the
+        // observer can only fire once per post regardless of how
+        // many times init runs.
+        NotificationCenter.default.removeObserver(
+            self,
+            name: .retailensIncrementalStateUpdate,
+            object: nil
+        )
         // Subscribe once at construction.  The handler self-checks
         // `hasListeners` before forwarding, so we don't have to
         // unsubscribe / resubscribe on every JS listener attach/detach.
@@ -58,9 +70,6 @@ public final class IncrementalStitcherBridge: RCTEventEmitter {
         return [Self.stateUpdateEvent]
     }
 
-    // (startObserving / stopObserving moved next to handleStateUpdate
-    //  for the PiP investigation; remove this comment after.)
-
     // MARK: - Module methods
 
     /// `options` (all optional, sensible defaults documented in
@@ -477,26 +486,53 @@ public final class IncrementalStitcherBridge: RCTEventEmitter {
 
     @objc private func handleStateUpdate(_ notification: Notification) {
         let hasPath = (notification.userInfo?["panoramaPath"] != nil)
-        if hasPath {
+        let refineStage = notification.userInfo?["refineStage"] as? String
+        if hasPath || refineStage != nil {
             IncrementalStitcher.fileLog(
-                "bridge handleStateUpdate hasListeners=\(hasListeners) hasPath=\(hasPath) thread=\(Thread.isMainThread ? "main" : "bg")"
+                "bridge handleStateUpdate hasListeners=\(hasListeners) hasPath=\(hasPath) refineStage=\(refineStage ?? "nil") thread=\(Thread.isMainThread ? "main" : "bg")"
             )
         }
-        guard hasListeners else { return }
+        guard hasListeners else {
+            if let stage = refineStage {
+                IncrementalStitcher.fileLog(
+                    "bridge handleStateUpdate DROPPED refineStage=\(stage) — hasListeners=false"
+                )
+            }
+            return
+        }
         guard let userInfo = notification.userInfo else { return }
-        // FIX: RCTEventEmitter.sendEvent is documented to be called
-        // from any thread, but in practice events from background
-        // threads can be dropped silently if the bridge is in
-        // certain states.  Dispatch to main queue to guarantee
-        // delivery.  See e.g. RN issues #19518, #28250.
+        // We deliver via `bridge.enqueueJSCall("RCTDeviceEventEmitter", "emit", ...)`
+        // rather than `RCTEventEmitter.sendEvent(...)` because under RN
+        // bridgeless interop `sendEvent` silently no-ops for some
+        // event-body shapes even when `_bridge` is non-nil and
+        // `_listenerCount > 0` (confirmed via os_log instrumentation
+        // during v0.10.0 PR B development — refine events with
+        // refineStage/refineProgress/refineFrames were not reaching
+        // any JS subscriber while live state events with a smaller
+        // body shape on the same channel were).  enqueueJSCall is
+        // the underlying mechanism sendEvent uses in Paper mode, so
+        // it is strictly at least as well-supported.
         DispatchQueue.main.async { [weak self] in
             guard let self = self else { return }
-            if hasPath {
+            if hasPath || refineStage != nil {
                 IncrementalStitcher.fileLog(
-                    "bridge sendEvent (main queue) body.panoramaPath=\(userInfo["panoramaPath"] ?? "MISSING")"
+                    "bridge enqueueJSCall (main queue) body.panoramaPath=\(userInfo["panoramaPath"] ?? "MISSING") refineStage=\(refineStage ?? "nil")"
                 )
             }
-            self.sendEvent(withName: Self.stateUpdateEvent, body: userInfo)
+            guard let bridge = self.bridge else {
+                if hasPath || refineStage != nil {
+                    IncrementalStitcher.fileLog(
+                        "bridge enqueueJSCall DROPPED — self.bridge is nil"
+                    )
+                }
+                return
+            }
+            bridge.enqueueJSCall(
+                "RCTDeviceEventEmitter",
+                method: "emit",
+                args: [Self.stateUpdateEvent, userInfo],
+                completion: nil
+            )
         }
     }
 

package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm

@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// SaveFrameAsJpegPlugin.mm — v0.9.0 Layer 1: vc Frame Processor plugin
+// that JPEG-encodes the supplied frame's pixel buffer to a host-
+// supplied path.  Worklet-callable; thin wrapper around the standard
+// iOS CIImage → CGImage → UIImage → UIImageJPEGRepresentation path.
+//
+// JS-side usage (from a worklet — typically inside `useFrameStream`
+// (Layer 3) or directly from a custom `useFrameProcessor` body):
+//
+//     const plugin = VisionCameraProxy.initFrameProcessorPlugin(
+//       'save_frame_as_jpeg', {},
+//     );
+//
+//     const fp = useFrameProcessor((frame) => {
+//       'worklet';
+//       if (plugin == null) return;
+//       const result = plugin.call(frame, {
+//         path: '/path/to/output.jpg',
+//         quality: 75,  // 0-100; defaults to 75
+//       });
+//       // result: { ok: true, path, width, height }  OR
+//       //         { ok: false, error: "..." }
+//     }, [plugin]);
+//
+// ## Why a separate plugin (not folded into KeyframeGateFrameProcessor)
+//
+// `cv_flow_gate_process_frame` (the existing plugin) drives the lib's
+// FIRST-PARTY stitching pipeline: it consumes the frame, evaluates
+// the keyframe gate, dispatches into `IncrementalStitcher`.  It owns
+// state.
+//
+// `save_frame_as_jpeg` is STATELESS — a pure encode-and-write function.
+// Mixing them would force every JS-side caller of either to pay both
+// codepaths' arg-parsing costs (and would confuse the use-case
+// boundary).  Two plugins, one job each.
+//
+// ## CONDITIONAL COMPILATION
+//
+// Same `__has_include` guard as `KeyframeGateFrameProcessor.mm` — if
+// vision-camera isn't on the host's classpath, this file is a no-op
+// translation unit.  See that file's header for the rationale.
+
+#import <Foundation/Foundation.h>
+
+#if __has_include(<VisionCamera/FrameProcessorPlugin.h>)
+
+#import <VisionCamera/Frame.h>
+#import <VisionCamera/FrameProcessorPlugin.h>
+#import <VisionCamera/FrameProcessorPluginRegistry.h>
+#import <VisionCamera/VisionCameraProxyHolder.h>
+#import <CoreVideo/CoreVideo.h>
+#import <CoreImage/CoreImage.h>
+#import <UIKit/UIKit.h>
+
+@interface SaveFrameAsJpegPlugin : FrameProcessorPlugin
+@end
+
+@implementation SaveFrameAsJpegPlugin
+
+- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy
+                   withOptions:(NSDictionary* _Nullable)options {
+  return [super initWithProxy:proxy withOptions:options];
+}
+
+// Helper: read a string arg with a fallback.  Returns nil only when
+// the arg is missing AND no fallback was supplied.
+static NSString* sfj_argString(NSDictionary* args, NSString* key,
+                                NSString* _Nullable fallback) {
+  id v = args[key];
+  if ([v isKindOfClass:[NSString class]]) return (NSString*)v;
+  return fallback;
+}
+
+// Helper: read a numeric arg (NSNumber or NSString-parseable) with a
+// fallback.  Matches the pattern in KeyframeGateFrameProcessor.mm.
+static double sfj_argDouble(NSDictionary* args, NSString* key,
+                             double fallback) {
+  id v = args[key];
+  if ([v isKindOfClass:[NSNumber class]]) return [(NSNumber*)v doubleValue];
+  if ([v isKindOfClass:[NSString class]]) return [(NSString*)v doubleValue];
+  return fallback;
+}
+
+// The host-callable plugin entry point.  vc dispatches each
+// `plugin.call(frame, args)` from a worklet here.
+//
+// ## Arguments
+//
+//   - `path` (string, REQUIRED): absolute filesystem path to write
+//     the JPEG to.  Parent directory must exist (we don't `mkdir -p`).
+//     Existing file is overwritten atomically.
+//   - `quality` (number, optional): 0-100 JPEG quality.  Default 75
+//     (matches `KeyframeGate.onAccept`'s encoder).  Clamped silently
+//     to `[1, 100]`.
+//
+// ## Returns
+//
+//   - On success: `{ ok: YES, path: <path>, width: <px>, height: <px> }`
+//   - On failure: `{ ok: NO, error: "<reason>" }`
+//
+// Errors are surfaced via the result dict, NOT thrown as `JSError` —
+// host worklets that want to react to encoder failures (e.g., to
+// rotate slot paths, or to back off) can branch on `result.ok`
+// without try/catch boilerplate.  Throwing would break the
+// Layer 3 `useFrameStream` flow which only sees the result.
+- (id)callback:(Frame*)frame withArguments:(NSDictionary*)arguments {
+  NSString* path = sfj_argString(arguments, @"path", nil);
+  if (path == nil) {
+    return @{@"ok": @NO, @"error": @"missing required `path` argument"};
+  }
+  double q = sfj_argDouble(arguments, @"quality", 75.0);
+  if (q < 1.0) q = 1.0;
+  if (q > 100.0) q = 100.0;
+
+  CMSampleBufferRef sampleBuffer = frame.buffer;
+  if (sampleBuffer == NULL) {
+    return @{@"ok": @NO, @"error": @"frame.buffer was NULL"};
+  }
+  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+  if (pixelBuffer == NULL) {
+    return @{@"ok": @NO, @"error": @"CMSampleBufferGetImageBuffer returned NULL"};
+  }
+
+  // CIImage → CGImage → UIImage → JPEG.  Standard iOS path; the
+  // CIContext + colorSpace are cheap to construct per-call (CoreImage
+  // caches GPU resources internally).  If profiling shows this in
+  // the hot path, lift the context to a static; for v0.9.0 baseline,
+  // per-call construction is fine.
+  CIImage* ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
+  if (ciImage == nil) {
+    return @{@"ok": @NO, @"error": @"CIImage imageWithCVPixelBuffer returned nil"};
+  }
+  CIContext* ctx = [CIContext context];
+  CGImageRef cgImage = [ctx createCGImage:ciImage fromRect:ciImage.extent];
+  if (cgImage == NULL) {
+    return @{@"ok": @NO, @"error": @"CIContext createCGImage failed"};
+  }
+  UIImage* uiImage = [UIImage imageWithCGImage:cgImage];
+  size_t width = CGImageGetWidth(cgImage);
+  size_t height = CGImageGetHeight(cgImage);
+  CGImageRelease(cgImage);
+
+  NSData* jpegData = UIImageJPEGRepresentation(uiImage, (CGFloat)(q / 100.0));
+  if (jpegData == nil) {
+    return @{@"ok": @NO, @"error": @"UIImageJPEGRepresentation returned nil"};
+  }
+
+  // Atomic write — under the hood NSData writes to a temp file then
+  // renames.  Avoids torn writes if a reader tries to open the path
+  // mid-write (would otherwise see a partial JPEG and choke).
+  NSError* err = nil;
+  BOOL ok = [jpegData writeToFile:path
+                          options:NSDataWritingAtomic
+                            error:&err];
+  if (!ok) {
+    NSString* msg = err.localizedDescription ?: @"NSData writeToFile returned NO";
+    return @{@"ok": @NO, @"error": msg};
+  }
+
+  return @{
+    @"ok": @YES,
+    @"path": path,
+    @"width": @(width),
+    @"height": @(height),
+  };
+}
+
+// Auto-register the plugin at class-load time.  Name must match what
+// JS passes to `VisionCameraProxy.initFrameProcessorPlugin('save_frame_as_jpeg')`.
+// Same pattern as KeyframeGateFrameProcessor's +load.
++ (void)load {
+  [FrameProcessorPluginRegistry
+    addFrameProcessorPlugin:@"save_frame_as_jpeg"
+            withInitializer:^FrameProcessorPlugin* _Nonnull(
+                VisionCameraProxyHolder* proxy,
+                NSDictionary* _Nullable options) {
+              return [[SaveFrameAsJpegPlugin alloc]
+                  initWithProxy:proxy withOptions:options];
+            }];
+}
+
+@end
+
+#endif // __has_include(<VisionCamera/FrameProcessorPlugin.h>)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-image-stitcher",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/camera/useCapture.ts

@@ -18,7 +18,7 @@
  *   - This hook does NOT persist captures.  Host apps hand the
  *     returned CaptureResult to their own storage layer (WatermelonDB
  *     insert, Redux dispatch, whatever).
- *   - Video recording lives in useVideoCapture (TODO).
+ *   - Video recording lives in useVideoCapture.
  *
  * The public API is designed to be minimal and replaceable: host apps
  * that prefer the raw vision-camera API can opt out of this hook and

package/src/index.ts

@@ -198,6 +198,25 @@ export type {
 // cross-runtime handoff (the AR runtime iterating the registry).
 // See the hook's docstring + StitcherFrame.ts for the contract.
 export { useFrameProcessor } from './stitching/useFrameProcessor';
+// v0.9.0 Layer 2 — `useThrottledFrameProcessor`.  Throttle gate over
+// `useFrameProcessor` for sub-frame-rate worklet-native processing
+// (native OCR via Vision.framework / ML Kit, TFLite ML detection,
+// LiDAR depth).  The worklet runtime has direct access to
+// `frame.toArrayBuffer()` / `frame.arDepth`; bridge small payloads
+// (bboxes, depth-derived metrics) to JS via `runOnJS`.  For JS-thread
+// JPEG consumers (file-path OCR libs, cloud upload, thumbnail UI),
+// prefer `useFrameStream` (Layer 3, ships in the same release).
+export { useThrottledFrameProcessor } from './stitching/useThrottledFrameProcessor';
+export type { ThrottledFrameProcessorOptions } from './types';
+// v0.9.0 Layer 3 — `useFrameStream`.  JS-thread sampled-frame
+// stream over Layer 1 (`save_frame_as_jpeg` vc plugin) + Layer 2
+// (`useThrottledFrameProcessor`).  Use for JS-thread consumers:
+// file-path OCR libs (RN modules), cloud upload, thumbnail UI.
+// For worklet-native processing (Vision/ML Kit as vc plugins,
+// TFLite ML, LiDAR depth), prefer `useThrottledFrameProcessor`
+// (Layer 2) — lower latency, no JPEG roundtrip.
+export { useFrameStream } from './stitching/useFrameStream';
+export type { FrameStreamOptions, SampledFrame } from './types';
 // vision-camera Frame Processor driver for non-AR captures.  As
 // of v0.6 the only non-AR driver exported (the legacy
 // `useIncrementalJSDriver` was removed; was deprecated in v0.5).