react-native-image-stitcher 0.1.0

package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// OpenCVKeyframeCollector — V16 Phase 1 helper that accumulates the
+// AR-keyframe-gate's accepted CVPixelBuffers as on-disk JPEGs while
+// the user pans, then hands the path list off to OpenCVStitcher's
+// `stitchKeyframePaths:withPoses:` on shutter release.
+//
+// Why a separate class:
+//   - CVPixelBuffer → cv::Mat → cv::imwrite has to live in ObjC++ /
+//     OpenCV-aware code.  IncrementalStitcher.swift can't
+//     call it directly.
+//   - The frame collection state (session dir, accepted-frame
+//     counter) is small and capture-scoped; isolating it from the
+//     much larger OpenCVStitcher class file keeps the surface
+//     small.
+//   - When KLT/multi-band incremental work lands later (Phase 3 LHF
+//     #2), this collector becomes the natural seam between the
+//     "frames are arriving live" path and the "stitch them now"
+//     path; centralising it now pays back later.
+
+#import <Foundation/Foundation.h>
+#import <CoreVideo/CoreVideo.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+/// Each saved keyframe ends up with a JPEG path + the index it was
+/// saved at + the on-disk size.  Returned from `saveKeyframe:…` so
+/// the host can build the path/pose list for `stitchKeyframePaths:`.
+@interface OpenCVKeyframeRecord : NSObject
+@property (nonatomic, copy, readonly) NSString *path;
+@property (nonatomic, assign, readonly) NSInteger index;
+@property (nonatomic, assign, readonly) NSInteger width;
+@property (nonatomic, assign, readonly) NSInteger height;
+- (instancetype)initWithPath:(NSString *)path
+                       index:(NSInteger)index
+                       width:(NSInteger)width
+                      height:(NSInteger)height NS_DESIGNATED_INITIALIZER;
+- (instancetype)init NS_UNAVAILABLE;
+@end
+
+
+@interface OpenCVKeyframeCollector : NSObject
+
+/// Active session directory under
+/// `Library/AppSupport/Captures/{capture-uuid}/`.  Persisted (NOT
+/// NSTemporaryDirectory) so the operator / debug menu can re-process
+/// captures later.  Caller is responsible for `cleanup` when no
+/// longer needed.
+@property (nonatomic, copy, readonly) NSString *sessionDir;
+
+/// Total keyframes saved so far.
+@property (nonatomic, assign, readonly) NSInteger acceptedCount;
+
+/// Create a new collector with a freshly-minted session directory
+/// under `Library/AppSupport/Captures/{NSUUID}/`.  Returns nil if
+/// the directory couldn't be created (out-of-space etc.) and
+/// populates `error`.  Imported into Swift as
+/// `try OpenCVKeyframeCollector()`.
+- (nullable instancetype)initWithError:(NSError **)error NS_DESIGNATED_INITIALIZER;
+/// Plain `init` is forwarded to `initWithError:` with a discarded
+/// error so Swift's `try Type()` translation works without colliding
+/// with NS_UNAVAILABLE machinery.  Don't call from ObjC — use the
+/// throwing initializer.
+- (nullable instancetype)init;
+
+/// Save one accepted ARFrame's pixel buffer as a JPEG inside the
+/// session directory.  Filename is `keyframe-{index zero-padded}.jpg`.
+/// Pixel buffer format must be NV12 (the ARFrame default) or BGRA;
+/// other formats fail with NSError code 1200.
+///
+/// `rotationDegrees`: 0/90/180/270.  The buffer is PHYSICALLY
+/// rotated by this amount before encoding.  Use 0 for batch-keyframe
+/// (the stitcher's intrinsics describe the unrotated landscape
+/// sensor; rotating breaks the camera-K-matrix contract).
+///
+/// `exifOrientation`: standard EXIF Orientation tag value (1..8).
+///   1 = no rotation; 6 = 90° CW for display; 3 = 180°; 8 = 90° CCW.
+/// Saved as JPEG metadata via ImageIO.  iOS Image renderers (RN's
+/// `<Image>`, Files.app, Photos) honour this and display the photo
+/// rotated for natural viewing.  cv::imread (when called with
+/// IMREAD_IGNORE_ORIENTATION) returns raw landscape pixels — match
+/// for the stitcher's intrinsics.
+- (nullable OpenCVKeyframeRecord *)saveKeyframe:(CVPixelBufferRef)pixelBuffer
+                                rotationDegrees:(NSInteger)rotationDegrees
+                                exifOrientation:(NSInteger)exifOrientation
+                                    jpegQuality:(NSInteger)jpegQuality
+                                          error:(NSError **)error;
+
+/// Remove the session directory and any saved keyframes.  Idempotent.
+/// Called from IncrementalStitcher's `cancel` / on
+/// successful finalize when the operator hasn't opted into
+/// "keep-for-reprocess" mode.
+- (void)cleanup;
+
+@end
+
+NS_ASSUME_NONNULL_END

package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm

@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#import "OpenCVKeyframeCollector.h"
+#import <ImageIO/ImageIO.h>
+#import <CoreServices/CoreServices.h>
+
+// Same #pragma dance the other ObjC++ stitcher files use to suppress
+// noisy header warnings before importing opencv2.
+#pragma push_macro("NO")
+#undef NO
+#include <opencv2/opencv.hpp>
+#include <opencv2/imgcodecs.hpp>
+#pragma pop_macro("NO")
+
+// V16 Phase 1.fix2 — write a JPEG with an EXIF Orientation tag so
+// iOS image renderers display the saved frame correctly while
+// cv::imread (with IMREAD_IGNORE_ORIENTATION) gets raw landscape
+// pixels for the stitcher.  Returns YES on success.
+static BOOL WriteJPEGWithEXIF(const cv::Mat &bgr,
+                              NSString *path,
+                              NSInteger exifOrientation,
+                              NSInteger quality) {
+  if (bgr.empty()) return NO;
+
+  // Convert BGR (OpenCV native) → RGBA (CoreGraphics expects).
+  cv::Mat rgba;
+  cv::cvtColor(bgr, rgba, cv::COLOR_BGR2RGBA);
+
+  CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
+  CGBitmapInfo bitmapInfo =
+      kCGBitmapByteOrderDefault | kCGImageAlphaNoneSkipLast;
+  CGContextRef ctx = CGBitmapContextCreate(
+      rgba.data,
+      (size_t)rgba.cols,
+      (size_t)rgba.rows,
+      8,
+      (size_t)rgba.step,
+      colorSpace,
+      bitmapInfo);
+  if (!ctx) {
+    CGColorSpaceRelease(colorSpace);
+    return NO;
+  }
+  CGImageRef cgImage = CGBitmapContextCreateImage(ctx);
+  CGContextRelease(ctx);
+  CGColorSpaceRelease(colorSpace);
+  if (!cgImage) return NO;
+
+  NSURL *url = [NSURL fileURLWithPath:path];
+  CGImageDestinationRef dst = CGImageDestinationCreateWithURL(
+      (__bridge CFURLRef)url,
+      // public.jpeg is the stable UTI for JPEG.  Avoiding kUTTypeJPEG
+      // (deprecated) and the iOS-15+ UTType class so this compiles
+      // against older deployment targets too.
+      CFSTR("public.jpeg"),
+      1,
+      NULL);
+  if (!dst) {
+    CGImageRelease(cgImage);
+    return NO;
+  }
+
+  NSInteger q = MAX(0, MIN(100, quality));
+  // Clamp EXIF orientation to the valid range (1..8).  Default to
+  // 1 (no rotation) for unrecognised values.
+  NSInteger exif = (exifOrientation >= 1 && exifOrientation <= 8)
+      ? exifOrientation : 1;
+  NSDictionary *props = @{
+    (id)kCGImageDestinationLossyCompressionQuality: @((double)q / 100.0),
+    (id)kCGImagePropertyOrientation: @(exif),
+  };
+  CGImageDestinationAddImage(
+      dst, cgImage, (__bridge CFDictionaryRef)props);
+  BOOL ok = CGImageDestinationFinalize(dst);
+  CFRelease(dst);
+  CGImageRelease(cgImage);
+  return ok;
+}
+
+
+// ─────────────────────────────────────────────────────────────────────
+// Record
+// ─────────────────────────────────────────────────────────────────────
+
+@implementation OpenCVKeyframeRecord
+- (instancetype)initWithPath:(NSString *)path
+                       index:(NSInteger)index
+                       width:(NSInteger)width
+                      height:(NSInteger)height {
+  if ((self = [super init])) {
+    _path = [path copy];
+    _index = index;
+    _width = width;
+    _height = height;
+  }
+  return self;
+}
+@end
+
+
+// ─────────────────────────────────────────────────────────────────────
+// Collector
+// ─────────────────────────────────────────────────────────────────────
+
+@interface OpenCVKeyframeCollector ()
+@property (nonatomic, copy, readwrite) NSString *sessionDir;
+@property (nonatomic, assign, readwrite) NSInteger acceptedCount;
+@end
+
+
+@implementation OpenCVKeyframeCollector
+
+- (nullable instancetype)init {
+  // Forward to the throwing init with a discarded error.  Used only
+  // when Swift's `try Type()` form chooses the no-error path; calls
+  // from ObjC should always use `initWithError:` directly.
+  return [self initWithError:nil];
+}
+
+- (nullable instancetype)initWithError:(NSError **)error {
+  if ((self = [super init])) {
+    NSURL *appSupport = [[NSFileManager defaultManager]
+        URLForDirectory:NSApplicationSupportDirectory
+               inDomain:NSUserDomainMask
+      appropriateForURL:nil
+                 create:YES
+                  error:error];
+    if (!appSupport) return nil;
+    NSString *captureUUID = [[NSUUID UUID] UUIDString];
+    NSString *sessionPath =
+        [[appSupport.path stringByAppendingPathComponent:@"Captures"]
+                          stringByAppendingPathComponent:captureUUID];
+    BOOL ok = [[NSFileManager defaultManager]
+                createDirectoryAtPath:sessionPath
+          withIntermediateDirectories:YES
+                           attributes:nil
+                                error:error];
+    if (!ok) return nil;
+    _sessionDir = [sessionPath copy];
+    _acceptedCount = 0;
+    NSLog(@"[KeyframeCollector] session dir: %@", _sessionDir);
+  }
+  return self;
+}
+
+- (nullable OpenCVKeyframeRecord *)saveKeyframe:(CVPixelBufferRef)pixelBuffer
+                                rotationDegrees:(NSInteger)rotationDegrees
+                                exifOrientation:(NSInteger)exifOrientation
+                                    jpegQuality:(NSInteger)jpegQuality
+                                          error:(NSError **)error {
+  if (!pixelBuffer) {
+    if (error) {
+      *error = [NSError errorWithDomain:@"OpenCVKeyframeCollector"
+                                   code:1200
+                               userInfo:@{
+        NSLocalizedDescriptionKey: @"Nil pixelBuffer.",
+      }];
+    }
+    return nil;
+  }
+
+  cv::Mat bgr;
+  if (![self convertPixelBuffer:pixelBuffer toMat:bgr]) {
+    if (error) {
+      *error = [NSError errorWithDomain:@"OpenCVKeyframeCollector"
+                                   code:1201
+                               userInfo:@{
+        NSLocalizedDescriptionKey:
+          @"Unsupported pixel-buffer format (need NV12 or BGRA).",
+      }];
+    }
+    return nil;
+  }
+
+  // Rotate to caller-requested orientation.  The JPEGs are saved in
+  // the orientation the stitcher expects (user-pan orientation), so
+  // OpenCVStitcher.stitchKeyframePaths can read them with no further
+  // rotation work.
+  cv::Mat rotated;
+  if (rotationDegrees == 90) {
+    cv::rotate(bgr, rotated, cv::ROTATE_90_CLOCKWISE);
+    bgr.release();
+  } else if (rotationDegrees == 180) {
+    cv::rotate(bgr, rotated, cv::ROTATE_180);
+    bgr.release();
+  } else if (rotationDegrees == 270) {
+    cv::rotate(bgr, rotated, cv::ROTATE_90_COUNTERCLOCKWISE);
+    bgr.release();
+  } else {
+    rotated = bgr;
+  }
+
+  NSInteger idx = self.acceptedCount;
+  NSString *filename =
+      [NSString stringWithFormat:@"keyframe-%03ld.jpg", (long)idx];
+  NSString *fullPath =
+      [self.sessionDir stringByAppendingPathComponent:filename];
+
+  // V16 Phase 1.fix2 — write JPEG via ImageIO so we can set the
+  // EXIF Orientation tag.  cv::imwrite doesn't support EXIF; iOS
+  // image renderers (RN's <Image>, Files.app) need the tag to display
+  // the saved landscape-sensor JPEG correctly when the user is
+  // holding the phone in portrait (which puts the sensor's natural
+  // long axis vertical to user — making un-tagged thumbnails appear
+  // sideways).
+  BOOL wrote = WriteJPEGWithEXIF(rotated, fullPath, exifOrientation,
+                                  jpegQuality);
+  if (!wrote) {
+    if (error) {
+      *error = [NSError errorWithDomain:@"OpenCVKeyframeCollector"
+                                   code:1202
+                               userInfo:@{
+        NSLocalizedDescriptionKey:
+          [NSString stringWithFormat:
+            @"WriteJPEGWithEXIF failed for %@ (orient=%ld q=%ld)",
+            fullPath, (long)exifOrientation, (long)jpegQuality],
+      }];
+    }
+    return nil;
+  }
+
+  self.acceptedCount = idx + 1;
+  NSInteger w = rotated.cols, h = rotated.rows;
+  rotated.release();
+  return [[OpenCVKeyframeRecord alloc] initWithPath:fullPath
+                                              index:idx
+                                              width:w
+                                             height:h];
+}
+
+- (void)cleanup {
+  if (self.sessionDir.length == 0) return;
+  [[NSFileManager defaultManager] removeItemAtPath:self.sessionDir
+                                             error:nil];
+}
+
+// ── CVPixelBuffer → cv::Mat (BGR) ──────────────────────────────────
+//
+// Mirrors `OpenCVIncrementalStitcher.convertPixelBuffer:toMat:` but
+// kept inline here so this file is self-contained.  Supports the two
+// pixel formats ARFrame.capturedImage uses on iOS (NV12 by default;
+// BGRA when the AR session is configured for it).  Lock-once, copy
+// out, unlock — buffer lifetime ends with the caller's accept frame.
+- (BOOL)convertPixelBuffer:(CVPixelBufferRef)pixelBuffer
+                     toMat:(cv::Mat &)outBGR {
+  OSType pf = CVPixelBufferGetPixelFormatType(pixelBuffer);
+  CVReturn lockResult =
+      CVPixelBufferLockBaseAddress(pixelBuffer,
+                                   kCVPixelBufferLock_ReadOnly);
+  if (lockResult != kCVReturnSuccess) return NO;
+
+  BOOL ok = NO;
+  @try {
+    if (pf == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
+        pf == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
+      // NV12 — Y plane (full res) + interleaved CbCr (half res).
+      size_t w = CVPixelBufferGetWidth(pixelBuffer);
+      size_t h = CVPixelBufferGetHeight(pixelBuffer);
+      size_t yStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
+      size_t cStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
+      uint8_t *yPlane =
+          (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
+      uint8_t *cPlane =
+          (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
+
+      cv::Mat nv12((int)(h * 3 / 2), (int)w, CV_8UC1);
+      // Copy Y plane row-by-row (strides may differ from width).
+      for (size_t r = 0; r < h; r++) {
+        memcpy(nv12.ptr((int)r), yPlane + r * yStride, w);
+      }
+      // Copy CbCr (interleaved) — height is h/2.
+      for (size_t r = 0; r < h / 2; r++) {
+        memcpy(nv12.ptr((int)(h + r)), cPlane + r * cStride, w);
+      }
+      cv::cvtColor(nv12, outBGR, cv::COLOR_YUV2BGR_NV12);
+      ok = YES;
+    } else if (pf == kCVPixelFormatType_32BGRA) {
+      size_t w = CVPixelBufferGetWidth(pixelBuffer);
+      size_t h = CVPixelBufferGetHeight(pixelBuffer);
+      size_t stride = CVPixelBufferGetBytesPerRow(pixelBuffer);
+      uint8_t *base = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer);
+      // Wrap (no copy), then convert into outBGR (which IS owned).
+      cv::Mat bgra((int)h, (int)w, CV_8UC4, base, stride);
+      cv::cvtColor(bgra, outBGR, cv::COLOR_BGRA2BGR);
+      ok = YES;
+    }
+  } @catch (NSException *e) {
+    NSLog(@"[KeyframeCollector] convertPixelBuffer exception: %@", e);
+    ok = NO;
+  }
+  CVPixelBufferUnlockBaseAddress(pixelBuffer,
+                                 kCVPixelBufferLock_ReadOnly);
+  return ok;
+}
+
+@end

package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h

@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// OpenCVFirstWinsCylindricalStitcher.h
+//
+// Apple-style slit-scan panorama engine.  Alternative to
+// OpenCVIncrementalStitcher (which is the Samsung-style hybrid
+// frame-based approach).  Both engines expose the same JS-facing
+// API contract; the host picks one via the `engine` flag in start
+// options.
+//
+// What slit-scan does differently:
+//
+//   Hybrid (v9): accepts WHOLE frames at intervals, warps each via
+//   cylindrical projection, blends with feather over a substantial
+//   overlap region (~30-50% of frame width).
+//
+//   Slit-scan (v10): continuously samples the camera buffer.  For
+//   each AR frame, takes a NARROW VERTICAL STRIP (typically 30-60
+//   pixels) whose width tracks the gyro angular delta since the
+//   last strip.  Strips are painted onto the cylindrical canvas at
+//   their exact angular positions, so the per-strip overlap is just
+//   1-3 pixels.  The "stitching" problem mostly disappears because
+//   the overlap region is too narrow to show parallax.
+//
+// Why both engines:
+//
+//   Slit-scan produces near-perfect output for clean rotational
+//   pans (Apple Camera-app quality) but is sensitive to gyro drift
+//   on long pans and to non-rotational motion (operator translates
+//   their body slightly).  Hybrid is more forgiving but has visible
+//   seams where alignment is imperfect.  Field captures decide which
+//   wins for the actual gesture our reps use.
+//
+
+#import <Foundation/Foundation.h>
+#import <CoreVideo/CoreVideo.h>
+#import "OpenCVIncrementalStitcher.h"  // RLISFrameOutcome, RLISFrameTelemetry, RLISSnapshot
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface OpenCVFirstWinsCylindricalStitcher : NSObject
+
+- (instancetype)initWithComposeWidth:(NSInteger)composeWidth
+                       composeHeight:(NSInteger)composeHeight
+                         canvasWidth:(NSInteger)canvasWidth
+                        canvasHeight:(NSInteger)canvasHeight
+                         featherPx:(NSInteger)featherPx
+              frameRotationDegrees:(NSInteger)frameRotationDegrees
+                     useRectilinear:(BOOL)useRectilinear NS_DESIGNATED_INITIALIZER;
+
+- (instancetype)init NS_UNAVAILABLE;
+
+/// V15 — set the per-stage correction config.  Should be called once
+/// after init, before any `ingestPixelBuffer:` call.  Subsequent calls
+/// replace the config but mid-capture changes can produce inconsistent
+/// state (prev features tracked under different settings).  Best
+/// practice: reset the engine when config changes.  If never called,
+/// the engine uses a default equivalent to
+/// `+[RLISStitcherConfig configForMode:@"slitscan-both"]`.
+- (void)setConfig:(RLISStitcherConfig *)config;
+
+/// V15.0b — set the world-frame plane transform (4×4, column-major,
+/// 16 floats).  Should be called once a vertical plane has been
+/// detected by ARKit and before any subsequent ingest.  If never
+/// called, the engine ignores `_config.useDetectedPlane` and falls
+/// back to pose-driven projection.
+- (void)setPlaneTransformFlat:(NSArray<NSNumber *> *)transform16;
+
+- (RLISFrameTelemetry *)ingestPixelBuffer:(CVPixelBufferRef)pixelBuffer
+                                       qx:(double)qx
+                                       qy:(double)qy
+                                       qz:(double)qz
+                                       qw:(double)qw
+                                       tx:(double)tx
+                                       ty:(double)ty
+                                       tz:(double)tz
+                                       fx:(double)fx
+                                       fy:(double)fy
+                                       cx:(double)cx
+                                       cy:(double)cy
+                              imageWidth:(NSInteger)imageWidth
+                             imageHeight:(NSInteger)imageHeight
+                                      yaw:(double)yaw
+                                    pitch:(double)pitch
+                          fovHorizDegrees:(double)fovHorizDegrees
+                           fovVertDegrees:(double)fovVertDegrees
+                             trackingPoor:(BOOL)trackingPoor
+    NS_SWIFT_NAME(ingest(pixelBuffer:qx:qy:qz:qw:tx:ty:tz:fx:fy:cx:cy:imageWidth:imageHeight:yaw:pitch:fovHorizDegrees:fovVertDegrees:trackingPoor:));
+
+- (nullable RLISSnapshot *)snapshotWithJpegQuality:(NSInteger)quality
+                                              error:(NSError **)error;
+
+- (nullable RLISSnapshot *)finalizeAtPath:(NSString *)outputPath
+                              jpegQuality:(NSInteger)quality
+                                    error:(NSError **)error;
+
+- (void)reset;
+
+@property (nonatomic, readonly) NSInteger acceptedCount;
+
+@end
+
+NS_ASSUME_NONNULL_END