react-native-image-stitcher 0.17.0 → 0.19.0

package/src/stitching/{StitcherFrame.ts → CameraFrame.ts}

@@ -4,7 +4,7 @@
  * v0.8.0 — unified frame contract for the lib's worklet processor.
  *
  * Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
- * this directory) receive a `StitcherFrame` regardless of capture mode.
+ * this directory) receive a `CameraFrame` regardless of capture mode.
  * The lib-owned worklet runtime guarantees the same JS-visible shape
  * whether the underlying source is a vision-camera `Frame` (non-AR
  * mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
@@ -18,10 +18,10 @@
  * (Phase-0 audit confirmed the iOS path).  But vision-camera's
  * **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
  * ARCore does NOT produce `ImageProxy` instances.  Forcing
- * `StitcherFrame extends Frame` would either (a) require reverse-
+ * `CameraFrame extends Frame` would either (a) require reverse-
  * engineering ImageProxy on Android (intractable + fragile), or
  * (b) make the type asymmetric per platform.  Both are worse than
- * making `StitcherFrame` a structural sibling type that vc Frames
+ * making `CameraFrame` a structural sibling type that vc Frames
  * happen to satisfy (because vc Frames carry the same width / height /
  * orientation / pixelFormat / timestamp / toArrayBuffer surface).
  *
@@ -38,10 +38,10 @@
  * a JPEG-encode frame-processor plugin).  Returning a reference and
  * reading it later will read into freed memory.
  */
-export interface StitcherFrame {
+export interface CameraFrame {
   // ── vision-camera-shaped fields (structural compat) ─────────────
   // Worklets written against a vc `Frame` work unchanged against a
-  // `StitcherFrame` (the fields below are a strict subset of vc
+  // `CameraFrame` (the fields below are a strict subset of vc
   // Frame's JS-visible surface).
 
   /** Pixel width of the camera image. */
@@ -167,6 +167,28 @@ export interface StitcherFrame {
    * tracking is degraded check this.  Undefined in non-AR mode.
    */
   arTrackingState?: 'notAvailable' | 'limited' | 'normal';
+
+  /**
+   * Camera intrinsics for THIS frame — focal lengths (`fx`,`fy`) and
+   * principal point (`cx`,`cy`) in PIXELS at the `imageWidth × imageHeight`
+   * capture resolution.  Needed to lift 2D image-space coordinates to 3D
+   * via pose + intrinsics (e.g. object-level reconstruction).
+   *
+   * Populated on **AR frames** (`source: 'ar'`) from ARKit
+   * `ARCamera.intrinsics` / ARCore `Camera` intrinsics.  **Undefined for
+   * non-AR (vision-camera) frames** — they are raw vc `Frame`s without an
+   * intrinsics surface; read vc's own APIs there if needed.  (The spec
+   * called this required; it's optional here because the non-AR frame
+   * shape genuinely can't carry it.)
+   */
+  intrinsics?: {
+    fx: number;
+    fy: number;
+    cx: number;
+    cy: number;
+    imageWidth: number;
+    imageHeight: number;
+  };
 }
 
 /**
@@ -177,21 +199,67 @@ export interface StitcherFrame {
 export interface ARAnchor {
   /** Stable per-session anchor identifier. */
   id: string;
-  /** Anchor kind.  `'point'` is Android (ARCore) only. */
-  type: 'plane' | 'image' | 'point';
   /**
-   * 4×4 row-major transform from anchor space to world space.
-   * 16 numbers.
+   * Anchor kind.  `'point'` is Android (ARCore) only; `'mesh'` is a
+   * scene-reconstruction mesh anchor, present only when the `enableMesh`
+   * `<Camera>` prop is on (and the device supports reconstruction).
+   */
+  type: 'plane' | 'image' | 'point' | 'mesh';
+  /**
+   * 4×4 row-major transform from anchor space to world space (16
+   * numbers).  For `'mesh'` anchors, the `meshGeometry.vertices` are in
+   * this anchor's LOCAL space — multiply by `transform` for world coords.
    */
   transform: number[];
+  /**
+   * Plane orientation — `'horizontal'` (floor / table / seat) vs
+   * `'vertical'` (wall / door / window).  Present on `'plane'` anchors;
+   * undefined for other anchor kinds.  Lets a host distinguish a shelf
+   * surface from the wall behind it.
+   */
+  alignment?: 'horizontal' | 'vertical';
+  /**
+   * Plane size in metres along its local x / z axes (`[x, z]`).  Present
+   * on `'plane'` anchors only.
+   */
+  extent?: [number, number];
+  /**
+   * ARKit semantic classification of the plane's surface, when the
+   * framework provides it (iOS; mostly horizontal planes).  Undefined
+   * when unknown / unsupported (incl. Android, which has no equivalent).
+   */
+  classification?:
+    | 'wall'
+    | 'floor'
+    | 'ceiling'
+    | 'table'
+    | 'seat'
+    | 'door'
+    | 'window'
+    | 'none';
+  /**
+   * Scene-reconstruction geometry — present only on `type: 'mesh'`
+   * anchors.  Buffers (wrap in the noted typed-array view):
+   *   - `vertices`  → `Float32Array`, xyz triplets in anchor-local space.
+   *   - `faces`     → `Uint32Array`, triangle indices into `vertices`.
+   *   - `classifications` → optional `Uint8Array`, one ARKit mesh class
+   *     per face (0=none, 1=wall, 2=floor, 3=ceiling, …).  **iOS only**
+   *     (from `ARMeshAnchor`); absent on Android, where the mesh is
+   *     reconstructed from the depth map and carries no semantics.
+   */
+  meshGeometry?: {
+    vertices: ArrayBuffer;
+    faces: ArrayBuffer;
+    classifications?: ArrayBuffer;
+  };
 }
 
 /**
  * v0.8.0 — worklet function signature for the unified frame processor.
  *
  * Must be a `'worklet'`-prefixed function (so it can run on the
- * worklet runtime).  Receives a `StitcherFrame` per camera frame; the
+ * worklet runtime).  Receives a `CameraFrame` per camera frame; the
  * return value is ignored (use `runOnJS` / shared values to surface
  * results back to the JS thread).
  */
-export type StitcherFrameProcessor = (frame: StitcherFrame) => void;
+export type CameraFrameProcessor = (frame: CameraFrame) => void;

package/src/stitching/useStitcherWorklet.ts

@@ -126,18 +126,18 @@ import type {
   FrameProcessorPlugin,
 } from 'react-native-vision-camera';
 
-import type { StitcherFrame } from './StitcherFrame';
+import type { CameraFrame } from './CameraFrame';
 
 /**
  * Frames the lib's stitching worklet accepts.  Accepting either a
  * vc `Frame` (what the host's `useFrameProcessor` body sees) or the
- * lib's `StitcherFrame` (what the lib's `useFrameProcessor` body
+ * lib's `CameraFrame` (what the lib's `useFrameProcessor` body
  * sees) keeps the same `useStitcherWorklet` usable from both kinds
  * of host worklet bodies without a cast on the call site.  The
  * worklet only reads `width` / `height`; the rest of the frame
  * object is forwarded verbatim to the native plugin.
  */
-export type StitcherWorkletInput = Frame | StitcherFrame;
+export type StitcherWorkletInput = Frame | CameraFrame;
 
 
 export interface UseStitcherWorkletOptions {
@@ -173,7 +173,7 @@ export interface UseStitcherWorkletOptions {
 
 export interface StitcherWorkletHandle {
   /**
-   * Worklet function: pass a `StitcherFrame` to perform one frame of
+   * Worklet function: pass a `CameraFrame` to perform one frame of
    * the lib's first-party stitching (throttle + pose synthesis +
    * native plugin call).  Safe to call from inside another
    * `'worklet'`-prefixed function (this is the canonical
@@ -341,7 +341,7 @@ export function useStitcherWorklet(
     // party callback installed in `RNSARWorkletRuntime`).  Calling
     // the vc Frame Processor plugin here would throw
     // `getPropertyAsObject: property '__frame' is undefined`
-    // because AR frames are `StitcherFrameHostObject` instances
+    // because AR frames are `CameraFrameHostObject` instances
     // and don't carry the vc `Frame` proxy's JSI marker.  The
     // throw is caught silently by the per-worklet error handler
     // (`RNSARWorkletRuntime.mm:284-301`) and bubbles up only to
@@ -353,13 +353,13 @@ export function useStitcherWorklet(
     // hook (the AR-side stitching path runs natively, independent
     // of the composed worklet body).
     //
-    // The `(frame as StitcherFrame).source` cast is safe: vc
+    // The `(frame as CameraFrame).source` cast is safe: vc
     // `Frame` doesn't carry a `source` property so the check
     // returns `undefined !== 'ar'` → `true`, and the worklet
     // proceeds normally.  Only frames that explicitly tag
     // themselves as AR-source (which our native AR dispatcher
-    // does — see `StitcherFrameHostObject.mm`) get short-circuited.
-    if ((frame as StitcherFrame).source === 'ar') return;
+    // does — see `CameraFrameHostObject.mm`) get short-circuited.
+    if ((frame as CameraFrame).source === 'ar') return;
 
     // Throttle (verbatim from useFrameProcessorDriver).
     sharedFrameCounter.value += 1;
@@ -388,7 +388,7 @@ export function useStitcherWorklet(
     const fy = h * sharedFyNumerator.value;
 
     // vc's `plugin.call` is typed against vc's `Frame`.  The worklet
-    // accepts the union (`Frame | StitcherFrame`); cast through
+    // accepts the union (`Frame | CameraFrame`); cast through
     // `unknown` because the union doesn't satisfy vc's interface
     // even though structurally both members do.
     plugin.call(frame as unknown as Frame, {

package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h

@@ -1,60 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-//
-// StitcherFrameHostObject.h — Obj-C facade for the v0.8.0
-// `StitcherFrame` JSI host object.  Header is intentionally
-// Obj-C-only (no `<jsi/jsi.h>` import) so this can land in the
-// public CocoaPods umbrella without breaking `use_frameworks!` hosts
-// (same rationale as `KeyframeGateBridge.h`).
-//
-// The C++ JSI host object class lives in the .mm; this facade
-// exposes only what cross-module callers need:
-//
-//   - Factory `+ fromARFrame:pose:` that the AR worklet runtime
-//     calls per ARFrame to construct a host object backed by the
-//     current AR session's frame.
-//   - Opaque accessor `- (void *)jsiHostObjectPtr` returning the
-//     `std::shared_ptr<facebook::jsi::HostObject> *` (boxed) that
-//     the worklet runtime hands to `jsi::Object::createFromHostObject`.
-//
-// Lifetime: the Obj-C wrapper holds the C++ shared_ptr; ARC frees
-// the wrapper when nothing references it.  Worklet runtime
-// invalidates the underlying ARFrame retain when the dispatch
-// returns; after invalidation, JSI access throws.
-
-#pragma once
-
-#import <Foundation/Foundation.h>
-#import <ARKit/ARKit.h>
-
-@class RNSARFramePose;
-
-NS_ASSUME_NONNULL_BEGIN
-
-NS_SWIFT_NAME(StitcherFrameHostObject)
-@interface StitcherFrameHostObject : NSObject
-
-/// Construct a host object backed by the supplied ARFrame + pose.
-/// Retains the ARFrame for the host object's lifetime — caller can
-/// safely release their reference.
-///
-/// Thread: safe to call from the ARSession delegate queue; the
-/// resulting host object's JSI access must happen on the worklet
-/// runtime's thread (separate queue).
-+ (instancetype)fromARFrame:(ARFrame *)arFrame pose:(RNSARFramePose *)pose;
-
-/// Mark the host object's underlying ARFrame as no longer accessible.
-/// Subsequent JSI property reads return `undefined` or throw,
-/// depending on the property.  Idempotent.
-- (void)invalidate;
-
-/// Opaque pointer to a `std::shared_ptr<facebook::jsi::HostObject>`.
-/// The worklet runtime (Obj-C++ context with JSI available) casts
-/// this back via `*reinterpret_cast<std::shared_ptr<facebook::jsi::HostObject>*>(ptr)`
-/// to hand to `jsi::Object::createFromHostObject`.
-///
-/// Returns `NULL` if the host object has been invalidated.
-- (nullable void *)jsiHostObjectPtr;
-
-@end
-
-NS_ASSUME_NONNULL_END

package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm

@@ -1,214 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-//
-// StitcherFrameHostObject.mm — iOS-specific wrapper for the shared
-// `retailens::StitcherFrameJsiHostObject` (defined in
-// `cpp/stitcher_frame_jsi.{hpp,cpp}`).
-//
-// Owns:
-//   - The Obj-C facade callable from Swift / other Obj-C / .mm files.
-//   - The iOS-specific `PixelBufferReader` impl (wraps a
-//     `CVPixelBufferRef` from `ARFrame.capturedImage`; lock / memcpy
-//     / unlock pattern).
-//   - The Obj-C → C++ extraction logic that builds a
-//     `retailens::StitcherFrameData` from an `ARFrame` + the lib's
-//     `RNSARFramePose`.
-//
-// Does NOT own:
-//   - The JSI `get` / `getPropertyNames` dispatch.  That lives in
-//     `cpp/stitcher_frame_jsi.cpp` and is identical to the Android
-//     implementation (DRY across platforms).
-
-#import "StitcherFrameHostObject.h"
-
-#import <Foundation/Foundation.h>
-#import <CoreVideo/CVPixelBuffer.h>
-#import <CoreMedia/CoreMedia.h>
-#import <os/log.h>
-
-#include <jsi/jsi.h>
-
-#include <algorithm>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <utility>
-
-#include "stitcher_frame_data.hpp"
-#include "stitcher_frame_jsi.hpp"
-
-using namespace facebook;
-
-// Forward-declare the Swift `RNSARFramePose` Obj-C surface we need.
-// This matches the pattern in `KeyframeGateFrameProcessor.mm`
-// (forward-declaring `IncrementalStitcher`) — avoids depending on
-// the autogenerated `RNImageStitcher-Swift.h`, which is created at
-// build time and not always available to .mm files in this pod.
-//
-// MUST stay in sync with `RNSARSession.swift::RNSARFramePose` —
-// adding a new field there means adding it here too.
-@class RNSARFramePose;
-@interface RNSARFramePose : NSObject
-@property (nonatomic, readonly) double tx;
-@property (nonatomic, readonly) double ty;
-@property (nonatomic, readonly) double tz;
-@property (nonatomic, readonly) double qx;
-@property (nonatomic, readonly) double qy;
-@property (nonatomic, readonly) double qz;
-@property (nonatomic, readonly) double qw;
-@property (nonatomic, readonly) NSInteger imageWidth;
-@property (nonatomic, readonly) NSInteger imageHeight;
-@property (nonatomic, readonly) double timestampMs;
-@end
-
-#pragma mark - iOS PixelBufferReader
-
-namespace {
-
-/// iOS-specific `retailens::PixelBufferReader` impl.  See the base
-/// class docstring for the general contract (thread-affinity,
-/// invalidation semantics, Y-plane-only constraint).  This subclass
-/// adds:
-///   - `CVPixelBuffer` lock/memcpy/unlock per copyTo
-///   - `CFBridgingRetain` of the parent `ARFrame` so ARKit's
-///     pool can't reclaim the underlying buffer mid-read
-class IOSPixelBufferReader : public retailens::PixelBufferReader {
- public:
-  explicit IOSPixelBufferReader(ARFrame* arFrame) {
-    // Retain the ARFrame for our lifetime.  CFBridgingRetain hands
-    // ARC ownership to our void*.  Released in destructor.
-    _retainedFrame = (void*)CFBridgingRetain(arFrame);
-    CVPixelBufferRef pixelBuffer = arFrame.capturedImage;
-    if (pixelBuffer != NULL) {
-      _bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
-      _height = CVPixelBufferGetHeight(pixelBuffer);
-    }
-  }
-
-  ~IOSPixelBufferReader() override {
-    // Transfer ownership back to ARC, which then releases.
-    if (_retainedFrame != nullptr) {
-      ARFrame* frame = CFBridgingRelease(_retainedFrame);
-      (void)frame;
-      _retainedFrame = nullptr;
-    }
-  }
-
-  std::size_t byteSize() const override {
-    return _bytesPerRow * _height;
-  }
-
-  std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) override {
-    if (_retainedFrame == nullptr) return 0;
-    ARFrame* frame = (__bridge ARFrame*)_retainedFrame;
-    CVPixelBufferRef pixelBuffer = frame.capturedImage;
-    if (pixelBuffer == NULL) return 0;
-
-    CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
-    const uint8_t* src = (const uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
-    std::size_t toCopy = std::min<std::size_t>(byteSize(), maxBytes);
-    if (src != nullptr && toCopy > 0) {
-      std::memcpy(dst, src, toCopy);
-    } else {
-      toCopy = 0;
-    }
-    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
-    return toCopy;
-  }
-
- private:
-  void* _retainedFrame = nullptr;   // CFBridgingRetain'd ARFrame
-  std::size_t _bytesPerRow = 0;
-  std::size_t _height = 0;
-};
-
-}  // anonymous namespace
-
-#pragma mark - Obj-C facade
-
-@implementation StitcherFrameHostObject {
-  std::shared_ptr<retailens::StitcherFrameJsiHostObject> _hostObject;
-}
-
-+ (instancetype)fromARFrame:(ARFrame*)arFrame pose:(RNSARFramePose*)pose {
-  StitcherFrameHostObject* obj = [[self alloc] init];
-
-  retailens::StitcherFrameData data;
-  data.source = "ar";
-  data.width = static_cast<int32_t>(pose.imageWidth);
-  data.height = static_cast<int32_t>(pose.imageHeight);
-  // ARKit's `kCVPixelFormatType_420YpCbCr8BiPlanarFullRange` (NV12)
-  // is reported as "yuv".  Other formats (rare in ARKit; possible if
-  // ARWorldTrackingConfiguration.videoFormat is overridden to BGRA)
-  // → "unknown" + os_log warning so worklets that gate on
-  // `pixelFormat === 'yuv'` can be debugged without a screen recording.
-  OSType pf = CVPixelBufferGetPixelFormatType(arFrame.capturedImage);
-  if (pf == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
-      pf == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
-    data.pixelFormat = "yuv";
-  } else {
-    data.pixelFormat = "unknown";
-    os_log_error(OS_LOG_DEFAULT,
-        "[StitcherFrame] unexpected ARKit pixel format 0x%x; "
-        "worklet receives pixelFormat='unknown' and toArrayBuffer() "
-        "bytes are first-plane only (layout undefined for unknown "
-        "formats).  See StitcherFrame.ts docstring.", (unsigned int)pf);
-  }
-  // ARKit doesn't have a `Frame.orientation` per se; pose carries
-  // the imageWidth >= imageHeight discriminator the lib uses
-  // elsewhere (`isLandscape`).  v0.8.0 ships a coarse mapping;
-  // worklets that need exact UI orientation can read it from
-  // device-orientation sensors.
-  data.orientation =
-      (pose.imageWidth >= pose.imageHeight) ? "landscape-right" : "portrait";
-  // `ARFrame.timestamp` is CFAbsoluteTime (seconds since epoch).
-  // Convert to ns to match vc Frame.timestamp.
-  data.timestampNs = arFrame.timestamp * 1e9;
-
-  data.qx = pose.qx;
-  data.qy = pose.qy;
-  data.qz = pose.qz;
-  data.qw = pose.qw;
-  data.tx = pose.tx;
-  data.ty = pose.ty;
-  data.tz = pose.tz;
-  data.hasTranslation = true;   // AR mode always has translation
-
-  switch (arFrame.camera.trackingState) {
-    case ARTrackingStateNotAvailable:
-      data.arTrackingState = "notAvailable";
-      break;
-    case ARTrackingStateLimited:
-      data.arTrackingState = "limited";
-      break;
-    case ARTrackingStateNormal:
-      data.arTrackingState = "normal";
-      break;
-  }
-
-  data.pixelReader = std::make_shared<IOSPixelBufferReader>(arFrame);
-
-  // Use the static factory (private ctor enforces shared_ptr
-  // ownership — required for `shared_from_this()` inside the JSI
-  // `toArrayBuffer` lambda).
-  obj->_hostObject =
-      retailens::StitcherFrameJsiHostObject::create(std::move(data));
-  return obj;
-}
-
-- (void)invalidate {
-  if (_hostObject) {
-    _hostObject->invalidate();
-  }
-}
-
-- (void*)jsiHostObjectPtr {
-  if (!_hostObject) return NULL;
-  // Box a heap-allocated copy of the shared_ptr to the abstract
-  // `jsi::HostObject` base.  Caller (worklet runtime) does:
-  //   auto sp = static_cast<std::shared_ptr<jsi::HostObject>*>(ptr);
-  //   auto jsObj = jsi::Object::createFromHostObject(rt, *sp);
-  //   delete sp;
-  return new std::shared_ptr<jsi::HostObject>(_hostObject);
-}
-
-@end