react-native-image-stitcher 0.16.2 → 0.18.0

package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm

@@ -0,0 +1,760 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// CameraFrameHostObject.mm — iOS-specific wrapper for the shared
+// `retailens::CameraFrameJsiHostObject` (defined in
+// `cpp/camera_frame_jsi.{hpp,cpp}`).
+//
+// Owns:
+//   - The Obj-C facade callable from Swift / other Obj-C / .mm files.
+//   - The iOS-specific `PixelBufferReader` impl (wraps a
+//     `CVPixelBufferRef` from `ARFrame.capturedImage`; lock / memcpy
+//     / unlock pattern).
+//   - The Obj-C → C++ extraction logic that builds a
+//     `retailens::CameraFrameData` from an `ARFrame` + the lib's
+//     `RNSARFramePose`.
+//
+// Does NOT own:
+//   - The JSI `get` / `getPropertyNames` dispatch.  That lives in
+//     `cpp/camera_frame_jsi.cpp` and is identical to the Android
+//     implementation (DRY across platforms).
+
+#import "CameraFrameHostObject.h"
+
+#import <Foundation/Foundation.h>
+#import <ARKit/ARKit.h>
+#import <CoreVideo/CVPixelBuffer.h>
+#import <CoreMedia/CoreMedia.h>
+#import <Metal/Metal.h>
+#import <simd/simd.h>
+#import <os/log.h>
+
+#include <jsi/jsi.h>
+
+#include <algorithm>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "camera_frame_data.hpp"
+#include "camera_frame_jsi.hpp"
+#include "stitcher_proxy_jsi.hpp"   // retailens::getExtractionConfig()
+
+using namespace facebook;
+
+// Forward-declare the Swift `RNSARFramePose` Obj-C surface we need.
+// This matches the pattern in `KeyframeGateFrameProcessor.mm`
+// (forward-declaring `IncrementalStitcher`) — avoids depending on
+// the autogenerated `RNImageStitcher-Swift.h`, which is created at
+// build time and not always available to .mm files in this pod.
+//
+// MUST stay in sync with `RNSARSession.swift::RNSARFramePose` —
+// adding a new field there means adding it here too.
+@class RNSARFramePose;
+@interface RNSARFramePose : NSObject
+@property (nonatomic, readonly) double tx;
+@property (nonatomic, readonly) double ty;
+@property (nonatomic, readonly) double tz;
+@property (nonatomic, readonly) double qx;
+@property (nonatomic, readonly) double qy;
+@property (nonatomic, readonly) double qz;
+@property (nonatomic, readonly) double qw;
+@property (nonatomic, readonly) double fx;
+@property (nonatomic, readonly) double fy;
+@property (nonatomic, readonly) double cx;
+@property (nonatomic, readonly) double cy;
+@property (nonatomic, readonly) NSInteger imageWidth;
+@property (nonatomic, readonly) NSInteger imageHeight;
+@property (nonatomic, readonly) double timestampMs;
+@end
+
+#pragma mark - iOS PixelBufferReader
+
+namespace {
+
+/// iOS-specific `retailens::PixelBufferReader` impl.  See the base
+/// class docstring for the general contract (thread-affinity,
+/// invalidation semantics, Y-plane-only constraint).  This subclass
+/// adds:
+///   - `CVPixelBuffer` lock/memcpy/unlock per copyTo
+///   - `CFBridgingRetain` of the parent `ARFrame` so ARKit's
+///     pool can't reclaim the underlying buffer mid-read
+class IOSPixelBufferReader : public retailens::PixelBufferReader {
+ public:
+  explicit IOSPixelBufferReader(ARFrame* arFrame) {
+    // Retain the ARFrame for our lifetime.  CFBridgingRetain hands
+    // ARC ownership to our void*.  Released in destructor.
+    _retainedFrame = (void*)CFBridgingRetain(arFrame);
+    CVPixelBufferRef pixelBuffer = arFrame.capturedImage;
+    if (pixelBuffer != NULL) {
+      _bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
+      _height = CVPixelBufferGetHeight(pixelBuffer);
+    }
+  }
+
+  ~IOSPixelBufferReader() override {
+    // Transfer ownership back to ARC, which then releases.
+    if (_retainedFrame != nullptr) {
+      ARFrame* frame = CFBridgingRelease(_retainedFrame);
+      (void)frame;
+      _retainedFrame = nullptr;
+    }
+  }
+
+  std::size_t byteSize() const override {
+    return _bytesPerRow * _height;
+  }
+
+  std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) override {
+    if (_retainedFrame == nullptr) return 0;
+    ARFrame* frame = (__bridge ARFrame*)_retainedFrame;
+    CVPixelBufferRef pixelBuffer = frame.capturedImage;
+    if (pixelBuffer == NULL) return 0;
+
+    CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+    const uint8_t* src = (const uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
+    std::size_t toCopy = std::min<std::size_t>(byteSize(), maxBytes);
+    if (src != nullptr && toCopy > 0) {
+      std::memcpy(dst, src, toCopy);
+    } else {
+      toCopy = 0;
+    }
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+    return toCopy;
+  }
+
+ private:
+  void* _retainedFrame = nullptr;   // CFBridgingRetain'd ARFrame
+  std::size_t _bytesPerRow = 0;
+  std::size_t _height = 0;
+};
+
+#pragma mark - AR depth + anchor extraction
+
+/// Copy a single-channel CVPixelBuffer into a TIGHTLY-PACKED byte
+/// vector, stripping any per-row padding.  ARKit's depth/confidence
+/// maps frequently have `bytesPerRow > width * elementSize` (rows are
+/// padded for alignment), so a bulk `memcpy(base, w*h*elemSize)` would
+/// copy garbage padding bytes into the wrong positions and shear the
+/// map.  We copy `width * elementSize` bytes per row from `base +
+/// row * bytesPerRow` so the result is row-packed exactly as the
+/// shared JSI layer (`camera_frame_jsi.cpp`) expects.
+///
+/// Returns `true` on success (out is filled with `w*h*elementSize`
+/// bytes); `false` if the buffer couldn't be locked or has no base
+/// address (out is left untouched).
+bool PackSingleChannelPixelBuffer(CVPixelBufferRef buffer,
+                                  std::size_t elementSize,
+                                  std::vector<uint8_t>& out) {
+  if (buffer == NULL) return false;
+  if (CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly) !=
+      kCVReturnSuccess) {
+    return false;
+  }
+  const std::size_t width = CVPixelBufferGetWidth(buffer);
+  const std::size_t height = CVPixelBufferGetHeight(buffer);
+  const std::size_t bytesPerRow = CVPixelBufferGetBytesPerRow(buffer);
+  const uint8_t* base =
+      reinterpret_cast<const uint8_t*>(CVPixelBufferGetBaseAddress(buffer));
+  const std::size_t rowBytes = width * elementSize;
+
+  bool ok = false;
+  if (base != nullptr && width > 0 && height > 0 && bytesPerRow >= rowBytes) {
+    out.resize(rowBytes * height);
+    for (std::size_t row = 0; row < height; ++row) {
+      std::memcpy(out.data() + row * rowBytes,
+                  base + row * bytesPerRow,
+                  rowBytes);
+    }
+    ok = true;
+  }
+  CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
+  return ok;
+}
+
+/// Extract ARKit `sceneDepth` (preferred) / `smoothedSceneDepth` into
+/// the shared `ArDepth` struct as `format="f32m"`:
+///   - depthBytes    = Float32 metres, row-packed (w*h*4 bytes)
+///   - confidenceBytes = Uint8 ARConfidenceLevel 0..2, row-packed (w*h)
+/// `width`/`height` are the DEPTH MAP's own dimensions (≈256x192),
+/// NOT the camera image's — the JSI layer derives `px = w*h` from
+/// these to validate the byte counts.  Leaves `data.arDepth` as
+/// `nullopt` when the device/session provides no depth (non-LiDAR
+/// devices, or before the first depth frame arrives).
+void ExtractARDepth(ARFrame* arFrame, retailens::CameraFrameData& data) {
+  ARDepthData* dd = arFrame.sceneDepth;
+  if (dd == nil) dd = arFrame.smoothedSceneDepth;
+  if (dd == nil) return;
+
+  CVPixelBufferRef depthMap = dd.depthMap;   // kCVPixelFormatType_DepthFloat32
+  if (depthMap == NULL) return;
+  const int32_t w = static_cast<int32_t>(CVPixelBufferGetWidth(depthMap));
+  const int32_t h = static_cast<int32_t>(CVPixelBufferGetHeight(depthMap));
+  if (w <= 0 || h <= 0) return;
+
+  std::vector<uint8_t> depthBytes;
+  if (!PackSingleChannelPixelBuffer(depthMap, sizeof(float), depthBytes)) {
+    return;
+  }
+
+  // Confidence is optional (some configs/devices omit it).  When
+  // present it's a Uint8 ARConfidenceLevel (0=low,1=medium,2=high),
+  // same w*h dimensions as the depth map.  Leave empty on failure —
+  // the JSI layer treats an empty confidence buffer as "no
+  // confidenceMap" (matching the JS `confidenceMap?` optional).
+  std::vector<uint8_t> confidenceBytes;
+  CVPixelBufferRef conf = dd.confidenceMap;  // kCVPixelFormatType_ConfidenceUint8
+  if (conf != NULL) {
+    if (!PackSingleChannelPixelBuffer(conf, sizeof(uint8_t), confidenceBytes)) {
+      confidenceBytes.clear();
+    }
+  }
+
+  retailens::ArDepth out;
+  out.width = w;
+  out.height = h;
+  out.format = "f32m";
+  out.depthBytes = std::move(depthBytes);
+  out.confidenceBytes = std::move(confidenceBytes);
+  data.arDepth = std::move(out);
+}
+
+/// Map an `ARPlaneClassification` to the JS `ARAnchor.classification`
+/// string union.  Returns `""` for anything unmapped (the JSI then
+/// exposes `classification === undefined`).  Caller gates this on
+/// `classificationStatus == .known` so an undetermined `.none` doesn't
+/// masquerade as a real "none" classification.
+static std::string PlaneClassificationString(ARPlaneClassification c) {
+  switch (c) {
+    case ARPlaneClassificationWall:    return "wall";
+    case ARPlaneClassificationFloor:   return "floor";
+    case ARPlaneClassificationCeiling: return "ceiling";
+    case ARPlaneClassificationTable:   return "table";
+    case ARPlaneClassificationSeat:    return "seat";
+    case ARPlaneClassificationDoor:    return "door";
+    case ARPlaneClassificationWindow:  return "window";
+    case ARPlaneClassificationNone:    return "none";
+    default:                           return "";
+  }
+}
+
+/// Extract the frame's tracked anchors into the shared `ArAnchor`
+/// vector.  Each anchor carries a stable id, a coarse type
+/// (`"plane"` / `"image"` / `"point"`), and a 4x4 anchor->world
+/// transform emitted ROW-MAJOR.  ARKit's `simd_float4x4` is
+/// COLUMN-MAJOR (`columns[c][r]`), so we transpose:
+/// `transform[r*4+c] = a.transform.columns[c][r]`.
+///
+/// Plane anchors additionally carry `alignment` (horizontal/vertical),
+/// `extent` ([x, z] metres), and — on classification-capable devices —
+/// a semantic `classification` (wall/floor/…).
+void ExtractARAnchors(ARFrame* arFrame, retailens::CameraFrameData& data) {
+  NSArray<ARAnchor*>* anchors = arFrame.anchors;
+  data.arAnchors.reserve(anchors.count);
+  for (ARAnchor* a in anchors) {
+    retailens::ArAnchor out;
+    out.id = std::string(a.identifier.UUIDString.UTF8String);
+    if ([a isKindOfClass:[ARPlaneAnchor class]]) {
+      out.type = "plane";
+      ARPlaneAnchor* plane = (ARPlaneAnchor*)a;
+      out.alignment =
+          (plane.alignment == ARPlaneAnchorAlignmentVertical) ? "vertical"
+                                                              : "horizontal";
+      // Deprecated `extent` (simd_float3 x,y,z; y≈0) rather than iOS-16
+      // `planeExtent` — the pod's deployment target still includes
+      // iOS 15, and the Swift plane math (RNSARSession) uses `.extent`
+      // too.  [extentX, extentZ] in plane-local metres.
+      out.hasExtent = true;
+      out.extentX = static_cast<double>(plane.extent.x);
+      out.extentZ = static_cast<double>(plane.extent.z);
+      // Semantic classification only on capable devices AND once ARKit
+      // has actually determined it (`.known`).  Otherwise `classification`
+      // is `.none` merely because it's undetermined — leave empty so JS
+      // sees `undefined`, not a misleading "none".
+      if (ARPlaneAnchor.isClassificationSupported &&
+          plane.classificationStatus == ARPlaneClassificationStatusKnown) {
+        out.classification = PlaneClassificationString(plane.classification);
+      }
+    } else if ([a isKindOfClass:[ARImageAnchor class]]) {
+      out.type = "image";
+    } else {
+      out.type = "point";
+    }
+    const simd_float4x4 m = a.transform;
+    for (int r = 0; r < 4; ++r) {
+      for (int c = 0; c < 4; ++c) {
+        out.transform[r * 4 + c] = static_cast<double>(m.columns[c][r]);
+      }
+    }
+    data.arAnchors.push_back(std::move(out));
+  }
+}
+
+/// Copy the `geometry.vertices` ARGeometrySource (format=float3,
+/// MTLBuffer-backed) into a TIGHTLY-PACKED Float32 xyz byte vector
+/// (count*3 floats).  ARKit reports `offset` (byte offset to the first
+/// element within the buffer) and `stride` (bytes between consecutive
+/// elements).  A simd_float3 is 16 bytes in MSL alignment (xyz + 4
+/// pad) but ARKit may also hand back a 12-byte tight stride — we never
+/// assume, we read `stride` per element and copy exactly 12 bytes
+/// (3×Float32) from `base + offset + i*stride`, dropping any pad.
+/// Returns false (and leaves `out` untouched) if the buffer is
+/// unreadable or the source isn't the expected float3 layout.
+bool PackMeshVertices(ARGeometrySource* src, std::vector<uint8_t>& out) {
+  if (src == nil) return false;
+  id<MTLBuffer> buffer = src.buffer;
+  if (buffer == nil) return false;
+  const NSInteger count = src.count;
+  if (count <= 0) return false;
+  // Vertices must be 3-component Float32 (ARKit's documented layout).
+  if (src.format != MTLVertexFormatFloat3 ||
+      src.componentsPerVector != 3) {
+    return false;
+  }
+  const NSUInteger offset = src.offset;
+  const NSUInteger stride = src.stride;
+  const uint8_t* contents =
+      reinterpret_cast<const uint8_t*>([buffer contents]);
+  if (contents == nullptr) return false;
+  const NSUInteger bufLen = [buffer length];
+  const std::size_t triple = 3 * sizeof(float);  // 12 bytes, tight
+
+  out.resize(static_cast<std::size_t>(count) * triple);
+  for (NSInteger i = 0; i < count; ++i) {
+    const NSUInteger elemOffset = offset + static_cast<NSUInteger>(i) * stride;
+    // Bounds guard — never read past the MTLBuffer.
+    if (elemOffset + triple > bufLen) {
+      out.clear();
+      return false;
+    }
+    std::memcpy(out.data() + static_cast<std::size_t>(i) * triple,
+                contents + elemOffset,
+                triple);
+  }
+  return true;
+}
+
+/// Convert an ARGeometryElement (triangle faces) into a tightly-packed
+/// Uint32 index vector (faces.count * 3 indices).  ARKit's
+/// `bytesPerIndex` is 2 or 4 — we widen 16-bit indices to Uint32 so the
+/// JS side always sees a Uint32Array (matches the cpp/ JSI contract
+/// which emits `meshFaces` verbatim as an ArrayBuffer of Uint32).
+/// Returns false (out untouched) on an unexpected primitive type /
+/// index width, or unreadable buffer.
+bool PackMeshFaces(ARGeometryElement* faces, std::vector<uint8_t>& out) {
+  if (faces == nil) return false;
+  if (faces.primitiveType != ARGeometryPrimitiveTypeTriangle) return false;
+  if (faces.indexCountPerPrimitive != 3) return false;
+  id<MTLBuffer> buffer = faces.buffer;
+  if (buffer == nil) return false;
+  const NSInteger primCount = faces.count;
+  if (primCount <= 0) return false;
+  const NSInteger bytesPerIndex = faces.bytesPerIndex;
+  if (bytesPerIndex != 2 && bytesPerIndex != 4) return false;
+
+  const uint8_t* contents =
+      reinterpret_cast<const uint8_t*>([buffer contents]);
+  if (contents == nullptr) return false;
+  const NSUInteger bufLen = [buffer length];
+
+  const std::size_t totalIndices =
+      static_cast<std::size_t>(primCount) * 3;
+  // Index buffer is tightly packed: count*3 indices of bytesPerIndex.
+  const NSUInteger neededBytes =
+      static_cast<NSUInteger>(totalIndices) *
+      static_cast<NSUInteger>(bytesPerIndex);
+  if (neededBytes > bufLen) return false;
+
+  out.resize(totalIndices * sizeof(uint32_t));
+  uint32_t* dst = reinterpret_cast<uint32_t*>(out.data());
+  if (bytesPerIndex == 4) {
+    // Already Uint32 — bulk copy.
+    std::memcpy(dst, contents, totalIndices * sizeof(uint32_t));
+  } else {
+    // 16-bit → widen each index to 32-bit.
+    const uint16_t* src = reinterpret_cast<const uint16_t*>(contents);
+    for (std::size_t i = 0; i < totalIndices; ++i) {
+      dst[i] = static_cast<uint32_t>(src[i]);
+    }
+  }
+  return true;
+}
+
+/// Copy the optional per-face classification ARGeometrySource (UInt8,
+/// one value per triangle) into a Uint8 byte vector.  ARKit's
+/// classification source is one element per face with format
+/// MTLVertexFormatUChar.  Leaves `out` empty (returns false) when nil
+/// or unreadable — the cpp/ JSI layer treats an empty
+/// `meshClassifications` as "no classifications" (optional in JS).
+bool PackMeshClassifications(ARGeometrySource* src, std::vector<uint8_t>& out) {
+  if (src == nil) return false;
+  id<MTLBuffer> buffer = src.buffer;
+  if (buffer == nil) return false;
+  const NSInteger count = src.count;
+  if (count <= 0) return false;
+  const NSUInteger offset = src.offset;
+  const NSUInteger stride = src.stride;
+  const uint8_t* contents =
+      reinterpret_cast<const uint8_t*>([buffer contents]);
+  if (contents == nullptr) return false;
+  const NSUInteger bufLen = [buffer length];
+
+  out.resize(static_cast<std::size_t>(count));
+  for (NSInteger i = 0; i < count; ++i) {
+    const NSUInteger elemOffset = offset + static_cast<NSUInteger>(i) * stride;
+    if (elemOffset + 1 > bufLen) {
+      out.clear();
+      return false;
+    }
+    out[static_cast<std::size_t>(i)] = contents[elemOffset];
+  }
+  return true;
+}
+
+/// Extract scene-reconstruction mesh anchors (`ARMeshAnchor`) into the
+/// shared `ArAnchor` vector as `type="mesh"` entries.  Each mesh anchor
+/// carries an anchor->world transform (ROW-MAJOR — same transpose as
+/// `ExtractARAnchors`) plus the marshalled `ARMeshGeometry`:
+///   - meshVertices: Float32 xyz triplets (anchor-local), tightly packed.
+///   - meshFaces: Uint32 triangle indices.
+///   - meshClassifications: optional Uint8 per-face class.
+///
+/// ARMeshGeometry's buffers are MTLBuffer-backed but CPU-accessible
+/// (ARKit allocates them with shared storage), so we read `.contents()`
+/// directly on the delegate thread (EAGER copy into owned vectors, like
+/// the depth bytes) — the marshalled `ArAnchor` then has no dependency
+/// on the ARFrame's lifetime.  A mesh anchor whose vertices/faces fail
+/// to marshal is SKIPPED (we never emit a `hasMesh=true` anchor with
+/// empty geometry).
+void ExtractARMesh(ARFrame* arFrame, retailens::CameraFrameData& data) {
+  NSArray<ARAnchor*>* anchors = arFrame.anchors;
+  for (ARAnchor* a in anchors) {
+    if (![a isKindOfClass:[ARMeshAnchor class]]) continue;
+    ARMeshAnchor* meshAnchor = (ARMeshAnchor*)a;
+    ARMeshGeometry* geometry = meshAnchor.geometry;
+    if (geometry == nil) continue;
+
+    std::vector<uint8_t> vertices;
+    if (!PackMeshVertices(geometry.vertices, vertices)) continue;
+    std::vector<uint8_t> faces;
+    if (!PackMeshFaces(geometry.faces, faces)) continue;
+
+    std::vector<uint8_t> classifications;
+    // Optional — leave empty if absent / unreadable.
+    if (geometry.classification != nil) {
+      if (!PackMeshClassifications(geometry.classification, classifications)) {
+        classifications.clear();
+      }
+    }
+
+    retailens::ArAnchor out;
+    out.id = std::string(a.identifier.UUIDString.UTF8String);
+    out.type = "mesh";
+    const simd_float4x4 m = a.transform;
+    for (int r = 0; r < 4; ++r) {
+      for (int c = 0; c < 4; ++c) {
+        out.transform[r * 4 + c] = static_cast<double>(m.columns[c][r]);
+      }
+    }
+    out.hasMesh = true;
+    out.meshVertices = std::move(vertices);
+    out.meshFaces = std::move(faces);
+    out.meshClassifications = std::move(classifications);
+    data.arAnchors.push_back(std::move(out));
+  }
+}
+
+}  // anonymous namespace
+
+#pragma mark - Obj-C facade
+
+@implementation CameraFrameHostObject {
+  std::shared_ptr<retailens::CameraFrameJsiHostObject> _hostObject;
+}
+
++ (instancetype)fromARFrame:(ARFrame*)arFrame pose:(RNSARFramePose*)pose {
+  CameraFrameHostObject* obj = [[self alloc] init];
+
+  retailens::CameraFrameData data;
+  data.source = "ar";
+  data.width = static_cast<int32_t>(pose.imageWidth);
+  data.height = static_cast<int32_t>(pose.imageHeight);
+  // ARKit's `kCVPixelFormatType_420YpCbCr8BiPlanarFullRange` (NV12)
+  // is reported as "yuv".  Other formats (rare in ARKit; possible if
+  // ARWorldTrackingConfiguration.videoFormat is overridden to BGRA)
+  // → "unknown" + os_log warning so worklets that gate on
+  // `pixelFormat === 'yuv'` can be debugged without a screen recording.
+  OSType pf = CVPixelBufferGetPixelFormatType(arFrame.capturedImage);
+  if (pf == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
+      pf == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
+    data.pixelFormat = "yuv";
+  } else {
+    data.pixelFormat = "unknown";
+    os_log_error(OS_LOG_DEFAULT,
+        "[StitcherFrame] unexpected ARKit pixel format 0x%x; "
+        "worklet receives pixelFormat='unknown' and toArrayBuffer() "
+        "bytes are first-plane only (layout undefined for unknown "
+        "formats).  See StitcherFrame.ts docstring.", (unsigned int)pf);
+  }
+  // ARKit doesn't have a `Frame.orientation` per se; pose carries
+  // the imageWidth >= imageHeight discriminator the lib uses
+  // elsewhere (`isLandscape`).  v0.8.0 ships a coarse mapping;
+  // worklets that need exact UI orientation can read it from
+  // device-orientation sensors.
+  data.orientation =
+      (pose.imageWidth >= pose.imageHeight) ? "landscape-right" : "portrait";
+  // `ARFrame.timestamp` is CFAbsoluteTime (seconds since epoch).
+  // Convert to ns to match vc Frame.timestamp.
+  data.timestampNs = arFrame.timestamp * 1e9;
+
+  data.qx = pose.qx;
+  data.qy = pose.qy;
+  data.qz = pose.qz;
+  data.qw = pose.qw;
+  data.tx = pose.tx;
+  data.ty = pose.ty;
+  data.tz = pose.tz;
+  data.hasTranslation = true;   // AR mode always has translation
+
+  // Per-frame camera intrinsics.  `pose` already carries them
+  // (`ARCamera.intrinsics` + `imageResolution`, marshalled in
+  // `RNSARSession.makePose`), so this is six scalars — effectively free.
+  // Always populated for AR frames; the JSI exposes
+  // `intrinsics === undefined` only for non-AR (vc) frames, which have
+  // no intrinsics surface.  NOT gated on the extraction config — it's
+  // too cheap to be worth a toggle, and pose-lift consumers expect it.
+  data.hasIntrinsics = true;
+  data.fx = pose.fx;
+  data.fy = pose.fy;
+  data.cx = pose.cx;
+  data.cy = pose.cy;
+  data.intrinsicsImageWidth = static_cast<int32_t>(pose.imageWidth);
+  data.intrinsicsImageHeight = static_cast<int32_t>(pose.imageHeight);
+
+  switch (arFrame.camera.trackingState) {
+    case ARTrackingStateNotAvailable:
+      data.arTrackingState = "notAvailable";
+      break;
+    case ARTrackingStateLimited:
+      data.arTrackingState = "limited";
+      break;
+    case ARTrackingStateNormal:
+      data.arTrackingState = "normal";
+      break;
+  }
+
+  data.pixelReader = std::make_shared<IOSPixelBufferReader>(arFrame);
+
+  // AR depth + anchors + mesh.  All EAGER-COPY out of the ARFrame here
+  // (depth/confidence bytes and mesh vertex/face/classification bytes
+  // are packed into owned vectors; anchor transforms read into
+  // std::array), so none depend on the ARFrame's lifetime the way the
+  // pixel reader does.  Depth is nullopt on non-LiDAR devices / before
+  // the first depth frame; anchors/mesh are empty when none are tracked.
+  //
+  // GATED on the per-frame extraction config (set from JS via
+  // `__stitcherProxy.setExtractionConfig(depth, anchors, mesh)`,
+  // driven by the <Camera> enableDepth/enableAnchors/enableMesh
+  // props).  Defaults are all-false, so a host that doesn't opt in
+  // pays ZERO arDepth/arAnchors/mesh extraction cost — only the
+  // always-cheap pose/tracking/pixels are populated.  Read the snapshot
+  // once so all three extractors see a consistent config for this frame.
+  const retailens::ExtractionConfig extractionConfig =
+      retailens::getExtractionConfig();
+  if (extractionConfig.depth) {
+    ExtractARDepth(arFrame, data);
+  }
+  if (extractionConfig.anchors) {
+    ExtractARAnchors(arFrame, data);
+  }
+  if (extractionConfig.mesh) {
+    ExtractARMesh(arFrame, data);
+  }
+
+  // Use the static factory (private ctor enforces shared_ptr
+  // ownership — required for `shared_from_this()` inside the JSI
+  // `toArrayBuffer` lambda).
+  obj->_hostObject =
+      retailens::CameraFrameJsiHostObject::create(std::move(data));
+  return obj;
+}
+
++ (NSDictionary *)lightArFrameMetaFromARFrame:(ARFrame *)arFrame
+                                         pose:(RNSARFramePose *)pose {
+  // ── Always-present scalars: timestamp / trackingState / pose ──────────
+  //
+  // timestamp is NANOSECONDS (AR-framework monotonic clock) to match the
+  // ARFrameMeta TS contract + CameraFrame.timestampNs.  ARFrame.timestamp
+  // is CFAbsoluteTime (seconds) → ×1e9.
+  NSMutableDictionary *meta = [NSMutableDictionary dictionary];
+  meta[@"timestamp"] = @(arFrame.timestamp * 1e9);
+
+  NSString *trackingState;
+  switch (arFrame.camera.trackingState) {
+    case ARTrackingStateNotAvailable: trackingState = @"notAvailable"; break;
+    case ARTrackingStateLimited:      trackingState = @"limited";      break;
+    case ARTrackingStateNormal:       trackingState = @"normal";       break;
+    default:                          trackingState = @"notAvailable"; break;
+  }
+  meta[@"trackingState"] = trackingState;
+
+  // pose: quaternion (x,y,z,w) + translation [x,y,z] — straight off the
+  // already-marshalled RNSARFramePose (no re-derivation from the matrix).
+  meta[@"pose"] = @{
+    @"rotation": @[ @(pose.qx), @(pose.qy), @(pose.qz), @(pose.qw) ],
+    @"translation": @[ @(pose.tx), @(pose.ty), @(pose.tz) ],
+  };
+
+  // intrinsics: always attempted; NSNull only when the frame reported a
+  // degenerate (zero) capture resolution (the TS contract's `null`).
+  if (pose.imageWidth > 0 && pose.imageHeight > 0) {
+    meta[@"intrinsics"] = @{
+      @"fx": @(pose.fx),
+      @"fy": @(pose.fy),
+      @"cx": @(pose.cx),
+      @"cy": @(pose.cy),
+      @"imageWidth": @(pose.imageWidth),
+      @"imageHeight": @(pose.imageHeight),
+    };
+  } else {
+    meta[@"intrinsics"] = [NSNull null];
+  }
+
+  // ── Gated, LIGHT fields: depth dims / anchors / mesh counts ───────────
+  //
+  // Same per-frame extraction config the full host-object path reads
+  // (set from JS via __stitcherProxy.setExtractionConfig, driven by the
+  // <Camera> enableDepth/enableAnchors/enableMesh props).  Snapshot once
+  // so all three see a consistent config for this frame.
+  const retailens::ExtractionConfig cfg = retailens::getExtractionConfig();
+
+  // depth: dimensions + whether a confidence channel exists.  NO pixel
+  // copy — just the depth map's own w/h and a confidenceMap != NULL probe.
+  // null when the prop is off OR the device produced no depth this frame.
+  id depthValue = [NSNull null];
+  if (cfg.depth) {
+    ARDepthData *dd = arFrame.sceneDepth;
+    if (dd == nil) dd = arFrame.smoothedSceneDepth;
+    if (dd != nil) {
+      CVPixelBufferRef depthMap = dd.depthMap;
+      if (depthMap != NULL) {
+        const int w = (int)CVPixelBufferGetWidth(depthMap);
+        const int h = (int)CVPixelBufferGetHeight(depthMap);
+        if (w > 0 && h > 0) {
+          depthValue = @{
+            @"width": @(w),
+            @"height": @(h),
+            @"hasConfidence": @(dd.confidenceMap != NULL),
+          };
+        }
+      }
+    }
+  }
+  meta[@"depth"] = depthValue;
+
+  // anchors: id / coarse type / row-major 4x4 transform, plus plane
+  // alignment + extent + (capable-device) classification.  Mirrors
+  // ExtractARAnchors above but into an NSDictionary array (no byte
+  // marshaling — that path is for the full host object).  Empty array
+  // when the prop is off (cheap + JSON-stable, matching the TS contract's
+  // `Array<...>` rather than null).  Mesh anchors are summarised under
+  // `mesh` (counts) below, NOT listed individually here unless enableMesh
+  // is off — to match Android's collectTrackingAnchors which surfaces
+  // plane/image anchors and emits mesh as a separate summary.
+  NSMutableArray *anchorsOut = [NSMutableArray array];
+  if (cfg.anchors) {
+    for (ARAnchor *a in arFrame.anchors) {
+      // ARMeshAnchors are summarised under `mesh`; skip here.
+      if ([a isKindOfClass:[ARMeshAnchor class]]) continue;
+
+      NSMutableDictionary *anchor = [NSMutableDictionary dictionary];
+      anchor[@"id"] = a.identifier.UUIDString;
+
+      if ([a isKindOfClass:[ARPlaneAnchor class]]) {
+        anchor[@"type"] = @"plane";
+        ARPlaneAnchor *plane = (ARPlaneAnchor *)a;
+        anchor[@"alignment"] =
+            (plane.alignment == ARPlaneAnchorAlignmentVertical) ? @"vertical"
+                                                                : @"horizontal";
+        // [extentX, extentZ] in plane-local metres (deprecated `extent`
+        // for iOS-15 parity, same as ExtractARAnchors).
+        anchor[@"extent"] = @[ @(plane.extent.x), @(plane.extent.z) ];
+        if (ARPlaneAnchor.isClassificationSupported &&
+            plane.classificationStatus == ARPlaneClassificationStatusKnown) {
+          std::string cls = PlaneClassificationString(plane.classification);
+          if (!cls.empty()) {
+            anchor[@"classification"] =
+                [NSString stringWithUTF8String:cls.c_str()];
+          }
+        }
+      } else if ([a isKindOfClass:[ARImageAnchor class]]) {
+        anchor[@"type"] = @"image";
+      } else {
+        anchor[@"type"] = @"point";
+      }
+
+      // Row-major anchor->world (transpose ARKit's column-major matrix),
+      // 16 NSNumbers — same transpose as ExtractARAnchors.
+      const simd_float4x4 m = a.transform;
+      NSMutableArray *transform = [NSMutableArray arrayWithCapacity:16];
+      for (int r = 0; r < 4; ++r) {
+        for (int c = 0; c < 4; ++c) {
+          [transform addObject:@((double)m.columns[c][r])];
+        }
+      }
+      anchor[@"transform"] = transform;
+      [anchorsOut addObject:anchor];
+    }
+  }
+  meta[@"anchors"] = anchorsOut;
+
+  // mesh: anchor / vertex / face COUNTS only (no vertex/face byte
+  // marshaling).  null when the prop is off.  Counts are read from each
+  // ARMeshAnchor's geometry sources without touching the MTLBuffer
+  // contents (just `.count` on the vertices source + faces element).
+  id meshValue = [NSNull null];
+  if (cfg.mesh) {
+    int anchorCount = 0;
+    long vertexCount = 0;
+    long faceCount = 0;
+    for (ARAnchor *a in arFrame.anchors) {
+      if (![a isKindOfClass:[ARMeshAnchor class]]) continue;
+      ARMeshAnchor *meshAnchor = (ARMeshAnchor *)a;
+      ARMeshGeometry *geometry = meshAnchor.geometry;
+      if (geometry == nil) continue;
+      anchorCount += 1;
+      if (geometry.vertices != nil) {
+        vertexCount += (long)geometry.vertices.count;
+      }
+      if (geometry.faces != nil) {
+        faceCount += (long)geometry.faces.count;  // triangle (primitive) count
+      }
+    }
+    meshValue = @{
+      @"anchorCount": @(anchorCount),
+      @"vertexCount": @(vertexCount),
+      @"faceCount": @(faceCount),
+    };
+  }
+  meta[@"mesh"] = meshValue;
+
+  return meta;
+}
+
+- (void)invalidate {
+  if (_hostObject) {
+    _hostObject->invalidate();
+  }
+}
+
+- (void*)jsiHostObjectPtr {
+  if (!_hostObject) return NULL;
+  // Box a heap-allocated copy of the shared_ptr to the abstract
+  // `jsi::HostObject` base.  Caller (worklet runtime) does:
+  //   auto sp = static_cast<std::shared_ptr<jsi::HostObject>*>(ptr);
+  //   auto jsObj = jsi::Object::createFromHostObject(rt, *sp);
+  //   delete sp;
+  return new std::shared_ptr<jsi::HostObject>(_hostObject);
+}
+
+@end