react-native-image-stitcher 0.7.1 → 0.9.0

package/cpp/stitcher_worklet_registry.cpp

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// stitcher_worklet_registry.cpp — implementation of the v0.8.0
+// Phase 4b native worklet registry.  See header for the public
+// contract + threading rules.
+
+#include "stitcher_worklet_registry.hpp"
+
+// Cross-platform worklets-core header include.  On iOS the
+// CocoaPods setup publishes worklets-core headers via
+// `HEADER_SEARCH_PATHS` at the root of `Pods/Headers/Public/`,
+// so `<WKTJsiWorklet.h>` works.  On Android the prefab puts
+// headers under a `react-native-worklets-core/` subdirectory of
+// the include path (matches the prefab name).  The angled
+// namespace-prefixed include works on BOTH — `<x/y.h>` resolves
+// to `Pods/Headers/Public/x/y.h` on iOS (CocoaPods auto-creates
+// symlinked subdirs per pod) and to `build/headers/.../x/y.h` on
+// Android.  Pattern lifted from vc's
+// `node_modules/react-native-vision-camera/android/src/main/cpp/`.
+#include <react-native-worklets-core/WKTJsiWorklet.h>
+
+#include <algorithm>
+#include <sstream>
+
+namespace retailens {
+
+StitcherWorkletRegistry& StitcherWorkletRegistry::shared() {
+  static StitcherWorkletRegistry s_instance;
+  return s_instance;
+}
+
+std::string StitcherWorkletRegistry::install(
+    facebook::jsi::Runtime& mainRuntime,
+    const facebook::jsi::Value& workletValue) {
+  // Construct the invoker outside the lock — the WorkletInvoker
+  // constructor calls into worklets-core which acquires its own
+  // locks; nesting our lock around that would invite a deadlock if
+  // worklets-core ever called back into our code synchronously.
+  auto invoker = std::make_shared<RNWorklet::WorkletInvoker>(
+      mainRuntime, workletValue);
+
+  std::lock_guard<std::mutex> lock(_mutex);
+  std::ostringstream idStream;
+  idStream << "host-" << _nextId++;
+  std::string id = idStream.str();
+  _entries.push_back({id, std::move(invoker)});
+  return id;
+}
+
+void StitcherWorkletRegistry::uninstall(const std::string& id) {
+  std::lock_guard<std::mutex> lock(_mutex);
+  // erase-remove with a predicate (single-pass O(n) — n is tiny in
+  // practice, typically 0-3).
+  _entries.erase(
+      std::remove_if(_entries.begin(), _entries.end(),
+                     [&id](const StitcherWorkletEntry& e) {
+                       return e.id == id;
+                     }),
+      _entries.end());
+}
+
+std::vector<StitcherWorkletEntry> StitcherWorkletRegistry::snapshot() {
+  std::lock_guard<std::mutex> lock(_mutex);
+  // Copy the vector — entries hold shared_ptrs so this is O(n)
+  // refcount bumps, no deep copies.  Returning by value lets the
+  // caller iterate without holding the lock.
+  return _entries;
+}
+
+std::size_t StitcherWorkletRegistry::count() {
+  std::lock_guard<std::mutex> lock(_mutex);
+  return _entries.size();
+}
+
+void StitcherWorkletRegistry::_resetForTests() {
+  std::lock_guard<std::mutex> lock(_mutex);
+  _entries.clear();
+  _nextId = 0;
+}
+
+}  // namespace retailens

package/cpp/stitcher_worklet_registry.hpp

@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// stitcher_worklet_registry.hpp — process-scope native registry of
+// host-supplied worklets (v0.8.0 Phase 4b).
+//
+// ## What this is
+//
+// The native-side counterpart to the JS-side `StitcherWorkletRegistry`
+// singleton (`src/stitching/StitcherWorkletRegistry.ts`).  When the
+// public `useFrameProcessor` hook is mounted from JS, it calls into a
+// JSI installable (`globalThis.__stitcherProxy.install(workletFn)`)
+// that wraps the worklet's JSI value into a
+// `RNWorklet::WorkletInvoker` and stores it here.  Unmount calls
+// `__stitcherProxy.uninstall(id)` which removes the entry.
+//
+// The AR worklet runtime's per-frame dispatch (`RNSARWorkletRuntime::
+// dispatchFrame:pose:` on iOS) reads from this registry to fan out
+// invocations across all registered host worklets.  The vc-mode
+// path (non-AR) does NOT touch this registry — vision-camera owns
+// the Frame Processor runtime in that mode and our public hook
+// passes the worklet through to vc unchanged.
+//
+// ## Threading
+//
+// `install` / `uninstall` are called from the main JS thread
+// (`useFrameProcessor`'s `useEffect` body).
+//
+// `snapshot` is called from the AR session callback thread (the
+// caller thread of `RNSARWorkletRuntime::dispatchFrame:pose:`).  The
+// snapshot returns shared_ptrs, so even if `uninstall` races on the
+// JS thread between the snapshot and the worklet runtime's
+// invocation, the WorkletInvoker stays alive until the caller drops
+// the shared_ptr.  WorkletInvoker itself does NOT need its caller
+// to live on a particular thread — the `call` method takes the
+// target `jsi::Runtime&` as an argument, so callers from any
+// thread can invoke it on any runtime they own.
+//
+// Mutation is serialised through `_mutex` (std::mutex).  Reads via
+// `snapshot` lock briefly to copy the entry vector; that's microseconds
+// at most (registry typically has 0-3 entries).  No worklet invocation
+// happens under the lock.
+//
+// ## Lifetime
+//
+// The registry is a `static`-local singleton, constructed on first
+// `shared()` call (function-static init = thread-safe per the C++11
+// memory model).  It outlives every JS / native runtime in the
+// process.  Entries are only added by `install` and only removed by
+// `uninstall` — no GC, no weak refs.  Hosts that bypass the
+// `useFrameProcessor` hook and call `install` directly without ever
+// calling `uninstall` leak entries (matches the JS-side singleton's
+// contract).
+//
+// ## Why a singleton (not per-runtime / per-AR-session)
+//
+// The AR worklet runtime (`RNSARWorkletRuntime` / `StitcherWorkletRuntime`)
+// is itself a process-scope singleton.  The hook lifecycle is
+// per-component-mount.  Registering against a per-AR-session registry
+// would mean re-registering each time AR mode starts — but the host
+// worklet's identity hasn't changed.  Process-scope = "registry
+// matches the host's mental model of `useFrameProcessor` semantics".
+
+#pragma once
+
+#include <jsi/jsi.h>
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <utility>
+#include <vector>
+
+// Forward-declare to avoid pulling the whole worklets-core header
+// into every translation unit that just needs to hold an invoker
+// pointer.  The .cpp includes the full header.
+namespace RNWorklet {
+class WorkletInvoker;
+}
+
+namespace retailens {
+
+/// One registered host worklet.  Public so callers iterating via
+/// `snapshot` can read both the ID and the invoker.
+struct StitcherWorkletEntry {
+  std::string id;
+  std::shared_ptr<RNWorklet::WorkletInvoker> invoker;
+};
+
+class StitcherWorkletRegistry {
+ public:
+  /// Process-scope singleton.  Thread-safe lazy init via C++11
+  /// function-static.
+  static StitcherWorkletRegistry& shared();
+
+  /// Install a host worklet.  Wraps the JSI value (the worklet's
+  /// `'worklet'`-decorated function from the main JS runtime) into
+  /// a `WorkletInvoker` and stores it.  Returns a stable string ID
+  /// the caller passes to `uninstall`.
+  ///
+  /// Thread: must be called from a thread that owns `mainRuntime`
+  /// (typically the main JS thread).  The wrapped `WorkletInvoker`
+  /// can then be invoked from any thread on any runtime via
+  /// `call(rt, ...)`.
+  std::string install(facebook::jsi::Runtime& mainRuntime,
+                       const facebook::jsi::Value& workletValue);
+
+  /// Remove a previously-installed entry by ID.  No-op for unknown
+  /// IDs (matches the JS-side `StitcherWorkletRegistry` semantics).
+  void uninstall(const std::string& id);
+
+  /// Snapshot the current entries.  The returned vector holds
+  /// shared_ptrs to `WorkletInvoker`; mutations against the
+  /// registry after `snapshot` returns do not affect the snapshot.
+  /// Callers iterate without holding the registry lock.
+  std::vector<StitcherWorkletEntry> snapshot();
+
+  /// Current entry count.  Used by `dispatchFrame:` for the
+  /// fast-path early-exit (no fan-out cost when no host worklets
+  /// are registered).
+  std::size_t count();
+
+  /// Test-only — clear all entries.  Used by C++ unit tests; not
+  /// exposed through the JSI surface.
+  void _resetForTests();
+
+ private:
+  StitcherWorkletRegistry() = default;
+  StitcherWorkletRegistry(const StitcherWorkletRegistry&) = delete;
+  StitcherWorkletRegistry& operator=(const StitcherWorkletRegistry&) = delete;
+
+  std::mutex _mutex;
+  std::vector<StitcherWorkletEntry> _entries;
+  int _nextId = 0;
+};
+
+}  // namespace retailens

package/dist/camera/Camera.d.ts

@@ -192,21 +192,71 @@ export interface CameraProps {
     onFramesDropped?: (info: FramesDroppedInfo) => void;
     onError?: (err: CameraError) => void;
     /**
-     * Optional vision-camera frame processor.  Only attached to the
-     * non-AR preview (AR mode uses ARCameraView, which doesn't expose
-     * a worklet seam).  Build the worklet on the host side with
-     * `useFrameProcessor` from `react-native-vision-camera`.
+     * Optional host-supplied vision-camera frame processor.
      *
-     * Introduced for F8 (FrameProcessor port) — see
-     * `docs/f8-frame-processor-plan.md`.
+     * ## When to set this prop
      *
-     * The SDK installs its own frame processor via
-     * `useFrameProcessorDriver`.  Setting this prop is ignored with
-     * a one-time `console.warn` — supplying a host worklet would
-     * race with the SDK's pixel-buffer feed.  Either remove the prop
-     * or fork the SDK if you genuinely need a custom worklet.
+     * v0.8.0+ canonical answer: use the lib's own `useFrameProcessor`
+     * hook, NOT `react-native-vision-camera`'s.  The lib's hook:
      *
-     * AR mode is irrelevant: vision-camera's Camera isn't mounted.
+     *   - **AR mode**: auto-registers the worklet in the native
+     *     `__stitcherProxy` registry; the AR session's per-frame
+     *     dispatch fans out to it alongside the lib's first-party
+     *     stitching.  No prop wiring needed — just mount the hook
+     *     anywhere in the tree.
+     *   - **Non-AR mode**: returns a vc processor object that this
+     *     prop accepts.  Wiring it through enables the host's
+     *     worklet to fire on vc's Frame Processor runtime.
+     *
+     * ```tsx
+     * import { Camera, useFrameProcessor, type StitcherFrame }
+     *   from 'react-native-image-stitcher';
+     *
+     * function MyScreen() {
+     *   const fp = useFrameProcessor((frame: StitcherFrame) => {
+     *     'worklet';
+     *     // ...
+     *   }, []);
+     *   return <Camera frameProcessor={fp} ... />;
+     * }
+     * ```
+     *
+     * ## Non-AR mode tradeoff (HONEST)
+     *
+     * vision-camera's `<Camera>` accepts ONLY ONE frame processor.
+     * The lib's internal `useFrameProcessorDriver` produces the
+     * processor that drives first-party panorama stitching in non-AR
+     * mode.  If you supply your own via this prop, **the lib's
+     * first-party stitching is replaced** — panorama capture in
+     * non-AR mode will not produce stitched output until you remove
+     * the prop or fork the SDK to compose both worklets manually.
+     *
+     * For the common case (host wants worklet + lib wants stitching
+     * concurrently), prefer AR mode: the AR-mode path natively fans
+     * out to both the lib's first-party stitching AND every
+     * registered host worklet on every frame, with per-worklet
+     * failure isolation.
+     *
+     * Composition for non-AR mode (lib stitching + host worklet on
+     * the same vc processor) is tracked as a v0.9+ follow-up;
+     * needs the lib's first-party logic exposed as a vc Frame
+     * Processor plugin the host's worklet can call.
+     *
+     * ## AR mode behaviour
+     *
+     * In AR mode (`defaultCaptureSource="ar"` or runtime-toggled),
+     * vc's `<Camera>` isn't mounted; this prop has no effect.
+     * Host worklets registered via the lib's `useFrameProcessor`
+     * fire automatically through the AR-session dispatch path
+     * (iOS Phase 4b.i / Android Phase 4b.iii).
+     *
+     * ## Backwards compatibility
+     *
+     * The pre-v0.8.0 behaviour (warn + ignore) is preserved when the
+     * supplied processor is recognisably from
+     * `react-native-vision-camera`'s `useFrameProcessor` directly
+     * (no `__stitcherFrame` marker).  Hosts should migrate to the
+     * lib's `useFrameProcessor` to benefit from AR-mode dispatch.
      *
      * (v0.5 had a `legacyDriver` escape hatch that routed back to
      * `useIncrementalJSDriver`.  That hook + prop were removed in

package/dist/camera/Camera.js

@@ -435,25 +435,40 @@ function Camera(props) {
     // Safety: stop the driver if the component unmounts mid-recording.
     // eslint-disable-next-line react-hooks/exhaustive-deps
     (0, react_1.useEffect)(() => () => { fpDriver.stop(); }, []);
-    // One-shot deprecation warning when the host supplies their own
-    // `frameProcessor` prop.  Two worklets racing on the same
-    // producer thread would corrupt the engine's workQueue ordering,
-    // so the SDK's own worklet wins and the host's is silently
-    // ignored.  (v0.5 had a `legacyDriver` opt-out for hosts that
-    // wanted to route around the SDK driver; that was removed in
-    // v0.6 along with `useIncrementalJSDriver`.)
-    const hostFrameProcessorIgnoredWarnedRef = (0, react_1.useRef)(false);
+    // v0.8.0 Phase 5 — frameProcessor prop semantics:
+    //
+    //   - Host supplied? → use host's processor; lib's first-party
+    //     stitching is DISABLED in non-AR mode (vc accepts only one
+    //     processor).  One-shot console.info documents the tradeoff
+    //     so the host isn't surprised by "panorama capture stopped
+    //     producing output" in non-AR mode.  AR-mode capture is
+    //     unaffected — the AR-session dispatch path fans out to BOTH
+    //     first-party and host worklets independently.
+    //
+    //   - No host processor? → use `fpDriver.frameProcessor` which is
+    //     the lib's internal worklet driving first-party stitching
+    //     via `useFrameProcessorDriver`.  Default behaviour for the
+    //     common "I just want panorama capture" case.
+    //
+    // The pre-v0.8.0 behaviour (host's prop silently ignored with
+    // a warning) is gone — Phase 5 plumbs the prop through.  The
+    // tradeoff is honestly documented in the CameraProps docstring.
+    const hostFrameProcessorAcceptedWarnedRef = (0, react_1.useRef)(false);
     if (hostFrameProcessor != null
-        && !hostFrameProcessorIgnoredWarnedRef.current) {
-        hostFrameProcessorIgnoredWarnedRef.current = true;
+        && !hostFrameProcessorAcceptedWarnedRef.current) {
+        hostFrameProcessorAcceptedWarnedRef.current = true;
         // eslint-disable-next-line no-console
-        console.warn('[react-native-image-stitcher] The `frameProcessor` prop on '
-            + '<Camera> is ignored — the SDK installs its own worklet '
-            + 'via useFrameProcessorDriver.  Remove the prop, or fork '
-            + 'the SDK if you genuinely need a custom worklet.');
+        console.info('[react-native-image-stitcher] Host frameProcessor supplied — '
+            + 'non-AR mode will run YOUR worklet instead of the lib\'s '
+            + 'first-party stitching plugin (vc accepts only one frame '
+            + 'processor).  Non-AR panorama capture will not produce '
+            + 'stitched output until this prop is removed.  AR-mode '
+            + 'capture is unaffected (AR-session dispatch fans out to '
+            + 'both first-party and host worklets independently).');
     }
     // The Frame Processor worklet bound to vision-camera's Camera.
-    const effectiveFrameProcessor = fpDriver.frameProcessor;
+    // Host's wins if supplied; lib's internal driver otherwise.
+    const effectiveFrameProcessor = hostFrameProcessor ?? fpDriver.frameProcessor;
     // ── Subscribe to engine state for live keyframe thumbs ──────────
     (0, react_1.useEffect)(() => {
         const sub = (0, incremental_1.subscribeIncrementalState)((state) => {

package/dist/index.d.ts

@@ -65,6 +65,12 @@ export { IncrementalOutcome, incrementalStitcherIsAvailable, subscribeIncrementa
 export type { IncrementalState, AcceptedKeyframe } from './stitching/incremental';
 export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
 export { useKeyframeStream } from './stitching/useKeyframeStream';
+export type { StitcherFrame, StitcherFrameProcessor, ARAnchor, } from './stitching/StitcherFrame';
+export { useFrameProcessor } from './stitching/useFrameProcessor';
+export { useThrottledFrameProcessor } from './stitching/useThrottledFrameProcessor';
+export type { ThrottledFrameProcessorOptions } from './types';
+export { useFrameStream } from './stitching/useFrameStream';
+export type { FrameStreamOptions, SampledFrame } from './types';
 export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
 export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
 export { stitchVideo } from './stitching/stitchVideo';

package/dist/index.js

@@ -22,7 +22,7 @@
  * adds RetaiLens-specific features on top.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.stitchVideo = exports.useFrameProcessorDriver = exports.useKeyframeStream = exports.useIncrementalStitcher = exports.cleanupOldKeyframes = exports.getIncrementalNativeModule = exports.subscribeIncrementalState = exports.incrementalStitcherIsAvailable = exports.IncrementalOutcome = exports.useDeviceOrientation = exports.useVideoCapture = exports.useCapture = exports.ViewportCropOverlay = exports.hybridSettingsToNativeConfig = exports.slitscanSettingsToNativeConfig = exports.panoramaSettingsToNativeConfig = exports.DEFAULT_HYBRID_SETTINGS = exports.DEFAULT_SLITSCAN_SETTINGS = exports.DEFAULT_FLOW_GATE_SETTINGS = exports.DEFAULT_PANORAMA_SETTINGS = exports.PanoramaSettingsModal = exports.PanoramaGuidance = exports.PanoramaBandOverlay = exports.IncrementalPanGuide = exports.CaptureThumbnailStrip = exports.useStitchStatsToast = exports.CaptureStitchStatsToast = exports.CaptureOrientationPill = exports.CaptureKeyframePill = exports.CaptureMemoryPill = exports.CaptureDebugOverlay = exports.CaptureStatusOverlay = exports.CapturePreview = exports.CaptureControlsBar = exports.CaptureHeader = exports.CameraView = exports.ARCameraView = exports.useIMUTranslationGate = exports.ARTrackingState = exports.useARSession = exports.CameraError = exports.Camera = void 0;
+exports.stitchVideo = exports.useFrameProcessorDriver = exports.useFrameStream = exports.useThrottledFrameProcessor = exports.useFrameProcessor = exports.useKeyframeStream = exports.useIncrementalStitcher = exports.cleanupOldKeyframes = exports.getIncrementalNativeModule = exports.subscribeIncrementalState = exports.incrementalStitcherIsAvailable = exports.IncrementalOutcome = exports.useDeviceOrientation = exports.useVideoCapture = exports.useCapture = exports.ViewportCropOverlay = exports.hybridSettingsToNativeConfig = exports.slitscanSettingsToNativeConfig = exports.panoramaSettingsToNativeConfig = exports.DEFAULT_HYBRID_SETTINGS = exports.DEFAULT_SLITSCAN_SETTINGS = exports.DEFAULT_FLOW_GATE_SETTINGS = exports.DEFAULT_PANORAMA_SETTINGS = exports.PanoramaSettingsModal = exports.PanoramaGuidance = exports.PanoramaBandOverlay = exports.IncrementalPanGuide = exports.CaptureThumbnailStrip = exports.useStitchStatsToast = exports.CaptureStitchStatsToast = exports.CaptureOrientationPill = exports.CaptureKeyframePill = exports.CaptureMemoryPill = exports.CaptureDebugOverlay = exports.CaptureStatusOverlay = exports.CapturePreview = exports.CaptureControlsBar = exports.CaptureHeader = exports.CameraView = exports.ARCameraView = exports.useIMUTranslationGate = exports.ARTrackingState = exports.useARSession = exports.CameraError = exports.Camera = void 0;
 // ─────────────────────────────────────────────────────────────────────
 // Layer 1 — the high-level <Camera> component
 // ─────────────────────────────────────────────────────────────────────
@@ -145,6 +145,35 @@ Object.defineProperty(exports, "useIncrementalStitcher", { enumerable: true, get
 // keyframe, packet detection, server-side analysis, etc.).
 var useKeyframeStream_1 = require("./stitching/useKeyframeStream");
 Object.defineProperty(exports, "useKeyframeStream", { enumerable: true, get: function () { return useKeyframeStream_1.useKeyframeStream; } });
+// v0.8.0 Phase 4a — public host-worklet hook.  Hosts that want a
+// per-frame callback (OCR overlay, packet detection, ML inference)
+// use this to attach a `'worklet'`-prefixed function that fires
+// on the camera producer thread.  Non-AR mode is fully wired
+// today via vision-camera passthrough; AR-mode dispatch is
+// API-stable but registration-only until Phase 4b lands the
+// cross-runtime handoff (the AR runtime iterating the registry).
+// See the hook's docstring + StitcherFrame.ts for the contract.
+var useFrameProcessor_1 = require("./stitching/useFrameProcessor");
+Object.defineProperty(exports, "useFrameProcessor", { enumerable: true, get: function () { return useFrameProcessor_1.useFrameProcessor; } });
+// v0.9.0 Layer 2 — `useThrottledFrameProcessor`.  Throttle gate over
+// `useFrameProcessor` for sub-frame-rate worklet-native processing
+// (native OCR via Vision.framework / ML Kit, TFLite ML detection,
+// LiDAR depth).  The worklet runtime has direct access to
+// `frame.toArrayBuffer()` / `frame.arDepth`; bridge small payloads
+// (bboxes, depth-derived metrics) to JS via `runOnJS`.  For JS-thread
+// JPEG consumers (file-path OCR libs, cloud upload, thumbnail UI),
+// prefer `useFrameStream` (Layer 3, ships in the same release).
+var useThrottledFrameProcessor_1 = require("./stitching/useThrottledFrameProcessor");
+Object.defineProperty(exports, "useThrottledFrameProcessor", { enumerable: true, get: function () { return useThrottledFrameProcessor_1.useThrottledFrameProcessor; } });
+// v0.9.0 Layer 3 — `useFrameStream`.  JS-thread sampled-frame
+// stream over Layer 1 (`save_frame_as_jpeg` vc plugin) + Layer 2
+// (`useThrottledFrameProcessor`).  Use for JS-thread consumers:
+// file-path OCR libs (RN modules), cloud upload, thumbnail UI.
+// For worklet-native processing (Vision/ML Kit as vc plugins,
+// TFLite ML, LiDAR depth), prefer `useThrottledFrameProcessor`
+// (Layer 2) — lower latency, no JPEG roundtrip.
+var useFrameStream_1 = require("./stitching/useFrameStream");
+Object.defineProperty(exports, "useFrameStream", { enumerable: true, get: function () { return useFrameStream_1.useFrameStream; } });
 // vision-camera Frame Processor driver for non-AR captures.  As
 // of v0.6 the only non-AR driver exported (the legacy
 // `useIncrementalJSDriver` was removed; was deprecated in v0.5).

package/dist/stitching/StitcherFrame.d.ts

@@ -0,0 +1,170 @@
+/**
+ * v0.8.0 — unified frame contract for the lib's worklet processor.
+ *
+ * Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
+ * this directory) receive a `StitcherFrame` regardless of capture mode.
+ * The lib-owned worklet runtime guarantees the same JS-visible shape
+ * whether the underlying source is a vision-camera `Frame` (non-AR
+ * mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
+ * `Frame` (AR mode, sourced from a lib-managed delegate that the AR
+ * worklet runtime drives).
+ *
+ * ## Why structural (NOT `extends Frame`)
+ *
+ * vision-camera's iOS `Frame` is `CMSampleBufferRef`-shaped; ARFrame's
+ * `capturedImage` (a `CVPixelBufferRef`) can be wrapped into one
+ * (Phase-0 audit confirmed the iOS path).  But vision-camera's
+ * **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
+ * ARCore does NOT produce `ImageProxy` instances.  Forcing
+ * `StitcherFrame extends Frame` would either (a) require reverse-
+ * engineering ImageProxy on Android (intractable + fragile), or
+ * (b) make the type asymmetric per platform.  Both are worse than
+ * making `StitcherFrame` a structural sibling type that vc Frames
+ * happen to satisfy (because vc Frames carry the same width / height /
+ * orientation / pixelFormat / timestamp / toArrayBuffer surface).
+ *
+ * The `__source: 'vc' | 'ar'` discriminator lets worklets gate on
+ * mode without a typeof / try-catch dance — e.g., skip work that
+ * needs AR tracking state when the source is `'vc'`.
+ *
+ * ## Buffer lifetime
+ *
+ * The underlying camera buffer (CMSampleBufferRef / ImageProxy /
+ * ARFrame.capturedImage) is valid only for the duration of the worklet
+ * call.  Worklets that need to retain frame data MUST copy
+ * synchronously inside the worklet body (via `toArrayBuffer()` or via
+ * a JPEG-encode frame-processor plugin).  Returning a reference and
+ * reading it later will read into freed memory.
+ */
+export interface StitcherFrame {
+    /** Pixel width of the camera image. */
+    width: number;
+    /** Pixel height of the camera image. */
+    height: number;
+    /**
+     * Pixel format identifier.  Both modes today emit `'yuv'` (NV12 on
+     * iOS, NV21 on Android).  Other vision-camera formats may appear
+     * in future releases.
+     *
+     * **`'unknown'` semantics:** the lib reached a code path that
+     * doesn't recognise the underlying camera buffer's pixel format
+     * (e.g., a future ARKit version emits BGRA when historically it
+     * only emitted NV12).  Worklets that depend on a known layout
+     * should treat `'unknown'` as "skip this frame".  `toArrayBuffer()`
+     * still returns bytes when the format is `'unknown'`, but the
+     * layout is undefined — the bytes are the underlying buffer's
+     * first plane and may not be interpretable.  When this happens
+     * the native side also emits an `os_log` / logcat warning.
+     */
+    pixelFormat: 'yuv' | 'rgb' | 'unknown';
+    /**
+     * Display orientation tag, matching vision-camera's
+     * `Frame.orientation`.
+     *
+     * **AR-mode limitation (v0.8.0):** AR-source frames return only
+     * the coarse two-value set `'landscape-right' | 'portrait'` (the
+     * lib reads `pose.imageWidth >= pose.imageHeight` as the
+     * discriminator since ARKit's `capturedImage` is always in the
+     * camera's native landscape-right orientation regardless of
+     * device pose).  Worklets that need to distinguish
+     * `landscape-left` (upside-down landscape) or
+     * `portrait-upside-down` should consult device-orientation sensors
+     * separately while running in AR mode.  Non-AR frames (vc source)
+     * return the full four-value set.  Fixing the AR side requires
+     * threading `UIDevice.current.orientation` through; deferred to
+     * v0.8.1+ unless a consumer hits it.
+     */
+    orientation: 'portrait' | 'portrait-upside-down' | 'landscape-left' | 'landscape-right';
+    /**
+     * Monotonic timestamp in **nanoseconds** (matches vision-camera's
+     * `Frame.timestamp` convention).  Use timestamp deltas for
+     * inter-frame timing; the absolute value is implementation-defined
+     * and not comparable to `Date.now()`.
+     */
+    timestamp: number;
+    /**
+     * Copies the underlying pixel buffer into a JSI `ArrayBuffer`.
+     * Worklet-callable.  Allocates O(width × height × bytesPerPixel)
+     * each call — avoid in tight inner loops; prefer plugin-side
+     * processing where possible.
+     */
+    toArrayBuffer(): ArrayBuffer;
+    /**
+     * Camera pose at frame-capture time.  Always present.
+     *
+     * Rotation quaternion order is `(x, y, z, w)`; the lib uses
+     * `q = q_yaw * q_pitch * q_roll` throughout the engine + sensor
+     * fusion.  Same convention surfaced by the v0.7.0
+     * `AcceptedKeyframe.pose` field.
+     *
+     * Translation is metres in world coordinates.  Populated by AR
+     * mode (real ARKit / ARCore camera transform); undefined in
+     * non-AR mode (gyro provides only rotation — no spatial anchor).
+     */
+    pose: {
+        rotation: [number, number, number, number];
+        translation?: [number, number, number];
+    };
+    /**
+     * Discriminator for the frame source.  Worklets branch on this to
+     * gate AR-only field access without try/catch.  Standard TS
+     * discriminated-union pattern.
+     *
+     *   - `'vc'` — vision-camera Frame Processor (non-AR mode)
+     *   - `'ar'` — AR-session frame (AR mode); `arDepth` / `arAnchors` /
+     *     `arTrackingState` fields may be populated
+     */
+    source: 'vc' | 'ar';
+    /**
+     * Depth data when available — AR mode + a device that supports
+     * the AR framework's depth API (iPhone Pro LiDAR; ARCore Depth
+     * API on supported Android devices).
+     *
+     * Resolution is typically lower than the camera image (e.g.,
+     * 256×192 on iPhone Pro LiDAR).  `confidenceMap` is per-pixel:
+     * `0` = low, `1` = medium, `2` = high confidence.  `Float32`
+     * depth in metres; `Uint8` confidence.
+     */
+    arDepth?: {
+        width: number;
+        height: number;
+        depthMap: ArrayBuffer;
+        confidenceMap?: ArrayBuffer;
+    };
+    /**
+     * Tracked AR anchors visible in this frame.  Empty array if AR
+     * is active but no anchors are tracked.  Undefined in non-AR mode.
+     */
+    arAnchors?: ARAnchor[];
+    /**
+     * AR tracking quality.  Worklets that should skip work when
+     * tracking is degraded check this.  Undefined in non-AR mode.
+     */
+    arTrackingState?: 'notAvailable' | 'limited' | 'normal';
+}
+/**
+ * v0.8.0 — public AR anchor type.  Subset of ARKit/ARCore anchor info
+ * exposed to JS worklets.  Extend with plane-extent / image-name
+ * fields as the JSI binding learns them.
+ */
+export interface ARAnchor {
+    /** Stable per-session anchor identifier. */
+    id: string;
+    /** Anchor kind.  `'point'` is Android (ARCore) only. */
+    type: 'plane' | 'image' | 'point';
+    /**
+     * 4×4 row-major transform from anchor space to world space.
+     * 16 numbers.
+     */
+    transform: number[];
+}
+/**
+ * v0.8.0 — worklet function signature for the unified frame processor.
+ *
+ * Must be a `'worklet'`-prefixed function (so it can run on the
+ * worklet runtime).  Receives a `StitcherFrame` per camera frame; the
+ * return value is ignored (use `runOnJS` / shared values to surface
+ * results back to the JS thread).
+ */
+export type StitcherFrameProcessor = (frame: StitcherFrame) => void;
+//# sourceMappingURL=StitcherFrame.d.ts.map

package/dist/stitching/StitcherFrame.js

@@ -0,0 +1,4 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=StitcherFrame.js.map