react-native-image-stitcher 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +121 -0
  2. package/RNImageStitcher.podspec +1 -1
  3. package/android/src/main/cpp/CMakeLists.txt +4 -4
  4. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +216 -7
  5. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +656 -0
  6. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +156 -0
  7. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +1 -1
  8. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +84 -2
  9. package/cpp/{stitcher_frame_data.hpp → camera_frame_data.hpp} +96 -13
  10. package/cpp/{stitcher_frame_jsi.cpp → camera_frame_jsi.cpp} +154 -11
  11. package/cpp/{stitcher_frame_jsi.hpp → camera_frame_jsi.hpp} +12 -12
  12. package/cpp/stitcher_proxy_jsi.cpp +31 -0
  13. package/cpp/stitcher_proxy_jsi.hpp +16 -0
  14. package/cpp/stitcher_worklet_dispatch.cpp +5 -5
  15. package/cpp/stitcher_worklet_dispatch.hpp +5 -5
  16. package/dist/camera/ARCameraView.d.ts +60 -3
  17. package/dist/camera/ARCameraView.js +68 -1
  18. package/dist/camera/Camera.d.ts +54 -7
  19. package/dist/camera/Camera.js +2 -2
  20. package/dist/index.d.ts +2 -1
  21. package/dist/stitching/ARFrameMeta.d.ts +100 -0
  22. package/dist/stitching/{StitcherFrame.js → ARFrameMeta.js} +1 -1
  23. package/dist/stitching/{StitcherFrame.d.ts → CameraFrame.d.ts} +70 -11
  24. package/dist/stitching/CameraFrame.js +4 -0
  25. package/dist/stitching/useStitcherWorklet.d.ts +4 -4
  26. package/dist/stitching/useStitcherWorklet.js +4 -4
  27. package/ios/Sources/RNImageStitcher/ARSessionBridge.m +23 -1
  28. package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +137 -2
  29. package/ios/Sources/RNImageStitcher/{StitcherFrameHostObject.h → CameraFrameHostObject.h} +26 -3
  30. package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm +760 -0
  31. package/ios/Sources/RNImageStitcher/RNSARSession.swift +292 -34
  32. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +2 -2
  33. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +4 -4
  34. package/package.json +1 -1
  35. package/src/camera/ARCameraView.tsx +165 -5
  36. package/src/camera/Camera.tsx +69 -7
  37. package/src/index.ts +7 -3
  38. package/src/stitching/ARFrameMeta.ts +107 -0
  39. package/src/stitching/{StitcherFrame.ts → CameraFrame.ts} +79 -11
  40. package/src/stitching/useStitcherWorklet.ts +9 -9
  41. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
@@ -41,7 +41,8 @@
41
41
  import React from 'react';
42
42
  import { type StyleProp, type ViewStyle } from 'react-native';
43
43
  import type { DrawableFrameProcessor, ReadonlyFrameProcessor } from 'react-native-vision-camera';
44
- import type { StitcherFrameProcessor } from '../stitching/StitcherFrame';
44
+ import type { CameraFrameProcessor } from '../stitching/CameraFrame';
45
+ import type { ARFrameMeta } from '../stitching/ARFrameMeta';
45
46
  import { type CaptureHeaderProps } from './CaptureHeader';
46
47
  import { type CapturePreviewAction } from './CapturePreview';
47
48
  import { type CaptureThumbnailItem } from './CaptureThumbnailStrip';
@@ -527,11 +528,11 @@ export interface CameraProps {
527
528
  * worklet to fire on vc's Frame Processor runtime.
528
529
  *
529
530
  * ```tsx
530
- * import { Camera, useFrameProcessor, type StitcherFrame }
531
+ * import { Camera, useFrameProcessor, type CameraFrame }
531
532
  * from 'react-native-image-stitcher';
532
533
  *
533
534
  * function MyScreen() {
534
- * const fp = useFrameProcessor((frame: StitcherFrame) => {
535
+ * const fp = useFrameProcessor((frame: CameraFrame) => {
535
536
  * 'worklet';
536
537
  * // ...
537
538
  * }, []);
@@ -552,12 +553,12 @@ export interface CameraProps {
552
553
  * ```tsx
553
554
  * import {
554
555
  * Camera, useFrameProcessor, useStitcherWorklet,
555
- * type StitcherFrame,
556
+ * type CameraFrame,
556
557
  * } from 'react-native-image-stitcher';
557
558
  *
558
559
  * function MyScreen() {
559
560
  * const stitcher = useStitcherWorklet();
560
- * const fp = useFrameProcessor((frame: StitcherFrame) => {
561
+ * const fp = useFrameProcessor((frame: CameraFrame) => {
561
562
  * 'worklet';
562
563
  * hostPreLogic(frame);
563
564
  * stitcher.call(frame); // ← first-party stitching
@@ -601,14 +602,60 @@ export interface CameraProps {
601
602
  /**
602
603
  * AR-mode host worklet, invoked once per ARKit / ARCore frame
603
604
  * ALONGSIDE the lib's first-party stitching (composition, not
604
- * replacement). Receives a `StitcherFrame` tagged `source: 'ar'`
605
+ * replacement). Receives a `CameraFrame` tagged `source: 'ar'`
605
606
  * with world-space `pose` + `arTrackingState`. Only fires in AR
606
607
  * capture (`captureSource === 'ar'`); the non-AR equivalent is
607
608
  * `frameProcessor` above (the two modes use different runtimes and
608
609
  * frame shapes). Must be a `'worklet'`-prefixed function; if the
609
610
  * native install is unavailable it silently never fires.
610
611
  */
611
- arFrameProcessor?: StitcherFrameProcessor;
612
+ arFrameProcessor?: CameraFrameProcessor;
613
+ /**
614
+ * Opt in to per-frame AR depth on the `arFrameProcessor` frame
615
+ * (`CameraFrame.arDepth`). Default `false` — depth is the costliest
616
+ * field (a per-frame buffer copy), so it's off until you need it.
617
+ */
618
+ enableDepth?: boolean;
619
+ /**
620
+ * Opt in to per-frame AR anchors (`CameraFrame.arAnchors` — detected
621
+ * planes / images). Default `false`.
622
+ */
623
+ enableAnchors?: boolean;
624
+ /**
625
+ * Opt in to scene-reconstruction mesh anchors (`type: 'mesh'` in
626
+ * `arAnchors`, with `meshGeometry`). Default `false`. iOS enables
627
+ * ARKit `sceneReconstruction` (LiDAR); Android reconstructs a rough
628
+ * mesh from the depth map. Expensive — only on when needed.
629
+ */
630
+ enableMesh?: boolean;
631
+ /**
632
+ * Which plane orientations to surface in `CameraFrame.arAnchors`
633
+ * (requires `enableAnchors`; AR capture only). Default `'vertical'`
634
+ * — the orientation the plane-projected stitch path has always used.
635
+ * `'horizontal'` surfaces floors / tables; `'both'` surfaces every
636
+ * detected plane. See `ARCameraView` for the per-platform details.
637
+ */
638
+ planeDetection?: 'vertical' | 'horizontal' | 'both';
639
+ /**
640
+ * v0.18.0 — LIGHT per-frame AR metadata callback, invoked on the JS
641
+ * MAIN thread (NOT a worklet). Only fires in AR capture
642
+ * (`captureSource === 'ar'`). Receives an {@link ARFrameMeta} carrying
643
+ * pose, tracking state, intrinsics, and (when the matching `enable*`
644
+ * prop is on) depth dimensions, anchors, and mesh counts.
645
+ *
646
+ * This is the recommended way to read AR metadata: it sidesteps the
647
+ * worklet path entirely (the `arFrameProcessor` worklet can only safely
648
+ * surface a worklets-core shared value, because capturing a host
649
+ * callback crashes the worklet closure-wrap). Native builds the meta
650
+ * and emits a device event; `<Camera>` threads the handler through to
651
+ * `<ARCameraView>`, which subscribes and invokes it on the main thread.
652
+ */
653
+ onArFrame?: (meta: ARFrameMeta) => void;
654
+ /**
655
+ * v0.18.0 — throttle interval (ms) for {@link onArFrame}. Default `100`
656
+ * (≈ 10 Hz). No effect unless `onArFrame` is provided.
657
+ */
658
+ arFrameMetaInterval?: number;
612
659
  /**
613
660
  * Which device holds the non-AR panorama capture accepts.
614
661
  *
@@ -310,7 +310,7 @@ function extractPanoramaOverrides(props) {
310
310
  * The public `<Camera>` component.
311
311
  */
312
312
  function Camera(props) {
313
- const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, engine = 'batch-keyframe',
313
+ const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, enableDepth, enableAnchors, enableMesh, planeDetection, onArFrame, arFrameMetaInterval, engine = 'batch-keyframe',
314
314
  // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
315
315
  panMode = 'vertical', panGuidance = true, maxPanDurationMs = 0, panTooFastThreshold, lateralBudgetCm = 4, rectCrop = false, showPreview = false, guidanceCopy, } = props;
316
316
  // Derived guidance state. The landscape-only gate decision itself is
@@ -1425,7 +1425,7 @@ function Camera(props) {
1425
1425
  // (V12.14.8 OOM fix). The CaptureStatusOverlay renders the
1426
1426
  // "Stitching…" state on top, so no placeholder label is needed
1427
1427
  // in that case — only for the camera-switch transition.
1428
- react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
1428
+ react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor, enableDepth: enableDepth, enableAnchors: enableAnchors, enableMesh: enableMesh, planeDetection: planeDetection, onArFrame: onArFrame, arFrameMetaInterval: arFrameMetaInterval })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
1429
1429
  // `video={true}` is REQUIRED for takeSnapshot to work on iOS.
1430
1430
  // vision-camera v4's iOS implementation of takeSnapshot waits
1431
1431
  // for a frame on the video pipeline; with video disabled, the
package/dist/index.d.ts CHANGED
@@ -91,7 +91,8 @@ export { IncrementalOutcome, incrementalStitcherIsAvailable, subscribeIncrementa
91
91
  export type { IncrementalState, AcceptedKeyframe } from './stitching/incremental';
92
92
  export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
93
93
  export { useKeyframeStream } from './stitching/useKeyframeStream';
94
- export type { StitcherFrame, StitcherFrameProcessor, ARAnchor, } from './stitching/StitcherFrame';
94
+ export type { CameraFrame, CameraFrameProcessor, ARAnchor, } from './stitching/CameraFrame';
95
+ export type { ARFrameMeta } from './stitching/ARFrameMeta';
95
96
  export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
96
97
  export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
97
98
  export { useStitcherWorklet } from './stitching/useStitcherWorklet';
@@ -0,0 +1,100 @@
1
+ /**
2
+ * v0.18.0 — LIGHT per-frame AR metadata delivered to JS on the MAIN
3
+ * thread via a normal callback (`onArFrame`), bypassing worklets entirely.
4
+ *
5
+ * ## Why a callback, NOT a worklet
6
+ *
7
+ * The AR worklet path (`arFrameProcessor` + the `__stitcherProxy` JSI
8
+ * registry) deep-copies the worklet's whole closure into the AR worklet
9
+ * runtime via react-native-worklets-core's `WorkletInvoker`. When the
10
+ * worklet captures a host object (e.g. a `createRunOnJS` callback) that
11
+ * closure-wrap recurses without termination → stack overflow → SIGBUS the
12
+ * instant AR mode mounts (verified on device). Worklets can therefore
13
+ * only safely capture a worklets-core *shared value* — an awkward,
14
+ * poll-from-JS pattern for getting structured data back.
15
+ *
16
+ * `onArFrame` sidesteps the whole problem: native builds the metadata and
17
+ * emits it as a plain `RNImageStitcherARFrame` device event carrying a
18
+ * JSON object; the JS side subscribes via `NativeEventEmitter` and invokes
19
+ * the host callback on the main thread. No worklet, no closure-wrap, no
20
+ * shared-value polling. This is the recommended way to read AR metadata.
21
+ *
22
+ * ## Cost / gating
23
+ *
24
+ * The metadata is intentionally LIGHT — no pixel / vertex / face byte
25
+ * marshaling. `depth` reports only the depth map's dimensions + whether a
26
+ * confidence channel exists (no buffer copy); `mesh` reports only anchor /
27
+ * vertex / face *counts*. Native gates each costly field on the matching
28
+ * extraction flag (`depth` ⇐ `enableDepth`, `mesh` ⇐ `enableMesh`,
29
+ * `anchors` ⇐ `enableAnchors`); `intrinsics` / `pose` / `trackingState` are
30
+ * always present. Emission is gated on a TS-set enabled flag (only true
31
+ * when `onArFrame` is provided) and throttled to `arFrameMetaInterval` ms
32
+ * (default 100 ≈ 10 Hz) on the native side.
33
+ */
34
+ export interface ARFrameMeta {
35
+ /** Frame-capture timestamp in NANOSECONDS (AR-framework monotonic clock). */
36
+ timestamp: number;
37
+ /** AR tracking quality at this frame. */
38
+ trackingState: 'notAvailable' | 'limited' | 'normal';
39
+ /**
40
+ * Camera pose in world coordinates at frame-capture time.
41
+ *
42
+ * - `rotation` — quaternion `(x, y, z, w)`, matching the convention
43
+ * used throughout the engine + the `CameraFrame.pose` field.
44
+ * - `translation` — metres in world space `[x, y, z]`.
45
+ */
46
+ pose: {
47
+ rotation: [number, number, number, number];
48
+ translation: [number, number, number];
49
+ };
50
+ /**
51
+ * Camera intrinsics for this frame — focal lengths (`fx`, `fy`) and
52
+ * principal point (`cx`, `cy`) in PIXELS at the `imageWidth × imageHeight`
53
+ * capture resolution. Always attempted (not gated); `null` only when the
54
+ * AR framework didn't provide them for this frame.
55
+ */
56
+ intrinsics: {
57
+ fx: number;
58
+ fy: number;
59
+ cx: number;
60
+ cy: number;
61
+ imageWidth: number;
62
+ imageHeight: number;
63
+ } | null;
64
+ /**
65
+ * Depth-map summary — dimensions + whether a per-pixel confidence channel
66
+ * is available. NO pixel buffer is copied (that's the costly part).
67
+ * Present only when the `enableDepth` prop is on AND the device produced a
68
+ * depth map this frame; `null` otherwise.
69
+ */
70
+ depth: {
71
+ width: number;
72
+ height: number;
73
+ hasConfidence: boolean;
74
+ } | null;
75
+ /**
76
+ * Tracked AR anchors visible in this frame (planes / images / points /
77
+ * mesh). Empty array when `enableAnchors` is on but nothing is tracked;
78
+ * effectively empty when `enableAnchors` is off. `transform` is a 4×4
79
+ * row-major anchor→world matrix (16 numbers).
80
+ */
81
+ anchors: Array<{
82
+ id: string;
83
+ type: 'plane' | 'image' | 'point' | 'mesh';
84
+ alignment?: 'horizontal' | 'vertical';
85
+ extent?: [number, number];
86
+ classification?: string;
87
+ transform: number[];
88
+ }>;
89
+ /**
90
+ * Scene-reconstruction mesh summary — anchor / vertex / face COUNTS only
91
+ * (no vertex / face byte marshaling). Present only when `enableMesh` is
92
+ * on; `null` otherwise.
93
+ */
94
+ mesh: {
95
+ anchorCount: number;
96
+ vertexCount: number;
97
+ faceCount: number;
98
+ } | null;
99
+ }
100
+ //# sourceMappingURL=ARFrameMeta.d.ts.map
@@ -1,4 +1,4 @@
1
1
  "use strict";
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  Object.defineProperty(exports, "__esModule", { value: true });
4
- //# sourceMappingURL=StitcherFrame.js.map
4
+ //# sourceMappingURL=ARFrameMeta.js.map
@@ -2,7 +2,7 @@
2
2
  * v0.8.0 — unified frame contract for the lib's worklet processor.
3
3
  *
4
4
  * Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
5
- * this directory) receive a `StitcherFrame` regardless of capture mode.
5
+ * this directory) receive a `CameraFrame` regardless of capture mode.
6
6
  * The lib-owned worklet runtime guarantees the same JS-visible shape
7
7
  * whether the underlying source is a vision-camera `Frame` (non-AR
8
8
  * mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
@@ -16,10 +16,10 @@
16
16
  * (Phase-0 audit confirmed the iOS path). But vision-camera's
17
17
  * **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
18
18
  * ARCore does NOT produce `ImageProxy` instances. Forcing
19
- * `StitcherFrame extends Frame` would either (a) require reverse-
19
+ * `CameraFrame extends Frame` would either (a) require reverse-
20
20
  * engineering ImageProxy on Android (intractable + fragile), or
21
21
  * (b) make the type asymmetric per platform. Both are worse than
22
- * making `StitcherFrame` a structural sibling type that vc Frames
22
+ * making `CameraFrame` a structural sibling type that vc Frames
23
23
  * happen to satisfy (because vc Frames carry the same width / height /
24
24
  * orientation / pixelFormat / timestamp / toArrayBuffer surface).
25
25
  *
@@ -36,7 +36,7 @@
36
36
  * a JPEG-encode frame-processor plugin). Returning a reference and
37
37
  * reading it later will read into freed memory.
38
38
  */
39
- export interface StitcherFrame {
39
+ export interface CameraFrame {
40
40
  /** Pixel width of the camera image. */
41
41
  width: number;
42
42
  /** Pixel height of the camera image. */
@@ -141,6 +141,27 @@ export interface StitcherFrame {
141
141
  * tracking is degraded check this. Undefined in non-AR mode.
142
142
  */
143
143
  arTrackingState?: 'notAvailable' | 'limited' | 'normal';
144
+ /**
145
+ * Camera intrinsics for THIS frame — focal lengths (`fx`,`fy`) and
146
+ * principal point (`cx`,`cy`) in PIXELS at the `imageWidth × imageHeight`
147
+ * capture resolution. Needed to lift 2D image-space coordinates to 3D
148
+ * via pose + intrinsics (e.g. object-level reconstruction).
149
+ *
150
+ * Populated on **AR frames** (`source: 'ar'`) from ARKit
151
+ * `ARCamera.intrinsics` / ARCore `Camera` intrinsics. **Undefined for
152
+ * non-AR (vision-camera) frames** — they are raw vc `Frame`s without an
153
+ * intrinsics surface; read vc's own APIs there if needed. (The spec
154
+ * called this required; it's optional here because the non-AR frame
155
+ * shape genuinely can't carry it.)
156
+ */
157
+ intrinsics?: {
158
+ fx: number;
159
+ fy: number;
160
+ cx: number;
161
+ cy: number;
162
+ imageWidth: number;
163
+ imageHeight: number;
164
+ };
144
165
  }
145
166
  /**
146
167
  * v0.8.0 — public AR anchor type. Subset of ARKit/ARCore anchor info
@@ -150,21 +171,59 @@ export interface StitcherFrame {
150
171
  export interface ARAnchor {
151
172
  /** Stable per-session anchor identifier. */
152
173
  id: string;
153
- /** Anchor kind. `'point'` is Android (ARCore) only. */
154
- type: 'plane' | 'image' | 'point';
155
174
  /**
156
- * 4×4 row-major transform from anchor space to world space.
157
- * 16 numbers.
175
+ * Anchor kind. `'point'` is Android (ARCore) only; `'mesh'` is a
176
+ * scene-reconstruction mesh anchor, present only when the `enableMesh`
177
+ * `<Camera>` prop is on (and the device supports reconstruction).
178
+ */
179
+ type: 'plane' | 'image' | 'point' | 'mesh';
180
+ /**
181
+ * 4×4 row-major transform from anchor space to world space (16
182
+ * numbers). For `'mesh'` anchors, the `meshGeometry.vertices` are in
183
+ * this anchor's LOCAL space — multiply by `transform` for world coords.
158
184
  */
159
185
  transform: number[];
186
+ /**
187
+ * Plane orientation — `'horizontal'` (floor / table / seat) vs
188
+ * `'vertical'` (wall / door / window). Present on `'plane'` anchors;
189
+ * undefined for other anchor kinds. Lets a host distinguish a shelf
190
+ * surface from the wall behind it.
191
+ */
192
+ alignment?: 'horizontal' | 'vertical';
193
+ /**
194
+ * Plane size in metres along its local x / z axes (`[x, z]`). Present
195
+ * on `'plane'` anchors only.
196
+ */
197
+ extent?: [number, number];
198
+ /**
199
+ * ARKit semantic classification of the plane's surface, when the
200
+ * framework provides it (iOS; mostly horizontal planes). Undefined
201
+ * when unknown / unsupported (incl. Android, which has no equivalent).
202
+ */
203
+ classification?: 'wall' | 'floor' | 'ceiling' | 'table' | 'seat' | 'door' | 'window' | 'none';
204
+ /**
205
+ * Scene-reconstruction geometry — present only on `type: 'mesh'`
206
+ * anchors. Buffers (wrap in the noted typed-array view):
207
+ * - `vertices` → `Float32Array`, xyz triplets in anchor-local space.
208
+ * - `faces` → `Uint32Array`, triangle indices into `vertices`.
209
+ * - `classifications` → optional `Uint8Array`, one ARKit mesh class
210
+ * per face (0=none, 1=wall, 2=floor, 3=ceiling, …). **iOS only**
211
+ * (from `ARMeshAnchor`); absent on Android, where the mesh is
212
+ * reconstructed from the depth map and carries no semantics.
213
+ */
214
+ meshGeometry?: {
215
+ vertices: ArrayBuffer;
216
+ faces: ArrayBuffer;
217
+ classifications?: ArrayBuffer;
218
+ };
160
219
  }
161
220
  /**
162
221
  * v0.8.0 — worklet function signature for the unified frame processor.
163
222
  *
164
223
  * Must be a `'worklet'`-prefixed function (so it can run on the
165
- * worklet runtime). Receives a `StitcherFrame` per camera frame; the
224
+ * worklet runtime). Receives a `CameraFrame` per camera frame; the
166
225
  * return value is ignored (use `runOnJS` / shared values to surface
167
226
  * results back to the JS thread).
168
227
  */
169
- export type StitcherFrameProcessor = (frame: StitcherFrame) => void;
170
- //# sourceMappingURL=StitcherFrame.d.ts.map
228
+ export type CameraFrameProcessor = (frame: CameraFrame) => void;
229
+ //# sourceMappingURL=CameraFrame.d.ts.map
@@ -0,0 +1,4 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ //# sourceMappingURL=CameraFrame.js.map
@@ -106,17 +106,17 @@
106
106
  * their own worklet via this hook must do the wiring themselves.
107
107
  */
108
108
  import type { Frame } from 'react-native-vision-camera';
109
- import type { StitcherFrame } from './StitcherFrame';
109
+ import type { CameraFrame } from './CameraFrame';
110
110
  /**
111
111
  * Frames the lib's stitching worklet accepts. Accepting either a
112
112
  * vc `Frame` (what the host's `useFrameProcessor` body sees) or the
113
- * lib's `StitcherFrame` (what the lib's `useFrameProcessor` body
113
+ * lib's `CameraFrame` (what the lib's `useFrameProcessor` body
114
114
  * sees) keeps the same `useStitcherWorklet` usable from both kinds
115
115
  * of host worklet bodies without a cast on the call site. The
116
116
  * worklet only reads `width` / `height`; the rest of the frame
117
117
  * object is forwarded verbatim to the native plugin.
118
118
  */
119
- export type StitcherWorkletInput = Frame | StitcherFrame;
119
+ export type StitcherWorkletInput = Frame | CameraFrame;
120
120
  export interface UseStitcherWorkletOptions {
121
121
  /**
122
122
  * Gyro sample interval in ms (~30 Hz default). Drives the JS-
@@ -145,7 +145,7 @@ export interface UseStitcherWorkletOptions {
145
145
  }
146
146
  export interface StitcherWorkletHandle {
147
147
  /**
148
- * Worklet function: pass a `StitcherFrame` to perform one frame of
148
+ * Worklet function: pass a `CameraFrame` to perform one frame of
149
149
  * the lib's first-party stitching (throttle + pose synthesis +
150
150
  * native plugin call). Safe to call from inside another
151
151
  * `'worklet'`-prefixed function (this is the canonical
@@ -228,7 +228,7 @@ function useStitcherWorklet(options = {}) {
228
228
  // party callback installed in `RNSARWorkletRuntime`). Calling
229
229
  // the vc Frame Processor plugin here would throw
230
230
  // `getPropertyAsObject: property '__frame' is undefined`
231
- // because AR frames are `StitcherFrameHostObject` instances
231
+ // because AR frames are `CameraFrameHostObject` instances
232
232
  // and don't carry the vc `Frame` proxy's JSI marker. The
233
233
  // throw is caught silently by the per-worklet error handler
234
234
  // (`RNSARWorkletRuntime.mm:284-301`) and bubbles up only to
@@ -240,12 +240,12 @@ function useStitcherWorklet(options = {}) {
240
240
  // hook (the AR-side stitching path runs natively, independent
241
241
  // of the composed worklet body).
242
242
  //
243
- // The `(frame as StitcherFrame).source` cast is safe: vc
243
+ // The `(frame as CameraFrame).source` cast is safe: vc
244
244
  // `Frame` doesn't carry a `source` property so the check
245
245
  // returns `undefined !== 'ar'` → `true`, and the worklet
246
246
  // proceeds normally. Only frames that explicitly tag
247
247
  // themselves as AR-source (which our native AR dispatcher
248
- // does — see `StitcherFrameHostObject.mm`) get short-circuited.
248
+ // does — see `CameraFrameHostObject.mm`) get short-circuited.
249
249
  if (frame.source === 'ar')
250
250
  return;
251
251
  // Throttle (verbatim from useFrameProcessorDriver).
@@ -273,7 +273,7 @@ function useStitcherWorklet(options = {}) {
273
273
  const fx = w * sharedFxNumerator.value;
274
274
  const fy = h * sharedFyNumerator.value;
275
275
  // vc's `plugin.call` is typed against vc's `Frame`. The worklet
276
- // accepts the union (`Frame | StitcherFrame`); cast through
276
+ // accepts the union (`Frame | CameraFrame`); cast through
277
277
  // `unknown` because the union doesn't satisfy vc's interface
278
278
  // even though structurally both members do.
279
279
  plugin.call(frame, {
@@ -5,6 +5,7 @@
5
5
  // imports as `NativeModules.RNSARSession`.
6
6
 
7
7
  #import <React/RCTBridgeModule.h>
8
+ #import <React/RCTEventEmitter.h>
8
9
 
9
10
  // REMAP form, NOT EXTERN_MODULE. The Swift singleton in
10
11
  // RNSARSession.swift takes the @objc name "RNSARSession"
@@ -20,7 +21,14 @@
20
21
  // `RNSARSessionBridge` and dispatch methods against THAT
21
22
  // class — where takePhoto / startRecording / stopRecording etc.
22
23
  // actually live.
23
- @interface RCT_EXTERN_REMAP_MODULE(RNSARSession, RNSARSessionBridge, NSObject)
24
+ //
25
+ // v0.18.0 — base class is now `RCTEventEmitter` (was `NSObject`) so the
26
+ // "RNSARSession" module can emit the `RNImageStitcherARFrame` device
27
+ // event for the `onArFrame` channel. RN auto-provides the
28
+ // `addListener:` / `removeListeners:` emitter selectors for a module
29
+ // whose remap base is RCTEventEmitter; the Swift class supplies
30
+ // `supportedEvents` / `startObserving` / `stopObserving`.
31
+ @interface RCT_EXTERN_REMAP_MODULE(RNSARSession, RNSARSessionBridge, RCTEventEmitter)
24
32
 
25
33
  RCT_EXTERN_METHOD(isSupported:(RCTPromiseResolveBlock)resolver
26
34
  rejecter:(RCTPromiseRejectBlock)rejecter)
@@ -34,6 +42,20 @@ RCT_EXTERN_METHOD(stop:(RCTPromiseResolveBlock)resolver
34
42
  RCT_EXTERN_METHOD(getState:(RCTPromiseResolveBlock)resolver
35
43
  rejecter:(RCTPromiseRejectBlock)rejecter)
36
44
 
45
+ RCT_EXTERN_METHOD(setSceneReconstructionEnabled:(nonnull NSNumber *)enabled
46
+ resolver:(RCTPromiseResolveBlock)resolver
47
+ rejecter:(RCTPromiseRejectBlock)rejecter)
48
+
49
+ RCT_EXTERN_METHOD(setPlaneDetection:(nonnull NSString *)mode
50
+ resolver:(RCTPromiseResolveBlock)resolver
51
+ rejecter:(RCTPromiseRejectBlock)rejecter)
52
+
53
+ // v0.18.0 — toggle the onArFrame LIGHT-metadata channel + its throttle.
54
+ RCT_EXTERN_METHOD(setArFrameMetaEnabled:(nonnull NSNumber *)enabled
55
+ intervalMs:(nonnull NSNumber *)intervalMs
56
+ resolver:(RCTPromiseResolveBlock)resolver
57
+ rejecter:(RCTPromiseRejectBlock)rejecter)
58
+
37
59
  RCT_EXTERN_METHOD(snapshotPoseLog:(RCTPromiseResolveBlock)resolver
38
60
  rejecter:(RCTPromiseRejectBlock)rejecter)
39
61
 
@@ -17,15 +17,109 @@
17
17
  import Foundation
18
18
  import React
19
19
 
20
+ // v0.18.0 — `RNSARSessionBridge` is now an `RCTEventEmitter` (was a
21
+ // plain `NSObject`) so it can deliver the `onArFrame` LIGHT-metadata
22
+ // channel as the JS `RNImageStitcherARFrame` device event. The JS side
23
+ // subscribes via `new NativeEventEmitter(NativeModules.RNSARSession)` —
24
+ // the same module name this bridge is remapped to (see ARSessionBridge.m).
25
+ //
26
+ // Pattern mirrors `IncrementalStitcherBridge`: observe a NotificationCenter
27
+ // post from the framework-free `RNSARSession` engine, then re-emit on the
28
+ // main queue via `bridge.enqueueJSCall("RCTDeviceEventEmitter", "emit", …)`
29
+ // rather than `RCTEventEmitter.sendEvent(…)`, because under RN bridgeless
30
+ // interop `sendEvent` silently no-ops for some event-body shapes (see the
31
+ // IncrementalStitcherBridge.handleStateUpdate docstring).
20
32
  @objc(RNSARSessionBridge)
21
- public final class RNSARSessionBridge: NSObject {
33
+ public final class RNSARSessionBridge: RCTEventEmitter {
34
+
35
+ /// Whether at least one JS listener is attached to the AR-frame event.
36
+ /// RN's EventEmitter contract: don't emit when no listeners are
37
+ /// registered. Toggled by `startObserving` / `stopObserving`.
38
+ private var hasListeners: Bool = false
39
+
40
+ private static let arFrameEvent = "RNImageStitcherARFrame"
41
+
42
+ public override init() {
43
+ super.init()
44
+ // Defensively de-dupe the observer: under RN bridgeless interop a
45
+ // bridge's init() can run twice on the same instance. Remove any
46
+ // prior registration for this notification before adding, so the
47
+ // observer fires at most once per post regardless.
48
+ NotificationCenter.default.removeObserver(
49
+ self,
50
+ name: .retailensARFrameMeta,
51
+ object: nil
52
+ )
53
+ NotificationCenter.default.addObserver(
54
+ self,
55
+ selector: #selector(handleArFrameMeta(_:)),
56
+ name: .retailensARFrameMeta,
57
+ object: nil
58
+ )
59
+ }
22
60
 
23
- @objc public static func requiresMainQueueSetup() -> Bool {
61
+ deinit {
62
+ NotificationCenter.default.removeObserver(self)
63
+ }
64
+
65
+ // MARK: - RCTEventEmitter protocol
66
+
67
+ public override static func requiresMainQueueSetup() -> Bool {
24
68
  // ARSession.start() must be called on the main thread —
25
69
  // ARKit needs to attach to the active CVDisplayLink.
26
70
  return true
27
71
  }
28
72
 
73
+ public override func supportedEvents() -> [String]! {
74
+ return [Self.arFrameEvent]
75
+ }
76
+
77
+ public override func startObserving() {
78
+ hasListeners = true
79
+ }
80
+
81
+ public override func stopObserving() {
82
+ hasListeners = false
83
+ }
84
+
85
+ /// Forward a posted `ARFrameMeta` dictionary to JS as the
86
+ /// `RNImageStitcherARFrame` device event. Dropped when no JS listener
87
+ /// is attached. Emits via `enqueueJSCall` on the main queue (see the
88
+ /// class docstring for why not `sendEvent`).
89
+ @objc private func handleArFrameMeta(_ notification: Notification) {
90
+ guard hasListeners else { return }
91
+ guard let userInfo = notification.userInfo else { return }
92
+ DispatchQueue.main.async { [weak self] in
93
+ guard let self = self, let bridge = self.bridge else { return }
94
+ bridge.enqueueJSCall(
95
+ "RCTDeviceEventEmitter",
96
+ method: "emit",
97
+ args: [Self.arFrameEvent, userInfo],
98
+ completion: nil
99
+ )
100
+ }
101
+ }
102
+
103
+ // MARK: - Module methods
104
+
105
+ /// v0.18.0 — toggle the `onArFrame` LIGHT-metadata channel. Called
106
+ /// from JS with `true` + the throttle interval (ms) when a host
107
+ /// supplies `<Camera onArFrame={...}>`, and `false` on
108
+ /// unmount / prop-removal. Resolves with no value.
109
+ @objc(setArFrameMetaEnabled:intervalMs:resolver:rejecter:)
110
+ public func setArFrameMetaEnabled(
111
+ enabled: NSNumber,
112
+ intervalMs: NSNumber,
113
+ resolver: @escaping RCTPromiseResolveBlock,
114
+ rejecter: @escaping RCTPromiseRejectBlock
115
+ ) {
116
+ RNSARSession.shared.setArFrameMetaEnabled(
117
+ enabled.boolValue,
118
+ intervalMs: intervalMs.doubleValue
119
+ )
120
+ resolver(nil)
121
+ }
122
+
29
123
  @objc(isSupported:rejecter:)
30
124
  public func isSupported(
31
125
  resolver: @escaping RCTPromiseResolveBlock,
@@ -68,6 +162,47 @@ public final class RNSARSessionBridge: NSObject {
68
162
  ])
69
163
  }
70
164
 
165
+ /// Toggle ARKit scene reconstruction (LiDAR mesh / `ARMeshAnchor`s).
166
+ /// Driven by the <Camera> `enableMesh` prop; gates the
167
+ /// StitcherFrame `meshGeometry` extraction at the SESSION level
168
+ /// (the per-frame `__stitcherProxy.setExtractionConfig(...mesh)`
169
+ /// gates the marshaling — both must be on for a host to receive
170
+ /// mesh). Resolves with no value.
171
+ ///
172
+ /// Hops to the main queue: `setSceneReconstructionEnabled` may call
173
+ /// `arSession.run(config)` to reconfigure a live session, and
174
+ /// ARKit session lifecycle must run on the main thread (same
175
+ /// constraint as `start`).
176
+ @objc(setSceneReconstructionEnabled:resolver:rejecter:)
177
+ public func setSceneReconstructionEnabled(
178
+ enabled: NSNumber,
179
+ resolver: @escaping RCTPromiseResolveBlock,
180
+ rejecter: @escaping RCTPromiseRejectBlock
181
+ ) {
182
+ let on = enabled.boolValue
183
+ DispatchQueue.main.async {
184
+ RNSARSession.shared.setSceneReconstructionEnabled(on)
185
+ resolver(nil)
186
+ }
187
+ }
188
+
189
+ /// Hops to the main queue: `setPlaneDetection` may call
190
+ /// `arSession.run(config)` to reconfigure a live session, and ARKit
191
+ /// session lifecycle must run on the main thread (same constraint as
192
+ /// `start` / `setSceneReconstructionEnabled`).
193
+ @objc(setPlaneDetection:resolver:rejecter:)
194
+ public func setPlaneDetection(
195
+ mode: NSString,
196
+ resolver: @escaping RCTPromiseResolveBlock,
197
+ rejecter: @escaping RCTPromiseRejectBlock
198
+ ) {
199
+ let m = mode as String
200
+ DispatchQueue.main.async {
201
+ RNSARSession.shared.setPlaneDetection(m)
202
+ resolver(nil)
203
+ }
204
+ }
205
+
71
206
  @objc(snapshotPoseLog:rejecter:)
72
207
  public func snapshotPoseLog(
73
208
  resolver: @escaping RCTPromiseResolveBlock,