react-native-image-stitcher 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +151 -0
  2. package/RNImageStitcher.podspec +1 -1
  3. package/android/src/main/cpp/CMakeLists.txt +4 -4
  4. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +216 -7
  5. package/android/src/main/java/io/imagestitcher/rn/ARFrameContext.kt +89 -0
  6. package/android/src/main/java/io/imagestitcher/rn/ARFramePlugin.kt +57 -0
  7. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +831 -6
  8. package/android/src/main/java/io/imagestitcher/rn/RNSARPluginRegistry.kt +109 -0
  9. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +184 -0
  10. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +1 -1
  11. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +84 -2
  12. package/cpp/{stitcher_frame_data.hpp → camera_frame_data.hpp} +96 -13
  13. package/cpp/{stitcher_frame_jsi.cpp → camera_frame_jsi.cpp} +154 -11
  14. package/cpp/{stitcher_frame_jsi.hpp → camera_frame_jsi.hpp} +12 -12
  15. package/cpp/stitcher_proxy_jsi.cpp +31 -0
  16. package/cpp/stitcher_proxy_jsi.hpp +16 -0
  17. package/cpp/stitcher_worklet_dispatch.cpp +5 -5
  18. package/cpp/stitcher_worklet_dispatch.hpp +5 -5
  19. package/dist/camera/ARCameraView.d.ts +81 -3
  20. package/dist/camera/ARCameraView.js +103 -1
  21. package/dist/camera/Camera.d.ts +73 -7
  22. package/dist/camera/Camera.js +2 -2
  23. package/dist/index.d.ts +3 -1
  24. package/dist/stitching/ARFrameMeta.d.ts +149 -0
  25. package/dist/stitching/{StitcherFrame.js → ARFrameMeta.js} +1 -1
  26. package/dist/stitching/{StitcherFrame.d.ts → CameraFrame.d.ts} +70 -11
  27. package/dist/stitching/CameraFrame.js +4 -0
  28. package/dist/stitching/useStitcherWorklet.d.ts +4 -4
  29. package/dist/stitching/useStitcherWorklet.js +4 -4
  30. package/ios/Sources/RNImageStitcher/ARSessionBridge.m +23 -1
  31. package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +172 -2
  32. package/ios/Sources/RNImageStitcher/CameraFrameHostObject.h +108 -0
  33. package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm +772 -0
  34. package/ios/Sources/RNImageStitcher/RNISARFramePlugin.swift +247 -0
  35. package/ios/Sources/RNImageStitcher/RNSARSession.swift +418 -34
  36. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +2 -2
  37. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +4 -4
  38. package/package.json +1 -1
  39. package/src/camera/ARCameraView.tsx +230 -5
  40. package/src/camera/Camera.tsx +91 -7
  41. package/src/index.ts +12 -3
  42. package/src/stitching/ARFrameMeta.ts +157 -0
  43. package/src/stitching/{StitcherFrame.ts → CameraFrame.ts} +79 -11
  44. package/src/stitching/useStitcherWorklet.ts +9 -9
  45. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  46. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
@@ -41,7 +41,8 @@
41
41
  import React from 'react';
42
42
  import { type StyleProp, type ViewStyle } from 'react-native';
43
43
  import type { DrawableFrameProcessor, ReadonlyFrameProcessor } from 'react-native-vision-camera';
44
- import type { StitcherFrameProcessor } from '../stitching/StitcherFrame';
44
+ import type { CameraFrameProcessor } from '../stitching/CameraFrame';
45
+ import type { ARFrameMeta, ARPluginResult } from '../stitching/ARFrameMeta';
45
46
  import { type CaptureHeaderProps } from './CaptureHeader';
46
47
  import { type CapturePreviewAction } from './CapturePreview';
47
48
  import { type CaptureThumbnailItem } from './CaptureThumbnailStrip';
@@ -527,11 +528,11 @@ export interface CameraProps {
527
528
  * worklet to fire on vc's Frame Processor runtime.
528
529
  *
529
530
  * ```tsx
530
- * import { Camera, useFrameProcessor, type StitcherFrame }
531
+ * import { Camera, useFrameProcessor, type CameraFrame }
531
532
  * from 'react-native-image-stitcher';
532
533
  *
533
534
  * function MyScreen() {
534
- * const fp = useFrameProcessor((frame: StitcherFrame) => {
535
+ * const fp = useFrameProcessor((frame: CameraFrame) => {
535
536
  * 'worklet';
536
537
  * // ...
537
538
  * }, []);
@@ -552,12 +553,12 @@ export interface CameraProps {
552
553
  * ```tsx
553
554
  * import {
554
555
  * Camera, useFrameProcessor, useStitcherWorklet,
555
- * type StitcherFrame,
556
+ * type CameraFrame,
556
557
  * } from 'react-native-image-stitcher';
557
558
  *
558
559
  * function MyScreen() {
559
560
  * const stitcher = useStitcherWorklet();
560
- * const fp = useFrameProcessor((frame: StitcherFrame) => {
561
+ * const fp = useFrameProcessor((frame: CameraFrame) => {
561
562
  * 'worklet';
562
563
  * hostPreLogic(frame);
563
564
  * stitcher.call(frame); // ← first-party stitching
@@ -601,14 +602,79 @@ export interface CameraProps {
601
602
  /**
602
603
  * AR-mode host worklet, invoked once per ARKit / ARCore frame
603
604
  * ALONGSIDE the lib's first-party stitching (composition, not
604
- * replacement). Receives a `StitcherFrame` tagged `source: 'ar'`
605
+ * replacement). Receives a `CameraFrame` tagged `source: 'ar'`
605
606
  * with world-space `pose` + `arTrackingState`. Only fires in AR
606
607
  * capture (`captureSource === 'ar'`); the non-AR equivalent is
607
608
  * `frameProcessor` above (the two modes use different runtimes and
608
609
  * frame shapes). Must be a `'worklet'`-prefixed function; if the
609
610
  * native install is unavailable it silently never fires.
610
611
  */
611
- arFrameProcessor?: StitcherFrameProcessor;
612
+ arFrameProcessor?: CameraFrameProcessor;
613
+ /**
614
+ * Opt in to per-frame AR depth on the `arFrameProcessor` frame
615
+ * (`CameraFrame.arDepth`). Default `false` — depth is the costliest
616
+ * field (a per-frame buffer copy), so it's off until you need it.
617
+ */
618
+ enableDepth?: boolean;
619
+ /**
620
+ * Opt in to per-frame AR anchors (`CameraFrame.arAnchors` — detected
621
+ * planes / images). Default `false`.
622
+ */
623
+ enableAnchors?: boolean;
624
+ /**
625
+ * Opt in to scene-reconstruction mesh anchors (`type: 'mesh'` in
626
+ * `arAnchors`, with `meshGeometry`). Default `false`. iOS enables
627
+ * ARKit `sceneReconstruction` (LiDAR); Android reconstructs a rough
628
+ * mesh from the depth map. Expensive — only on when needed.
629
+ */
630
+ enableMesh?: boolean;
631
+ /**
632
+ * Which plane orientations to surface in `CameraFrame.arAnchors`
633
+ * (requires `enableAnchors`; AR capture only). Default `'vertical'`
634
+ * — the orientation the plane-projected stitch path has always used.
635
+ * `'horizontal'` surfaces floors / tables; `'both'` surfaces every
636
+ * detected plane. See `ARCameraView` for the per-platform details.
637
+ */
638
+ planeDetection?: 'vertical' | 'horizontal' | 'both';
639
+ /**
640
+ * v0.18.0 — LIGHT per-frame AR metadata callback, invoked on the JS
641
+ * MAIN thread (NOT a worklet). Only fires in AR capture
642
+ * (`captureSource === 'ar'`). Receives an {@link ARFrameMeta} carrying
643
+ * pose, tracking state, intrinsics, and (when the matching `enable*`
644
+ * prop is on) depth dimensions, anchors, and mesh counts.
645
+ *
646
+ * This is the recommended way to read AR metadata: it sidesteps the
647
+ * worklet path entirely (the `arFrameProcessor` worklet can only safely
648
+ * surface a worklets-core shared value, because capturing a host
649
+ * callback crashes the worklet closure-wrap). Native builds the meta
650
+ * and emits a device event; `<Camera>` threads the handler through to
651
+ * `<ARCameraView>`, which subscribes and invokes it on the main thread.
652
+ */
653
+ onArFrame?: (meta: ARFrameMeta) => void;
654
+ /**
655
+ * v0.18.0 — throttle interval (ms) for {@link onArFrame}. Default `100`
656
+ * (≈ 10 Hz). No effect unless `onArFrame` is provided.
657
+ */
658
+ arFrameMetaInterval?: number;
659
+ /**
660
+ * v0.19.0 — ASYNCHRONOUS AR-plugin result callback (the AR plugin
661
+ * framework), invoked on the JS MAIN thread (NOT a worklet). Only fires in
662
+ * AR capture (`captureSource === 'ar'`). Host-registered native plugins
663
+ * (see `RNISARPluginRegistry` / `RNSARPluginRegistry`) that offload heavy
664
+ * per-frame work to their own queue push results via
665
+ * `registry.emit(name, result)`; `<Camera>` threads this handler to
666
+ * `<ARCameraView>`, which subscribes to the `RNImageStitcherARPluginResult`
667
+ * device event and invokes it with `{ plugin, result }`.
668
+ *
669
+ * SYNCHRONOUS plugin results (computed inline on the AR thread) instead ride
670
+ * the throttled {@link onArFrame} event on {@link ARFrameMeta.plugins}.
671
+ * Use `onArFrame` for the in-band sync channel and `onArPluginResult` for
672
+ * the out-of-band async channel — a host can wire either or both.
673
+ *
674
+ * The SDK ships ONLY the generic plugin framework; there are no built-in
675
+ * plugins, so this never fires unless the host registers native plugins.
676
+ */
677
+ onArPluginResult?: (e: ARPluginResult) => void;
612
678
  /**
613
679
  * Which device holds the non-AR panorama capture accepts.
614
680
  *
@@ -310,7 +310,7 @@ function extractPanoramaOverrides(props) {
310
310
  * The public `<Camera>` component.
311
311
  */
312
312
  function Camera(props) {
313
- const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, engine = 'batch-keyframe',
313
+ const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, enableDepth, enableAnchors, enableMesh, planeDetection, onArFrame, arFrameMetaInterval, onArPluginResult, engine = 'batch-keyframe',
314
314
  // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
315
315
  panMode = 'vertical', panGuidance = true, maxPanDurationMs = 0, panTooFastThreshold, lateralBudgetCm = 4, rectCrop = false, showPreview = false, guidanceCopy, } = props;
316
316
  // Derived guidance state. The landscape-only gate decision itself is
@@ -1425,7 +1425,7 @@ function Camera(props) {
1425
1425
  // (V12.14.8 OOM fix). The CaptureStatusOverlay renders the
1426
1426
  // "Stitching…" state on top, so no placeholder label is needed
1427
1427
  // in that case — only for the camera-switch transition.
1428
- react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
1428
+ react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor, enableDepth: enableDepth, enableAnchors: enableAnchors, enableMesh: enableMesh, planeDetection: planeDetection, onArFrame: onArFrame, arFrameMetaInterval: arFrameMetaInterval, onArPluginResult: onArPluginResult })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
1429
1429
  // `video={true}` is REQUIRED for takeSnapshot to work on iOS.
1430
1430
  // vision-camera v4's iOS implementation of takeSnapshot waits
1431
1431
  // for a frame on the video pipeline; with video disabled, the
package/dist/index.d.ts CHANGED
@@ -91,7 +91,9 @@ export { IncrementalOutcome, incrementalStitcherIsAvailable, subscribeIncrementa
91
91
  export type { IncrementalState, AcceptedKeyframe } from './stitching/incremental';
92
92
  export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
93
93
  export { useKeyframeStream } from './stitching/useKeyframeStream';
94
- export type { StitcherFrame, StitcherFrameProcessor, ARAnchor, } from './stitching/StitcherFrame';
94
+ export type { CameraFrame, CameraFrameProcessor, ARAnchor, } from './stitching/CameraFrame';
95
+ export type { ARFrameMeta } from './stitching/ARFrameMeta';
96
+ export type { ARPluginResult } from './stitching/ARFrameMeta';
95
97
  export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
96
98
  export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
97
99
  export { useStitcherWorklet } from './stitching/useStitcherWorklet';
@@ -0,0 +1,149 @@
1
+ /**
2
+ * v0.18.0 — LIGHT per-frame AR metadata delivered to JS on the MAIN
3
+ * thread via a normal callback (`onArFrame`), bypassing worklets entirely.
4
+ *
5
+ * ## Why a callback, NOT a worklet
6
+ *
7
+ * The AR worklet path (`arFrameProcessor` + the `__stitcherProxy` JSI
8
+ * registry) deep-copies the worklet's whole closure into the AR worklet
9
+ * runtime via react-native-worklets-core's `WorkletInvoker`. When the
10
+ * worklet captures a host object (e.g. a `createRunOnJS` callback) that
11
+ * closure-wrap recurses without termination → stack overflow → SIGBUS the
12
+ * instant AR mode mounts (verified on device). Worklets can therefore
13
+ * only safely capture a worklets-core *shared value* — an awkward,
14
+ * poll-from-JS pattern for getting structured data back.
15
+ *
16
+ * `onArFrame` sidesteps the whole problem: native builds the metadata and
17
+ * emits it as a plain `RNImageStitcherARFrame` device event carrying a
18
+ * JSON object; the JS side subscribes via `NativeEventEmitter` and invokes
19
+ * the host callback on the main thread. No worklet, no closure-wrap, no
20
+ * shared-value polling. This is the recommended way to read AR metadata.
21
+ *
22
+ * ## Cost / gating
23
+ *
24
+ * The metadata is intentionally LIGHT — no pixel / vertex / face byte
25
+ * marshaling. `depth` reports only the depth map's dimensions + whether a
26
+ * confidence channel exists (no buffer copy); `mesh` reports only anchor /
27
+ * vertex / face *counts*. Native gates each costly field on the matching
28
+ * extraction flag (`depth` ⇐ `enableDepth`, `mesh` ⇐ `enableMesh`,
29
+ * `anchors` ⇐ `enableAnchors`); `intrinsics` / `pose` / `trackingState` are
30
+ * always present. Emission is gated on a TS-set enabled flag (only true
31
+ * when `onArFrame` is provided) and throttled to `arFrameMetaInterval` ms
32
+ * (default 100 ≈ 10 Hz) on the native side.
33
+ */
34
+ export interface ARFrameMeta {
35
+ /** Frame-capture timestamp in NANOSECONDS (AR-framework monotonic clock). */
36
+ timestamp: number;
37
+ /** AR tracking quality at this frame. */
38
+ trackingState: 'notAvailable' | 'limited' | 'normal';
39
+ /**
40
+ * Camera pose in world coordinates at frame-capture time.
41
+ *
42
+ * - `rotation` — quaternion `(x, y, z, w)`, matching the convention
43
+ * used throughout the engine + the `CameraFrame.pose` field.
44
+ * - `translation` — metres in world space `[x, y, z]`.
45
+ */
46
+ pose: {
47
+ rotation: [number, number, number, number];
48
+ translation: [number, number, number];
49
+ };
50
+ /**
51
+ * Camera intrinsics for this frame — focal lengths (`fx`, `fy`) and
52
+ * principal point (`cx`, `cy`) in PIXELS at the `imageWidth × imageHeight`
53
+ * capture resolution. Always attempted (not gated); `null` only when the
54
+ * AR framework didn't provide them for this frame.
55
+ */
56
+ intrinsics: {
57
+ fx: number;
58
+ fy: number;
59
+ cx: number;
60
+ cy: number;
61
+ imageWidth: number;
62
+ imageHeight: number;
63
+ } | null;
64
+ /**
65
+ * Depth-map summary — dimensions + whether a per-pixel confidence channel
66
+ * is available. NO pixel buffer is copied (that's the costly part).
67
+ * Present only when the `enableDepth` prop is on AND the device produced a
68
+ * depth map this frame; `null` otherwise.
69
+ */
70
+ depth: {
71
+ width: number;
72
+ height: number;
73
+ hasConfidence: boolean;
74
+ } | null;
75
+ /**
76
+ * Tracked AR anchors visible in this frame (planes / images / points /
77
+ * mesh). Empty array when `enableAnchors` is on but nothing is tracked;
78
+ * effectively empty when `enableAnchors` is off. `transform` is a 4×4
79
+ * row-major anchor→world matrix (16 numbers).
80
+ */
81
+ anchors: Array<{
82
+ id: string;
83
+ type: 'plane' | 'image' | 'point' | 'mesh';
84
+ alignment?: 'horizontal' | 'vertical';
85
+ extent?: [number, number];
86
+ classification?: string;
87
+ transform: number[];
88
+ }>;
89
+ /**
90
+ * Scene-reconstruction mesh summary — anchor / vertex / face COUNTS only
91
+ * (no vertex / face byte marshaling). Present only when `enableMesh` is
92
+ * on; `null` otherwise.
93
+ */
94
+ mesh: {
95
+ anchorCount: number;
96
+ vertexCount: number;
97
+ faceCount: number;
98
+ } | null;
99
+ /**
100
+ * v0.19.0 — SYNCHRONOUS results from host-registered AR frame plugins
101
+ * (the AR plugin framework). Keyed by each plugin's `name()`; the value
102
+ * is the light JSON result the plugin's `process(ctx)` returned on the AR
103
+ * thread (`nil`/`null`-returning plugins are omitted). Only present when
104
+ * the native plugin registry is non-empty AND at least one plugin returned
105
+ * a sync result for this frame; otherwise omitted entirely (zero-plugin
106
+ * apps pay nothing — native skips building the context).
107
+ *
108
+ * The SDK ships ONLY the generic framework — there are no built-in
109
+ * plugins. Hosts register native plugins via `RNISARPluginRegistry`
110
+ * (iOS) / `RNSARPluginRegistry` (Android) at startup; each plugin is
111
+ * called once per AR frame while the registry is non-empty. Result
112
+ * values are `unknown` because each plugin defines its own shape — cast
113
+ * after reading the entry you care about (e.g.
114
+ * `meta.plugins?.brightness as number`).
115
+ *
116
+ * ## Sync vs async results
117
+ *
118
+ * This field carries only the LIGHT, in-band SYNC results (computed fast
119
+ * enough to ride the throttled `onArFrame` event). Plugins that offload
120
+ * heavy work to their own queue deliver results out-of-band via
121
+ * `registry.emit(name, result)`, which surfaces through the separate
122
+ * `onArPluginResult` callback (the `RNImageStitcherARPluginResult`
123
+ * event) — NOT here.
124
+ */
125
+ plugins?: {
126
+ [name: string]: unknown;
127
+ };
128
+ }
129
+ /**
130
+ * v0.19.0 — an ASYNCHRONOUS result from a host-registered AR frame plugin,
131
+ * delivered via the `onArPluginResult` callback.
132
+ *
133
+ * Unlike the in-band SYNC results carried on {@link ARFrameMeta.plugins}
134
+ * (which ride the throttled `onArFrame` event), a plugin produces an async
135
+ * result by offloading heavy work to its own queue and later calling
136
+ * `registry.emit(name, result)` on the native side. The SDK routes that to
137
+ * JS as a `RNImageStitcherARPluginResult` device event; `<ARCameraView>`
138
+ * subscribes and invokes `onArPluginResult` on the JS MAIN thread.
139
+ *
140
+ * `result` is `unknown` because each plugin defines its own result shape —
141
+ * branch on `plugin` (the emitting plugin's `name()`) and cast accordingly.
142
+ */
143
+ export interface ARPluginResult {
144
+ /** The `name()` of the plugin that emitted this result. */
145
+ plugin: string;
146
+ /** The plugin-defined result payload (cast after branching on `plugin`). */
147
+ result: unknown;
148
+ }
149
+ //# sourceMappingURL=ARFrameMeta.d.ts.map
@@ -1,4 +1,4 @@
1
1
  "use strict";
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  Object.defineProperty(exports, "__esModule", { value: true });
4
- //# sourceMappingURL=StitcherFrame.js.map
4
+ //# sourceMappingURL=ARFrameMeta.js.map
@@ -2,7 +2,7 @@
2
2
  * v0.8.0 — unified frame contract for the lib's worklet processor.
3
3
  *
4
4
  * Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
5
- * this directory) receive a `StitcherFrame` regardless of capture mode.
5
+ * this directory) receive a `CameraFrame` regardless of capture mode.
6
6
  * The lib-owned worklet runtime guarantees the same JS-visible shape
7
7
  * whether the underlying source is a vision-camera `Frame` (non-AR
8
8
  * mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
@@ -16,10 +16,10 @@
16
16
  * (Phase-0 audit confirmed the iOS path). But vision-camera's
17
17
  * **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
18
18
  * ARCore does NOT produce `ImageProxy` instances. Forcing
19
- * `StitcherFrame extends Frame` would either (a) require reverse-
19
+ * `CameraFrame extends Frame` would either (a) require reverse-
20
20
  * engineering ImageProxy on Android (intractable + fragile), or
21
21
  * (b) make the type asymmetric per platform. Both are worse than
22
- * making `StitcherFrame` a structural sibling type that vc Frames
22
+ * making `CameraFrame` a structural sibling type that vc Frames
23
23
  * happen to satisfy (because vc Frames carry the same width / height /
24
24
  * orientation / pixelFormat / timestamp / toArrayBuffer surface).
25
25
  *
@@ -36,7 +36,7 @@
36
36
  * a JPEG-encode frame-processor plugin). Returning a reference and
37
37
  * reading it later will read into freed memory.
38
38
  */
39
- export interface StitcherFrame {
39
+ export interface CameraFrame {
40
40
  /** Pixel width of the camera image. */
41
41
  width: number;
42
42
  /** Pixel height of the camera image. */
@@ -141,6 +141,27 @@ export interface StitcherFrame {
141
141
  * tracking is degraded check this. Undefined in non-AR mode.
142
142
  */
143
143
  arTrackingState?: 'notAvailable' | 'limited' | 'normal';
144
+ /**
145
+ * Camera intrinsics for THIS frame — focal lengths (`fx`,`fy`) and
146
+ * principal point (`cx`,`cy`) in PIXELS at the `imageWidth × imageHeight`
147
+ * capture resolution. Needed to lift 2D image-space coordinates to 3D
148
+ * via pose + intrinsics (e.g. object-level reconstruction).
149
+ *
150
+ * Populated on **AR frames** (`source: 'ar'`) from ARKit
151
+ * `ARCamera.intrinsics` / ARCore `Camera` intrinsics. **Undefined for
152
+ * non-AR (vision-camera) frames** — they are raw vc `Frame`s without an
153
+ * intrinsics surface; read vc's own APIs there if needed. (The spec
154
+ * called this required; it's optional here because the non-AR frame
155
+ * shape genuinely can't carry it.)
156
+ */
157
+ intrinsics?: {
158
+ fx: number;
159
+ fy: number;
160
+ cx: number;
161
+ cy: number;
162
+ imageWidth: number;
163
+ imageHeight: number;
164
+ };
144
165
  }
145
166
  /**
146
167
  * v0.8.0 — public AR anchor type. Subset of ARKit/ARCore anchor info
@@ -150,21 +171,59 @@ export interface StitcherFrame {
150
171
  export interface ARAnchor {
151
172
  /** Stable per-session anchor identifier. */
152
173
  id: string;
153
- /** Anchor kind. `'point'` is Android (ARCore) only. */
154
- type: 'plane' | 'image' | 'point';
155
174
  /**
156
- * 4×4 row-major transform from anchor space to world space.
157
- * 16 numbers.
175
+ * Anchor kind. `'point'` is Android (ARCore) only; `'mesh'` is a
176
+ * scene-reconstruction mesh anchor, present only when the `enableMesh`
177
+ * `<Camera>` prop is on (and the device supports reconstruction).
178
+ */
179
+ type: 'plane' | 'image' | 'point' | 'mesh';
180
+ /**
181
+ * 4×4 row-major transform from anchor space to world space (16
182
+ * numbers). For `'mesh'` anchors, the `meshGeometry.vertices` are in
183
+ * this anchor's LOCAL space — multiply by `transform` for world coords.
158
184
  */
159
185
  transform: number[];
186
+ /**
187
+ * Plane orientation — `'horizontal'` (floor / table / seat) vs
188
+ * `'vertical'` (wall / door / window). Present on `'plane'` anchors;
189
+ * undefined for other anchor kinds. Lets a host distinguish a shelf
190
+ * surface from the wall behind it.
191
+ */
192
+ alignment?: 'horizontal' | 'vertical';
193
+ /**
194
+ * Plane size in metres along its local x / z axes (`[x, z]`). Present
195
+ * on `'plane'` anchors only.
196
+ */
197
+ extent?: [number, number];
198
+ /**
199
+ * ARKit semantic classification of the plane's surface, when the
200
+ * framework provides it (iOS; mostly horizontal planes). Undefined
201
+ * when unknown / unsupported (incl. Android, which has no equivalent).
202
+ */
203
+ classification?: 'wall' | 'floor' | 'ceiling' | 'table' | 'seat' | 'door' | 'window' | 'none';
204
+ /**
205
+ * Scene-reconstruction geometry — present only on `type: 'mesh'`
206
+ * anchors. Buffers (wrap in the noted typed-array view):
207
+ * - `vertices` → `Float32Array`, xyz triplets in anchor-local space.
208
+ * - `faces` → `Uint32Array`, triangle indices into `vertices`.
209
+ * - `classifications` → optional `Uint8Array`, one ARKit mesh class
210
+ * per face (0=none, 1=wall, 2=floor, 3=ceiling, …). **iOS only**
211
+ * (from `ARMeshAnchor`); absent on Android, where the mesh is
212
+ * reconstructed from the depth map and carries no semantics.
213
+ */
214
+ meshGeometry?: {
215
+ vertices: ArrayBuffer;
216
+ faces: ArrayBuffer;
217
+ classifications?: ArrayBuffer;
218
+ };
160
219
  }
161
220
  /**
162
221
  * v0.8.0 — worklet function signature for the unified frame processor.
163
222
  *
164
223
  * Must be a `'worklet'`-prefixed function (so it can run on the
165
- * worklet runtime). Receives a `StitcherFrame` per camera frame; the
224
+ * worklet runtime). Receives a `CameraFrame` per camera frame; the
166
225
  * return value is ignored (use `runOnJS` / shared values to surface
167
226
  * results back to the JS thread).
168
227
  */
169
- export type StitcherFrameProcessor = (frame: StitcherFrame) => void;
170
- //# sourceMappingURL=StitcherFrame.d.ts.map
228
+ export type CameraFrameProcessor = (frame: CameraFrame) => void;
229
+ //# sourceMappingURL=CameraFrame.d.ts.map
@@ -0,0 +1,4 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ //# sourceMappingURL=CameraFrame.js.map
@@ -106,17 +106,17 @@
106
106
  * their own worklet via this hook must do the wiring themselves.
107
107
  */
108
108
  import type { Frame } from 'react-native-vision-camera';
109
- import type { StitcherFrame } from './StitcherFrame';
109
+ import type { CameraFrame } from './CameraFrame';
110
110
  /**
111
111
  * Frames the lib's stitching worklet accepts. Accepting either a
112
112
  * vc `Frame` (what the host's `useFrameProcessor` body sees) or the
113
- * lib's `StitcherFrame` (what the lib's `useFrameProcessor` body
113
+ * lib's `CameraFrame` (what the lib's `useFrameProcessor` body
114
114
  * sees) keeps the same `useStitcherWorklet` usable from both kinds
115
115
  * of host worklet bodies without a cast on the call site. The
116
116
  * worklet only reads `width` / `height`; the rest of the frame
117
117
  * object is forwarded verbatim to the native plugin.
118
118
  */
119
- export type StitcherWorkletInput = Frame | StitcherFrame;
119
+ export type StitcherWorkletInput = Frame | CameraFrame;
120
120
  export interface UseStitcherWorkletOptions {
121
121
  /**
122
122
  * Gyro sample interval in ms (~30 Hz default). Drives the JS-
@@ -145,7 +145,7 @@ export interface UseStitcherWorkletOptions {
145
145
  }
146
146
  export interface StitcherWorkletHandle {
147
147
  /**
148
- * Worklet function: pass a `StitcherFrame` to perform one frame of
148
+ * Worklet function: pass a `CameraFrame` to perform one frame of
149
149
  * the lib's first-party stitching (throttle + pose synthesis +
150
150
  * native plugin call). Safe to call from inside another
151
151
  * `'worklet'`-prefixed function (this is the canonical
@@ -228,7 +228,7 @@ function useStitcherWorklet(options = {}) {
228
228
  // party callback installed in `RNSARWorkletRuntime`). Calling
229
229
  // the vc Frame Processor plugin here would throw
230
230
  // `getPropertyAsObject: property '__frame' is undefined`
231
- // because AR frames are `StitcherFrameHostObject` instances
231
+ // because AR frames are `CameraFrameHostObject` instances
232
232
  // and don't carry the vc `Frame` proxy's JSI marker. The
233
233
  // throw is caught silently by the per-worklet error handler
234
234
  // (`RNSARWorkletRuntime.mm:284-301`) and bubbles up only to
@@ -240,12 +240,12 @@ function useStitcherWorklet(options = {}) {
240
240
  // hook (the AR-side stitching path runs natively, independent
241
241
  // of the composed worklet body).
242
242
  //
243
- // The `(frame as StitcherFrame).source` cast is safe: vc
243
+ // The `(frame as CameraFrame).source` cast is safe: vc
244
244
  // `Frame` doesn't carry a `source` property so the check
245
245
  // returns `undefined !== 'ar'` → `true`, and the worklet
246
246
  // proceeds normally. Only frames that explicitly tag
247
247
  // themselves as AR-source (which our native AR dispatcher
248
- // does — see `StitcherFrameHostObject.mm`) get short-circuited.
248
+ // does — see `CameraFrameHostObject.mm`) get short-circuited.
249
249
  if (frame.source === 'ar')
250
250
  return;
251
251
  // Throttle (verbatim from useFrameProcessorDriver).
@@ -273,7 +273,7 @@ function useStitcherWorklet(options = {}) {
273
273
  const fx = w * sharedFxNumerator.value;
274
274
  const fy = h * sharedFyNumerator.value;
275
275
  // vc's `plugin.call` is typed against vc's `Frame`. The worklet
276
- // accepts the union (`Frame | StitcherFrame`); cast through
276
+ // accepts the union (`Frame | CameraFrame`); cast through
277
277
  // `unknown` because the union doesn't satisfy vc's interface
278
278
  // even though structurally both members do.
279
279
  plugin.call(frame, {
@@ -5,6 +5,7 @@
5
5
  // imports as `NativeModules.RNSARSession`.
6
6
 
7
7
  #import <React/RCTBridgeModule.h>
8
+ #import <React/RCTEventEmitter.h>
8
9
 
9
10
  // REMAP form, NOT EXTERN_MODULE. The Swift singleton in
10
11
  // RNSARSession.swift takes the @objc name "RNSARSession"
@@ -20,7 +21,14 @@
20
21
  // `RNSARSessionBridge` and dispatch methods against THAT
21
22
  // class — where takePhoto / startRecording / stopRecording etc.
22
23
  // actually live.
23
- @interface RCT_EXTERN_REMAP_MODULE(RNSARSession, RNSARSessionBridge, NSObject)
24
+ //
25
+ // v0.18.0 — base class is now `RCTEventEmitter` (was `NSObject`) so the
26
+ // "RNSARSession" module can emit the `RNImageStitcherARFrame` device
27
+ // event for the `onArFrame` channel. RN auto-provides the
28
+ // `addListener:` / `removeListeners:` emitter selectors for a module
29
+ // whose remap base is RCTEventEmitter; the Swift class supplies
30
+ // `supportedEvents` / `startObserving` / `stopObserving`.
31
+ @interface RCT_EXTERN_REMAP_MODULE(RNSARSession, RNSARSessionBridge, RCTEventEmitter)
24
32
 
25
33
  RCT_EXTERN_METHOD(isSupported:(RCTPromiseResolveBlock)resolver
26
34
  rejecter:(RCTPromiseRejectBlock)rejecter)
@@ -34,6 +42,20 @@ RCT_EXTERN_METHOD(stop:(RCTPromiseResolveBlock)resolver
34
42
  RCT_EXTERN_METHOD(getState:(RCTPromiseResolveBlock)resolver
35
43
  rejecter:(RCTPromiseRejectBlock)rejecter)
36
44
 
45
+ RCT_EXTERN_METHOD(setSceneReconstructionEnabled:(nonnull NSNumber *)enabled
46
+ resolver:(RCTPromiseResolveBlock)resolver
47
+ rejecter:(RCTPromiseRejectBlock)rejecter)
48
+
49
+ RCT_EXTERN_METHOD(setPlaneDetection:(nonnull NSString *)mode
50
+ resolver:(RCTPromiseResolveBlock)resolver
51
+ rejecter:(RCTPromiseRejectBlock)rejecter)
52
+
53
+ // v0.18.0 — toggle the onArFrame LIGHT-metadata channel + its throttle.
54
+ RCT_EXTERN_METHOD(setArFrameMetaEnabled:(nonnull NSNumber *)enabled
55
+ intervalMs:(nonnull NSNumber *)intervalMs
56
+ resolver:(RCTPromiseResolveBlock)resolver
57
+ rejecter:(RCTPromiseRejectBlock)rejecter)
58
+
37
59
  RCT_EXTERN_METHOD(snapshotPoseLog:(RCTPromiseResolveBlock)resolver
38
60
  rejecter:(RCTPromiseRejectBlock)rejecter)
39
61