react-native-image-stitcher 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +121 -0
  2. package/RNImageStitcher.podspec +1 -1
  3. package/android/src/main/cpp/CMakeLists.txt +4 -4
  4. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +216 -7
  5. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +656 -0
  6. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +156 -0
  7. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +1 -1
  8. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +84 -2
  9. package/cpp/{stitcher_frame_data.hpp → camera_frame_data.hpp} +96 -13
  10. package/cpp/{stitcher_frame_jsi.cpp → camera_frame_jsi.cpp} +154 -11
  11. package/cpp/{stitcher_frame_jsi.hpp → camera_frame_jsi.hpp} +12 -12
  12. package/cpp/stitcher_proxy_jsi.cpp +31 -0
  13. package/cpp/stitcher_proxy_jsi.hpp +16 -0
  14. package/cpp/stitcher_worklet_dispatch.cpp +5 -5
  15. package/cpp/stitcher_worklet_dispatch.hpp +5 -5
  16. package/dist/camera/ARCameraView.d.ts +60 -3
  17. package/dist/camera/ARCameraView.js +68 -1
  18. package/dist/camera/Camera.d.ts +54 -7
  19. package/dist/camera/Camera.js +2 -2
  20. package/dist/index.d.ts +2 -1
  21. package/dist/stitching/ARFrameMeta.d.ts +100 -0
  22. package/dist/stitching/{StitcherFrame.js → ARFrameMeta.js} +1 -1
  23. package/dist/stitching/{StitcherFrame.d.ts → CameraFrame.d.ts} +70 -11
  24. package/dist/stitching/CameraFrame.js +4 -0
  25. package/dist/stitching/useStitcherWorklet.d.ts +4 -4
  26. package/dist/stitching/useStitcherWorklet.js +4 -4
  27. package/ios/Sources/RNImageStitcher/ARSessionBridge.m +23 -1
  28. package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +137 -2
  29. package/ios/Sources/RNImageStitcher/{StitcherFrameHostObject.h → CameraFrameHostObject.h} +26 -3
  30. package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm +760 -0
  31. package/ios/Sources/RNImageStitcher/RNSARSession.swift +292 -34
  32. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +2 -2
  33. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +4 -4
  34. package/package.json +1 -1
  35. package/src/camera/ARCameraView.tsx +165 -5
  36. package/src/camera/Camera.tsx +69 -7
  37. package/src/index.ts +7 -3
  38. package/src/stitching/ARFrameMeta.ts +107 -0
  39. package/src/stitching/{StitcherFrame.ts → CameraFrame.ts} +79 -11
  40. package/src/stitching/useStitcherWorklet.ts +9 -9
  41. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
@@ -66,7 +66,8 @@ import type {
66
66
  } from 'react-native-vision-camera';
67
67
 
68
68
  import { useARSession } from '../ar/useARSession';
69
- import type { StitcherFrameProcessor } from '../stitching/StitcherFrame';
69
+ import type { CameraFrameProcessor } from '../stitching/CameraFrame';
70
+ import type { ARFrameMeta } from '../stitching/ARFrameMeta';
70
71
  import { ARCameraView, type ARCameraViewHandle } from './ARCameraView';
71
72
  import { CameraShutter } from './CameraShutter';
72
73
  import { CameraView } from './CameraView';
@@ -680,11 +681,11 @@ export interface CameraProps {
680
681
  * worklet to fire on vc's Frame Processor runtime.
681
682
  *
682
683
  * ```tsx
683
- * import { Camera, useFrameProcessor, type StitcherFrame }
684
+ * import { Camera, useFrameProcessor, type CameraFrame }
684
685
  * from 'react-native-image-stitcher';
685
686
  *
686
687
  * function MyScreen() {
687
- * const fp = useFrameProcessor((frame: StitcherFrame) => {
688
+ * const fp = useFrameProcessor((frame: CameraFrame) => {
688
689
  * 'worklet';
689
690
  * // ...
690
691
  * }, []);
@@ -705,12 +706,12 @@ export interface CameraProps {
705
706
  * ```tsx
706
707
  * import {
707
708
  * Camera, useFrameProcessor, useStitcherWorklet,
708
- * type StitcherFrame,
709
+ * type CameraFrame,
709
710
  * } from 'react-native-image-stitcher';
710
711
  *
711
712
  * function MyScreen() {
712
713
  * const stitcher = useStitcherWorklet();
713
- * const fp = useFrameProcessor((frame: StitcherFrame) => {
714
+ * const fp = useFrameProcessor((frame: CameraFrame) => {
714
715
  * 'worklet';
715
716
  * hostPreLogic(frame);
716
717
  * stitcher.call(frame); // ← first-party stitching
@@ -755,14 +756,63 @@ export interface CameraProps {
755
756
  /**
756
757
  * AR-mode host worklet, invoked once per ARKit / ARCore frame
757
758
  * ALONGSIDE the lib's first-party stitching (composition, not
758
- * replacement). Receives a `StitcherFrame` tagged `source: 'ar'`
759
+ * replacement). Receives a `CameraFrame` tagged `source: 'ar'`
759
760
  * with world-space `pose` + `arTrackingState`. Only fires in AR
760
761
  * capture (`captureSource === 'ar'`); the non-AR equivalent is
761
762
  * `frameProcessor` above (the two modes use different runtimes and
762
763
  * frame shapes). Must be a `'worklet'`-prefixed function; if the
763
764
  * native install is unavailable it silently never fires.
764
765
  */
765
- arFrameProcessor?: StitcherFrameProcessor;
766
+ arFrameProcessor?: CameraFrameProcessor;
767
+
768
+ /**
769
+ * Opt in to per-frame AR depth on the `arFrameProcessor` frame
770
+ * (`CameraFrame.arDepth`). Default `false` — depth is the costliest
771
+ * field (a per-frame buffer copy), so it's off until you need it.
772
+ */
773
+ enableDepth?: boolean;
774
+ /**
775
+ * Opt in to per-frame AR anchors (`CameraFrame.arAnchors` — detected
776
+ * planes / images). Default `false`.
777
+ */
778
+ enableAnchors?: boolean;
779
+ /**
780
+ * Opt in to scene-reconstruction mesh anchors (`type: 'mesh'` in
781
+ * `arAnchors`, with `meshGeometry`). Default `false`. iOS enables
782
+ * ARKit `sceneReconstruction` (LiDAR); Android reconstructs a rough
783
+ * mesh from the depth map. Expensive — only on when needed.
784
+ */
785
+ enableMesh?: boolean;
786
+ /**
787
+ * Which plane orientations to surface in `CameraFrame.arAnchors`
788
+ * (requires `enableAnchors`; AR capture only). Default `'vertical'`
789
+ * — the orientation the plane-projected stitch path has always used.
790
+ * `'horizontal'` surfaces floors / tables; `'both'` surfaces every
791
+ * detected plane. See `ARCameraView` for the per-platform details.
792
+ */
793
+ planeDetection?: 'vertical' | 'horizontal' | 'both';
794
+
795
+ /**
796
+ * v0.18.0 — LIGHT per-frame AR metadata callback, invoked on the JS
797
+ * MAIN thread (NOT a worklet). Only fires in AR capture
798
+ * (`captureSource === 'ar'`). Receives an {@link ARFrameMeta} carrying
799
+ * pose, tracking state, intrinsics, and (when the matching `enable*`
800
+ * prop is on) depth dimensions, anchors, and mesh counts.
801
+ *
802
+ * This is the recommended way to read AR metadata: it sidesteps the
803
+ * worklet path entirely (the `arFrameProcessor` worklet can only safely
804
+ * surface a worklets-core shared value, because capturing a host
805
+ * callback crashes the worklet closure-wrap). Native builds the meta
806
+ * and emits a device event; `<Camera>` threads the handler through to
807
+ * `<ARCameraView>`, which subscribes and invokes it on the main thread.
808
+ */
809
+ onArFrame?: (meta: ARFrameMeta) => void;
810
+
811
+ /**
812
+ * v0.18.0 — throttle interval (ms) for {@link onArFrame}. Default `100`
813
+ * (≈ 10 Hz). No effect unless `onArFrame` is provided.
814
+ */
815
+ arFrameMetaInterval?: number;
766
816
 
767
817
  // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────────
768
818
  /**
@@ -1171,6 +1221,12 @@ export function Camera(props: CameraProps): React.JSX.Element {
1171
1221
  onCapturePreviewClose,
1172
1222
  frameProcessor: hostFrameProcessor,
1173
1223
  arFrameProcessor,
1224
+ enableDepth,
1225
+ enableAnchors,
1226
+ enableMesh,
1227
+ planeDetection,
1228
+ onArFrame,
1229
+ arFrameMetaInterval,
1174
1230
  engine = 'batch-keyframe',
1175
1231
  // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
1176
1232
  panMode = 'vertical',
@@ -2435,6 +2491,12 @@ export function Camera(props: CameraProps): React.JSX.Element {
2435
2491
  ref={arViewRef}
2436
2492
  style={StyleSheet.absoluteFill}
2437
2493
  arFrameProcessor={arFrameProcessor}
2494
+ enableDepth={enableDepth}
2495
+ enableAnchors={enableAnchors}
2496
+ enableMesh={enableMesh}
2497
+ planeDetection={planeDetection}
2498
+ onArFrame={onArFrame}
2499
+ arFrameMetaInterval={arFrameMetaInterval}
2438
2500
  />
2439
2501
  ) : (
2440
2502
  <CameraView
package/src/index.ts CHANGED
@@ -259,10 +259,14 @@ export { useKeyframeStream } from './stitching/useKeyframeStream';
259
259
  // v0.8.0 — unified frame contract for the worklet processor. Same
260
260
  // JS-visible shape regardless of capture mode (AR vs non-AR).
261
261
  export type {
262
- StitcherFrame,
263
- StitcherFrameProcessor,
262
+ CameraFrame,
263
+ CameraFrameProcessor,
264
264
  ARAnchor,
265
- } from './stitching/StitcherFrame';
265
+ } from './stitching/CameraFrame';
266
+ // v0.18.0 — LIGHT per-frame AR metadata delivered via the `onArFrame`
267
+ // callback (main-thread, worklet-free). See the type's docstring for why
268
+ // it bypasses the worklet path.
269
+ export type { ARFrameMeta } from './stitching/ARFrameMeta';
266
270
  // NOTE: the host-worklet / frame-stream hooks `useFrameProcessor`,
267
271
  // `useThrottledFrameProcessor` and `useFrameStream` (v0.8–v0.9) were
268
272
  // archived in the batch-keyframe cleanup — they drove the third-party
@@ -0,0 +1,107 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+
3
+ /**
4
+ * v0.18.0 — LIGHT per-frame AR metadata delivered to JS on the MAIN
5
+ * thread via a normal callback (`onArFrame`), bypassing worklets entirely.
6
+ *
7
+ * ## Why a callback, NOT a worklet
8
+ *
9
+ * The AR worklet path (`arFrameProcessor` + the `__stitcherProxy` JSI
10
+ * registry) deep-copies the worklet's whole closure into the AR worklet
11
+ * runtime via react-native-worklets-core's `WorkletInvoker`. When the
12
+ * worklet captures a host object (e.g. a `createRunOnJS` callback) that
13
+ * closure-wrap recurses without termination → stack overflow → SIGBUS the
14
+ * instant AR mode mounts (verified on device). Worklets can therefore
15
+ * only safely capture a worklets-core *shared value* — an awkward,
16
+ * poll-from-JS pattern for getting structured data back.
17
+ *
18
+ * `onArFrame` sidesteps the whole problem: native builds the metadata and
19
+ * emits it as a plain `RNImageStitcherARFrame` device event carrying a
20
+ * JSON object; the JS side subscribes via `NativeEventEmitter` and invokes
21
+ * the host callback on the main thread. No worklet, no closure-wrap, no
22
+ * shared-value polling. This is the recommended way to read AR metadata.
23
+ *
24
+ * ## Cost / gating
25
+ *
26
+ * The metadata is intentionally LIGHT — no pixel / vertex / face byte
27
+ * marshaling. `depth` reports only the depth map's dimensions + whether a
28
+ * confidence channel exists (no buffer copy); `mesh` reports only anchor /
29
+ * vertex / face *counts*. Native gates each costly field on the matching
30
+ * extraction flag (`depth` ⇐ `enableDepth`, `mesh` ⇐ `enableMesh`,
31
+ * `anchors` ⇐ `enableAnchors`); `intrinsics` / `pose` / `trackingState` are
32
+ * always present. Emission is gated on a TS-set enabled flag (only true
33
+ * when `onArFrame` is provided) and throttled to `arFrameMetaInterval` ms
34
+ * (default 100 ≈ 10 Hz) on the native side.
35
+ */
36
+ export interface ARFrameMeta {
37
+ /** Frame-capture timestamp in NANOSECONDS (AR-framework monotonic clock). */
38
+ timestamp: number;
39
+
40
+ /** AR tracking quality at this frame. */
41
+ trackingState: 'notAvailable' | 'limited' | 'normal';
42
+
43
+ /**
44
+ * Camera pose in world coordinates at frame-capture time.
45
+ *
46
+ * - `rotation` — quaternion `(x, y, z, w)`, matching the convention
47
+ * used throughout the engine + the `CameraFrame.pose` field.
48
+ * - `translation` — metres in world space `[x, y, z]`.
49
+ */
50
+ pose: {
51
+ rotation: [number, number, number, number];
52
+ translation: [number, number, number];
53
+ };
54
+
55
+ /**
56
+ * Camera intrinsics for this frame — focal lengths (`fx`, `fy`) and
57
+ * principal point (`cx`, `cy`) in PIXELS at the `imageWidth × imageHeight`
58
+ * capture resolution. Always attempted (not gated); `null` only when the
59
+ * AR framework didn't provide them for this frame.
60
+ */
61
+ intrinsics: {
62
+ fx: number;
63
+ fy: number;
64
+ cx: number;
65
+ cy: number;
66
+ imageWidth: number;
67
+ imageHeight: number;
68
+ } | null;
69
+
70
+ /**
71
+ * Depth-map summary — dimensions + whether a per-pixel confidence channel
72
+ * is available. NO pixel buffer is copied (that's the costly part).
73
+ * Present only when the `enableDepth` prop is on AND the device produced a
74
+ * depth map this frame; `null` otherwise.
75
+ */
76
+ depth: {
77
+ width: number;
78
+ height: number;
79
+ hasConfidence: boolean;
80
+ } | null;
81
+
82
+ /**
83
+ * Tracked AR anchors visible in this frame (planes / images / points /
84
+ * mesh). Empty array when `enableAnchors` is on but nothing is tracked;
85
+ * effectively empty when `enableAnchors` is off. `transform` is a 4×4
86
+ * row-major anchor→world matrix (16 numbers).
87
+ */
88
+ anchors: Array<{
89
+ id: string;
90
+ type: 'plane' | 'image' | 'point' | 'mesh';
91
+ alignment?: 'horizontal' | 'vertical';
92
+ extent?: [number, number];
93
+ classification?: string;
94
+ transform: number[];
95
+ }>;
96
+
97
+ /**
98
+ * Scene-reconstruction mesh summary — anchor / vertex / face COUNTS only
99
+ * (no vertex / face byte marshaling). Present only when `enableMesh` is
100
+ * on; `null` otherwise.
101
+ */
102
+ mesh: {
103
+ anchorCount: number;
104
+ vertexCount: number;
105
+ faceCount: number;
106
+ } | null;
107
+ }
@@ -4,7 +4,7 @@
4
4
  * v0.8.0 — unified frame contract for the lib's worklet processor.
5
5
  *
6
6
  * Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
7
- * this directory) receive a `StitcherFrame` regardless of capture mode.
7
+ * this directory) receive a `CameraFrame` regardless of capture mode.
8
8
  * The lib-owned worklet runtime guarantees the same JS-visible shape
9
9
  * whether the underlying source is a vision-camera `Frame` (non-AR
10
10
  * mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
@@ -18,10 +18,10 @@
18
18
  * (Phase-0 audit confirmed the iOS path). But vision-camera's
19
19
  * **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
20
20
  * ARCore does NOT produce `ImageProxy` instances. Forcing
21
- * `StitcherFrame extends Frame` would either (a) require reverse-
21
+ * `CameraFrame extends Frame` would either (a) require reverse-
22
22
  * engineering ImageProxy on Android (intractable + fragile), or
23
23
  * (b) make the type asymmetric per platform. Both are worse than
24
- * making `StitcherFrame` a structural sibling type that vc Frames
24
+ * making `CameraFrame` a structural sibling type that vc Frames
25
25
  * happen to satisfy (because vc Frames carry the same width / height /
26
26
  * orientation / pixelFormat / timestamp / toArrayBuffer surface).
27
27
  *
@@ -38,10 +38,10 @@
38
38
  * a JPEG-encode frame-processor plugin). Returning a reference and
39
39
  * reading it later will read into freed memory.
40
40
  */
41
- export interface StitcherFrame {
41
+ export interface CameraFrame {
42
42
  // ── vision-camera-shaped fields (structural compat) ─────────────
43
43
  // Worklets written against a vc `Frame` work unchanged against a
44
- // `StitcherFrame` (the fields below are a strict subset of vc
44
+ // `CameraFrame` (the fields below are a strict subset of vc
45
45
  // Frame's JS-visible surface).
46
46
 
47
47
  /** Pixel width of the camera image. */
@@ -167,6 +167,28 @@ export interface StitcherFrame {
167
167
  * tracking is degraded check this. Undefined in non-AR mode.
168
168
  */
169
169
  arTrackingState?: 'notAvailable' | 'limited' | 'normal';
170
+
171
+ /**
172
+ * Camera intrinsics for THIS frame — focal lengths (`fx`,`fy`) and
173
+ * principal point (`cx`,`cy`) in PIXELS at the `imageWidth × imageHeight`
174
+ * capture resolution. Needed to lift 2D image-space coordinates to 3D
175
+ * via pose + intrinsics (e.g. object-level reconstruction).
176
+ *
177
+ * Populated on **AR frames** (`source: 'ar'`) from ARKit
178
+ * `ARCamera.intrinsics` / ARCore `Camera` intrinsics. **Undefined for
179
+ * non-AR (vision-camera) frames** — they are raw vc `Frame`s without an
180
+ * intrinsics surface; read vc's own APIs there if needed. (The spec
181
+ * called this required; it's optional here because the non-AR frame
182
+ * shape genuinely can't carry it.)
183
+ */
184
+ intrinsics?: {
185
+ fx: number;
186
+ fy: number;
187
+ cx: number;
188
+ cy: number;
189
+ imageWidth: number;
190
+ imageHeight: number;
191
+ };
170
192
  }
171
193
 
172
194
  /**
@@ -177,21 +199,67 @@ export interface StitcherFrame {
177
199
  export interface ARAnchor {
178
200
  /** Stable per-session anchor identifier. */
179
201
  id: string;
180
- /** Anchor kind. `'point'` is Android (ARCore) only. */
181
- type: 'plane' | 'image' | 'point';
182
202
  /**
183
- * 4×4 row-major transform from anchor space to world space.
184
- * 16 numbers.
203
+ * Anchor kind. `'point'` is Android (ARCore) only; `'mesh'` is a
204
+ * scene-reconstruction mesh anchor, present only when the `enableMesh`
205
+ * `<Camera>` prop is on (and the device supports reconstruction).
206
+ */
207
+ type: 'plane' | 'image' | 'point' | 'mesh';
208
+ /**
209
+ * 4×4 row-major transform from anchor space to world space (16
210
+ * numbers). For `'mesh'` anchors, the `meshGeometry.vertices` are in
211
+ * this anchor's LOCAL space — multiply by `transform` for world coords.
185
212
  */
186
213
  transform: number[];
214
+ /**
215
+ * Plane orientation — `'horizontal'` (floor / table / seat) vs
216
+ * `'vertical'` (wall / door / window). Present on `'plane'` anchors;
217
+ * undefined for other anchor kinds. Lets a host distinguish a shelf
218
+ * surface from the wall behind it.
219
+ */
220
+ alignment?: 'horizontal' | 'vertical';
221
+ /**
222
+ * Plane size in metres along its local x / z axes (`[x, z]`). Present
223
+ * on `'plane'` anchors only.
224
+ */
225
+ extent?: [number, number];
226
+ /**
227
+ * ARKit semantic classification of the plane's surface, when the
228
+ * framework provides it (iOS; mostly horizontal planes). Undefined
229
+ * when unknown / unsupported (incl. Android, which has no equivalent).
230
+ */
231
+ classification?:
232
+ | 'wall'
233
+ | 'floor'
234
+ | 'ceiling'
235
+ | 'table'
236
+ | 'seat'
237
+ | 'door'
238
+ | 'window'
239
+ | 'none';
240
+ /**
241
+ * Scene-reconstruction geometry — present only on `type: 'mesh'`
242
+ * anchors. Buffers (wrap in the noted typed-array view):
243
+ * - `vertices` → `Float32Array`, xyz triplets in anchor-local space.
244
+ * - `faces` → `Uint32Array`, triangle indices into `vertices`.
245
+ * - `classifications` → optional `Uint8Array`, one ARKit mesh class
246
+ * per face (0=none, 1=wall, 2=floor, 3=ceiling, …). **iOS only**
247
+ * (from `ARMeshAnchor`); absent on Android, where the mesh is
248
+ * reconstructed from the depth map and carries no semantics.
249
+ */
250
+ meshGeometry?: {
251
+ vertices: ArrayBuffer;
252
+ faces: ArrayBuffer;
253
+ classifications?: ArrayBuffer;
254
+ };
187
255
  }
188
256
 
189
257
  /**
190
258
  * v0.8.0 — worklet function signature for the unified frame processor.
191
259
  *
192
260
  * Must be a `'worklet'`-prefixed function (so it can run on the
193
- * worklet runtime). Receives a `StitcherFrame` per camera frame; the
261
+ * worklet runtime). Receives a `CameraFrame` per camera frame; the
194
262
  * return value is ignored (use `runOnJS` / shared values to surface
195
263
  * results back to the JS thread).
196
264
  */
197
- export type StitcherFrameProcessor = (frame: StitcherFrame) => void;
265
+ export type CameraFrameProcessor = (frame: CameraFrame) => void;
@@ -126,18 +126,18 @@ import type {
126
126
  FrameProcessorPlugin,
127
127
  } from 'react-native-vision-camera';
128
128
 
129
- import type { StitcherFrame } from './StitcherFrame';
129
+ import type { CameraFrame } from './CameraFrame';
130
130
 
131
131
  /**
132
132
  * Frames the lib's stitching worklet accepts. Accepting either a
133
133
  * vc `Frame` (what the host's `useFrameProcessor` body sees) or the
134
- * lib's `StitcherFrame` (what the lib's `useFrameProcessor` body
134
+ * lib's `CameraFrame` (what the lib's `useFrameProcessor` body
135
135
  * sees) keeps the same `useStitcherWorklet` usable from both kinds
136
136
  * of host worklet bodies without a cast on the call site. The
137
137
  * worklet only reads `width` / `height`; the rest of the frame
138
138
  * object is forwarded verbatim to the native plugin.
139
139
  */
140
- export type StitcherWorkletInput = Frame | StitcherFrame;
140
+ export type StitcherWorkletInput = Frame | CameraFrame;
141
141
 
142
142
 
143
143
  export interface UseStitcherWorkletOptions {
@@ -173,7 +173,7 @@ export interface UseStitcherWorkletOptions {
173
173
 
174
174
  export interface StitcherWorkletHandle {
175
175
  /**
176
- * Worklet function: pass a `StitcherFrame` to perform one frame of
176
+ * Worklet function: pass a `CameraFrame` to perform one frame of
177
177
  * the lib's first-party stitching (throttle + pose synthesis +
178
178
  * native plugin call). Safe to call from inside another
179
179
  * `'worklet'`-prefixed function (this is the canonical
@@ -341,7 +341,7 @@ export function useStitcherWorklet(
341
341
  // party callback installed in `RNSARWorkletRuntime`). Calling
342
342
  // the vc Frame Processor plugin here would throw
343
343
  // `getPropertyAsObject: property '__frame' is undefined`
344
- // because AR frames are `StitcherFrameHostObject` instances
344
+ // because AR frames are `CameraFrameHostObject` instances
345
345
  // and don't carry the vc `Frame` proxy's JSI marker. The
346
346
  // throw is caught silently by the per-worklet error handler
347
347
  // (`RNSARWorkletRuntime.mm:284-301`) and bubbles up only to
@@ -353,13 +353,13 @@ export function useStitcherWorklet(
353
353
  // hook (the AR-side stitching path runs natively, independent
354
354
  // of the composed worklet body).
355
355
  //
356
- // The `(frame as StitcherFrame).source` cast is safe: vc
356
+ // The `(frame as CameraFrame).source` cast is safe: vc
357
357
  // `Frame` doesn't carry a `source` property so the check
358
358
  // returns `undefined !== 'ar'` → `true`, and the worklet
359
359
  // proceeds normally. Only frames that explicitly tag
360
360
  // themselves as AR-source (which our native AR dispatcher
361
- // does — see `StitcherFrameHostObject.mm`) get short-circuited.
362
- if ((frame as StitcherFrame).source === 'ar') return;
361
+ // does — see `CameraFrameHostObject.mm`) get short-circuited.
362
+ if ((frame as CameraFrame).source === 'ar') return;
363
363
 
364
364
  // Throttle (verbatim from useFrameProcessorDriver).
365
365
  sharedFrameCounter.value += 1;
@@ -388,7 +388,7 @@ export function useStitcherWorklet(
388
388
  const fy = h * sharedFyNumerator.value;
389
389
 
390
390
  // vc's `plugin.call` is typed against vc's `Frame`. The worklet
391
- // accepts the union (`Frame | StitcherFrame`); cast through
391
+ // accepts the union (`Frame | CameraFrame`); cast through
392
392
  // `unknown` because the union doesn't satisfy vc's interface
393
393
  // even though structurally both members do.
394
394
  plugin.call(frame as unknown as Frame, {
@@ -1,214 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // StitcherFrameHostObject.mm — iOS-specific wrapper for the shared
4
- // `retailens::StitcherFrameJsiHostObject` (defined in
5
- // `cpp/stitcher_frame_jsi.{hpp,cpp}`).
6
- //
7
- // Owns:
8
- // - The Obj-C facade callable from Swift / other Obj-C / .mm files.
9
- // - The iOS-specific `PixelBufferReader` impl (wraps a
10
- // `CVPixelBufferRef` from `ARFrame.capturedImage`; lock / memcpy
11
- // / unlock pattern).
12
- // - The Obj-C → C++ extraction logic that builds a
13
- // `retailens::StitcherFrameData` from an `ARFrame` + the lib's
14
- // `RNSARFramePose`.
15
- //
16
- // Does NOT own:
17
- // - The JSI `get` / `getPropertyNames` dispatch. That lives in
18
- // `cpp/stitcher_frame_jsi.cpp` and is identical to the Android
19
- // implementation (DRY across platforms).
20
-
21
- #import "StitcherFrameHostObject.h"
22
-
23
- #import <Foundation/Foundation.h>
24
- #import <CoreVideo/CVPixelBuffer.h>
25
- #import <CoreMedia/CoreMedia.h>
26
- #import <os/log.h>
27
-
28
- #include <jsi/jsi.h>
29
-
30
- #include <algorithm>
31
- #include <cstring>
32
- #include <memory>
33
- #include <string>
34
- #include <utility>
35
-
36
- #include "stitcher_frame_data.hpp"
37
- #include "stitcher_frame_jsi.hpp"
38
-
39
- using namespace facebook;
40
-
41
- // Forward-declare the Swift `RNSARFramePose` Obj-C surface we need.
42
- // This matches the pattern in `KeyframeGateFrameProcessor.mm`
43
- // (forward-declaring `IncrementalStitcher`) — avoids depending on
44
- // the autogenerated `RNImageStitcher-Swift.h`, which is created at
45
- // build time and not always available to .mm files in this pod.
46
- //
47
- // MUST stay in sync with `RNSARSession.swift::RNSARFramePose` —
48
- // adding a new field there means adding it here too.
49
- @class RNSARFramePose;
50
- @interface RNSARFramePose : NSObject
51
- @property (nonatomic, readonly) double tx;
52
- @property (nonatomic, readonly) double ty;
53
- @property (nonatomic, readonly) double tz;
54
- @property (nonatomic, readonly) double qx;
55
- @property (nonatomic, readonly) double qy;
56
- @property (nonatomic, readonly) double qz;
57
- @property (nonatomic, readonly) double qw;
58
- @property (nonatomic, readonly) NSInteger imageWidth;
59
- @property (nonatomic, readonly) NSInteger imageHeight;
60
- @property (nonatomic, readonly) double timestampMs;
61
- @end
62
-
63
- #pragma mark - iOS PixelBufferReader
64
-
65
- namespace {
66
-
67
- /// iOS-specific `retailens::PixelBufferReader` impl. See the base
68
- /// class docstring for the general contract (thread-affinity,
69
- /// invalidation semantics, Y-plane-only constraint). This subclass
70
- /// adds:
71
- /// - `CVPixelBuffer` lock/memcpy/unlock per copyTo
72
- /// - `CFBridgingRetain` of the parent `ARFrame` so ARKit's
73
- /// pool can't reclaim the underlying buffer mid-read
74
- class IOSPixelBufferReader : public retailens::PixelBufferReader {
75
- public:
76
- explicit IOSPixelBufferReader(ARFrame* arFrame) {
77
- // Retain the ARFrame for our lifetime. CFBridgingRetain hands
78
- // ARC ownership to our void*. Released in destructor.
79
- _retainedFrame = (void*)CFBridgingRetain(arFrame);
80
- CVPixelBufferRef pixelBuffer = arFrame.capturedImage;
81
- if (pixelBuffer != NULL) {
82
- _bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
83
- _height = CVPixelBufferGetHeight(pixelBuffer);
84
- }
85
- }
86
-
87
- ~IOSPixelBufferReader() override {
88
- // Transfer ownership back to ARC, which then releases.
89
- if (_retainedFrame != nullptr) {
90
- ARFrame* frame = CFBridgingRelease(_retainedFrame);
91
- (void)frame;
92
- _retainedFrame = nullptr;
93
- }
94
- }
95
-
96
- std::size_t byteSize() const override {
97
- return _bytesPerRow * _height;
98
- }
99
-
100
- std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) override {
101
- if (_retainedFrame == nullptr) return 0;
102
- ARFrame* frame = (__bridge ARFrame*)_retainedFrame;
103
- CVPixelBufferRef pixelBuffer = frame.capturedImage;
104
- if (pixelBuffer == NULL) return 0;
105
-
106
- CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
107
- const uint8_t* src = (const uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
108
- std::size_t toCopy = std::min<std::size_t>(byteSize(), maxBytes);
109
- if (src != nullptr && toCopy > 0) {
110
- std::memcpy(dst, src, toCopy);
111
- } else {
112
- toCopy = 0;
113
- }
114
- CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
115
- return toCopy;
116
- }
117
-
118
- private:
119
- void* _retainedFrame = nullptr; // CFBridgingRetain'd ARFrame
120
- std::size_t _bytesPerRow = 0;
121
- std::size_t _height = 0;
122
- };
123
-
124
- } // anonymous namespace
125
-
126
- #pragma mark - Obj-C facade
127
-
128
- @implementation StitcherFrameHostObject {
129
- std::shared_ptr<retailens::StitcherFrameJsiHostObject> _hostObject;
130
- }
131
-
132
- + (instancetype)fromARFrame:(ARFrame*)arFrame pose:(RNSARFramePose*)pose {
133
- StitcherFrameHostObject* obj = [[self alloc] init];
134
-
135
- retailens::StitcherFrameData data;
136
- data.source = "ar";
137
- data.width = static_cast<int32_t>(pose.imageWidth);
138
- data.height = static_cast<int32_t>(pose.imageHeight);
139
- // ARKit's `kCVPixelFormatType_420YpCbCr8BiPlanarFullRange` (NV12)
140
- // is reported as "yuv". Other formats (rare in ARKit; possible if
141
- // ARWorldTrackingConfiguration.videoFormat is overridden to BGRA)
142
- // → "unknown" + os_log warning so worklets that gate on
143
- // `pixelFormat === 'yuv'` can be debugged without a screen recording.
144
- OSType pf = CVPixelBufferGetPixelFormatType(arFrame.capturedImage);
145
- if (pf == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
146
- pf == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
147
- data.pixelFormat = "yuv";
148
- } else {
149
- data.pixelFormat = "unknown";
150
- os_log_error(OS_LOG_DEFAULT,
151
- "[StitcherFrame] unexpected ARKit pixel format 0x%x; "
152
- "worklet receives pixelFormat='unknown' and toArrayBuffer() "
153
- "bytes are first-plane only (layout undefined for unknown "
154
- "formats). See StitcherFrame.ts docstring.", (unsigned int)pf);
155
- }
156
- // ARKit doesn't have a `Frame.orientation` per se; pose carries
157
- // the imageWidth >= imageHeight discriminator the lib uses
158
- // elsewhere (`isLandscape`). v0.8.0 ships a coarse mapping;
159
- // worklets that need exact UI orientation can read it from
160
- // device-orientation sensors.
161
- data.orientation =
162
- (pose.imageWidth >= pose.imageHeight) ? "landscape-right" : "portrait";
163
- // `ARFrame.timestamp` is CFAbsoluteTime (seconds since epoch).
164
- // Convert to ns to match vc Frame.timestamp.
165
- data.timestampNs = arFrame.timestamp * 1e9;
166
-
167
- data.qx = pose.qx;
168
- data.qy = pose.qy;
169
- data.qz = pose.qz;
170
- data.qw = pose.qw;
171
- data.tx = pose.tx;
172
- data.ty = pose.ty;
173
- data.tz = pose.tz;
174
- data.hasTranslation = true; // AR mode always has translation
175
-
176
- switch (arFrame.camera.trackingState) {
177
- case ARTrackingStateNotAvailable:
178
- data.arTrackingState = "notAvailable";
179
- break;
180
- case ARTrackingStateLimited:
181
- data.arTrackingState = "limited";
182
- break;
183
- case ARTrackingStateNormal:
184
- data.arTrackingState = "normal";
185
- break;
186
- }
187
-
188
- data.pixelReader = std::make_shared<IOSPixelBufferReader>(arFrame);
189
-
190
- // Use the static factory (private ctor enforces shared_ptr
191
- // ownership — required for `shared_from_this()` inside the JSI
192
- // `toArrayBuffer` lambda).
193
- obj->_hostObject =
194
- retailens::StitcherFrameJsiHostObject::create(std::move(data));
195
- return obj;
196
- }
197
-
198
- - (void)invalidate {
199
- if (_hostObject) {
200
- _hostObject->invalidate();
201
- }
202
- }
203
-
204
- - (void*)jsiHostObjectPtr {
205
- if (!_hostObject) return NULL;
206
- // Box a heap-allocated copy of the shared_ptr to the abstract
207
- // `jsi::HostObject` base. Caller (worklet runtime) does:
208
- // auto sp = static_cast<std::shared_ptr<jsi::HostObject>*>(ptr);
209
- // auto jsObj = jsi::Object::createFromHostObject(rt, *sp);
210
- // delete sp;
211
- return new std::shared_ptr<jsi::HostObject>(_hostObject);
212
- }
213
-
214
- @end