react-native-image-stitcher 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +121 -0
- package/RNImageStitcher.podspec +1 -1
- package/android/src/main/cpp/CMakeLists.txt +4 -4
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +216 -7
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +656 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +156 -0
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +1 -1
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +84 -2
- package/cpp/{stitcher_frame_data.hpp → camera_frame_data.hpp} +96 -13
- package/cpp/{stitcher_frame_jsi.cpp → camera_frame_jsi.cpp} +154 -11
- package/cpp/{stitcher_frame_jsi.hpp → camera_frame_jsi.hpp} +12 -12
- package/cpp/stitcher_proxy_jsi.cpp +31 -0
- package/cpp/stitcher_proxy_jsi.hpp +16 -0
- package/cpp/stitcher_worklet_dispatch.cpp +5 -5
- package/cpp/stitcher_worklet_dispatch.hpp +5 -5
- package/dist/camera/ARCameraView.d.ts +60 -3
- package/dist/camera/ARCameraView.js +68 -1
- package/dist/camera/Camera.d.ts +54 -7
- package/dist/camera/Camera.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/stitching/ARFrameMeta.d.ts +100 -0
- package/dist/stitching/{StitcherFrame.js → ARFrameMeta.js} +1 -1
- package/dist/stitching/{StitcherFrame.d.ts → CameraFrame.d.ts} +70 -11
- package/dist/stitching/CameraFrame.js +4 -0
- package/dist/stitching/useStitcherWorklet.d.ts +4 -4
- package/dist/stitching/useStitcherWorklet.js +4 -4
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +23 -1
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +137 -2
- package/ios/Sources/RNImageStitcher/{StitcherFrameHostObject.h → CameraFrameHostObject.h} +26 -3
- package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm +760 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +292 -34
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +2 -2
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +4 -4
- package/package.json +1 -1
- package/src/camera/ARCameraView.tsx +165 -5
- package/src/camera/Camera.tsx +69 -7
- package/src/index.ts +7 -3
- package/src/stitching/ARFrameMeta.ts +107 -0
- package/src/stitching/{StitcherFrame.ts → CameraFrame.ts} +79 -11
- package/src/stitching/useStitcherWorklet.ts +9 -9
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
package/src/camera/Camera.tsx
CHANGED
|
@@ -66,7 +66,8 @@ import type {
|
|
|
66
66
|
} from 'react-native-vision-camera';
|
|
67
67
|
|
|
68
68
|
import { useARSession } from '../ar/useARSession';
|
|
69
|
-
import type {
|
|
69
|
+
import type { CameraFrameProcessor } from '../stitching/CameraFrame';
|
|
70
|
+
import type { ARFrameMeta } from '../stitching/ARFrameMeta';
|
|
70
71
|
import { ARCameraView, type ARCameraViewHandle } from './ARCameraView';
|
|
71
72
|
import { CameraShutter } from './CameraShutter';
|
|
72
73
|
import { CameraView } from './CameraView';
|
|
@@ -680,11 +681,11 @@ export interface CameraProps {
|
|
|
680
681
|
* worklet to fire on vc's Frame Processor runtime.
|
|
681
682
|
*
|
|
682
683
|
* ```tsx
|
|
683
|
-
* import { Camera, useFrameProcessor, type
|
|
684
|
+
* import { Camera, useFrameProcessor, type CameraFrame }
|
|
684
685
|
* from 'react-native-image-stitcher';
|
|
685
686
|
*
|
|
686
687
|
* function MyScreen() {
|
|
687
|
-
* const fp = useFrameProcessor((frame:
|
|
688
|
+
* const fp = useFrameProcessor((frame: CameraFrame) => {
|
|
688
689
|
* 'worklet';
|
|
689
690
|
* // ...
|
|
690
691
|
* }, []);
|
|
@@ -705,12 +706,12 @@ export interface CameraProps {
|
|
|
705
706
|
* ```tsx
|
|
706
707
|
* import {
|
|
707
708
|
* Camera, useFrameProcessor, useStitcherWorklet,
|
|
708
|
-
* type
|
|
709
|
+
* type CameraFrame,
|
|
709
710
|
* } from 'react-native-image-stitcher';
|
|
710
711
|
*
|
|
711
712
|
* function MyScreen() {
|
|
712
713
|
* const stitcher = useStitcherWorklet();
|
|
713
|
-
* const fp = useFrameProcessor((frame:
|
|
714
|
+
* const fp = useFrameProcessor((frame: CameraFrame) => {
|
|
714
715
|
* 'worklet';
|
|
715
716
|
* hostPreLogic(frame);
|
|
716
717
|
* stitcher.call(frame); // ← first-party stitching
|
|
@@ -755,14 +756,63 @@ export interface CameraProps {
|
|
|
755
756
|
/**
|
|
756
757
|
* AR-mode host worklet, invoked once per ARKit / ARCore frame
|
|
757
758
|
* ALONGSIDE the lib's first-party stitching (composition, not
|
|
758
|
-
* replacement). Receives a `
|
|
759
|
+
* replacement). Receives a `CameraFrame` tagged `source: 'ar'`
|
|
759
760
|
* with world-space `pose` + `arTrackingState`. Only fires in AR
|
|
760
761
|
* capture (`captureSource === 'ar'`); the non-AR equivalent is
|
|
761
762
|
* `frameProcessor` above (the two modes use different runtimes and
|
|
762
763
|
* frame shapes). Must be a `'worklet'`-prefixed function; if the
|
|
763
764
|
* native install is unavailable it silently never fires.
|
|
764
765
|
*/
|
|
765
|
-
arFrameProcessor?:
|
|
766
|
+
arFrameProcessor?: CameraFrameProcessor;
|
|
767
|
+
|
|
768
|
+
/**
|
|
769
|
+
* Opt in to per-frame AR depth on the `arFrameProcessor` frame
|
|
770
|
+
* (`CameraFrame.arDepth`). Default `false` — depth is the costliest
|
|
771
|
+
* field (a per-frame buffer copy), so it's off until you need it.
|
|
772
|
+
*/
|
|
773
|
+
enableDepth?: boolean;
|
|
774
|
+
/**
|
|
775
|
+
* Opt in to per-frame AR anchors (`CameraFrame.arAnchors` — detected
|
|
776
|
+
* planes / images). Default `false`.
|
|
777
|
+
*/
|
|
778
|
+
enableAnchors?: boolean;
|
|
779
|
+
/**
|
|
780
|
+
* Opt in to scene-reconstruction mesh anchors (`type: 'mesh'` in
|
|
781
|
+
* `arAnchors`, with `meshGeometry`). Default `false`. iOS enables
|
|
782
|
+
* ARKit `sceneReconstruction` (LiDAR); Android reconstructs a rough
|
|
783
|
+
* mesh from the depth map. Expensive — only on when needed.
|
|
784
|
+
*/
|
|
785
|
+
enableMesh?: boolean;
|
|
786
|
+
/**
|
|
787
|
+
* Which plane orientations to surface in `CameraFrame.arAnchors`
|
|
788
|
+
* (requires `enableAnchors`; AR capture only). Default `'vertical'`
|
|
789
|
+
* — the orientation the plane-projected stitch path has always used.
|
|
790
|
+
* `'horizontal'` surfaces floors / tables; `'both'` surfaces every
|
|
791
|
+
* detected plane. See `ARCameraView` for the per-platform details.
|
|
792
|
+
*/
|
|
793
|
+
planeDetection?: 'vertical' | 'horizontal' | 'both';
|
|
794
|
+
|
|
795
|
+
/**
|
|
796
|
+
* v0.18.0 — LIGHT per-frame AR metadata callback, invoked on the JS
|
|
797
|
+
* MAIN thread (NOT a worklet). Only fires in AR capture
|
|
798
|
+
* (`captureSource === 'ar'`). Receives an {@link ARFrameMeta} carrying
|
|
799
|
+
* pose, tracking state, intrinsics, and (when the matching `enable*`
|
|
800
|
+
* prop is on) depth dimensions, anchors, and mesh counts.
|
|
801
|
+
*
|
|
802
|
+
* This is the recommended way to read AR metadata: it sidesteps the
|
|
803
|
+
* worklet path entirely (the `arFrameProcessor` worklet can only safely
|
|
804
|
+
* surface a worklets-core shared value, because capturing a host
|
|
805
|
+
* callback crashes the worklet closure-wrap). Native builds the meta
|
|
806
|
+
* and emits a device event; `<Camera>` threads the handler through to
|
|
807
|
+
* `<ARCameraView>`, which subscribes and invokes it on the main thread.
|
|
808
|
+
*/
|
|
809
|
+
onArFrame?: (meta: ARFrameMeta) => void;
|
|
810
|
+
|
|
811
|
+
/**
|
|
812
|
+
* v0.18.0 — throttle interval (ms) for {@link onArFrame}. Default `100`
|
|
813
|
+
* (≈ 10 Hz). No effect unless `onArFrame` is provided.
|
|
814
|
+
*/
|
|
815
|
+
arFrameMetaInterval?: number;
|
|
766
816
|
|
|
767
817
|
// ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────────
|
|
768
818
|
/**
|
|
@@ -1171,6 +1221,12 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
1171
1221
|
onCapturePreviewClose,
|
|
1172
1222
|
frameProcessor: hostFrameProcessor,
|
|
1173
1223
|
arFrameProcessor,
|
|
1224
|
+
enableDepth,
|
|
1225
|
+
enableAnchors,
|
|
1226
|
+
enableMesh,
|
|
1227
|
+
planeDetection,
|
|
1228
|
+
onArFrame,
|
|
1229
|
+
arFrameMetaInterval,
|
|
1174
1230
|
engine = 'batch-keyframe',
|
|
1175
1231
|
// ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
|
|
1176
1232
|
panMode = 'vertical',
|
|
@@ -2435,6 +2491,12 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
2435
2491
|
ref={arViewRef}
|
|
2436
2492
|
style={StyleSheet.absoluteFill}
|
|
2437
2493
|
arFrameProcessor={arFrameProcessor}
|
|
2494
|
+
enableDepth={enableDepth}
|
|
2495
|
+
enableAnchors={enableAnchors}
|
|
2496
|
+
enableMesh={enableMesh}
|
|
2497
|
+
planeDetection={planeDetection}
|
|
2498
|
+
onArFrame={onArFrame}
|
|
2499
|
+
arFrameMetaInterval={arFrameMetaInterval}
|
|
2438
2500
|
/>
|
|
2439
2501
|
) : (
|
|
2440
2502
|
<CameraView
|
package/src/index.ts
CHANGED
|
@@ -259,10 +259,14 @@ export { useKeyframeStream } from './stitching/useKeyframeStream';
|
|
|
259
259
|
// v0.8.0 — unified frame contract for the worklet processor. Same
|
|
260
260
|
// JS-visible shape regardless of capture mode (AR vs non-AR).
|
|
261
261
|
export type {
|
|
262
|
-
|
|
263
|
-
|
|
262
|
+
CameraFrame,
|
|
263
|
+
CameraFrameProcessor,
|
|
264
264
|
ARAnchor,
|
|
265
|
-
} from './stitching/
|
|
265
|
+
} from './stitching/CameraFrame';
|
|
266
|
+
// v0.18.0 — LIGHT per-frame AR metadata delivered via the `onArFrame`
|
|
267
|
+
// callback (main-thread, worklet-free). See the type's docstring for why
|
|
268
|
+
// it bypasses the worklet path.
|
|
269
|
+
export type { ARFrameMeta } from './stitching/ARFrameMeta';
|
|
266
270
|
// NOTE: the host-worklet / frame-stream hooks `useFrameProcessor`,
|
|
267
271
|
// `useThrottledFrameProcessor` and `useFrameStream` (v0.8–v0.9) were
|
|
268
272
|
// archived in the batch-keyframe cleanup — they drove the third-party
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* v0.18.0 — LIGHT per-frame AR metadata delivered to JS on the MAIN
|
|
5
|
+
* thread via a normal callback (`onArFrame`), bypassing worklets entirely.
|
|
6
|
+
*
|
|
7
|
+
* ## Why a callback, NOT a worklet
|
|
8
|
+
*
|
|
9
|
+
* The AR worklet path (`arFrameProcessor` + the `__stitcherProxy` JSI
|
|
10
|
+
* registry) deep-copies the worklet's whole closure into the AR worklet
|
|
11
|
+
* runtime via react-native-worklets-core's `WorkletInvoker`. When the
|
|
12
|
+
* worklet captures a host object (e.g. a `createRunOnJS` callback) that
|
|
13
|
+
* closure-wrap recurses without termination → stack overflow → SIGBUS the
|
|
14
|
+
* instant AR mode mounts (verified on device). Worklets can therefore
|
|
15
|
+
* only safely capture a worklets-core *shared value* — an awkward,
|
|
16
|
+
* poll-from-JS pattern for getting structured data back.
|
|
17
|
+
*
|
|
18
|
+
* `onArFrame` sidesteps the whole problem: native builds the metadata and
|
|
19
|
+
* emits it as a plain `RNImageStitcherARFrame` device event carrying a
|
|
20
|
+
* JSON object; the JS side subscribes via `NativeEventEmitter` and invokes
|
|
21
|
+
* the host callback on the main thread. No worklet, no closure-wrap, no
|
|
22
|
+
* shared-value polling. This is the recommended way to read AR metadata.
|
|
23
|
+
*
|
|
24
|
+
* ## Cost / gating
|
|
25
|
+
*
|
|
26
|
+
* The metadata is intentionally LIGHT — no pixel / vertex / face byte
|
|
27
|
+
* marshaling. `depth` reports only the depth map's dimensions + whether a
|
|
28
|
+
* confidence channel exists (no buffer copy); `mesh` reports only anchor /
|
|
29
|
+
* vertex / face *counts*. Native gates each costly field on the matching
|
|
30
|
+
* extraction flag (`depth` ⇐ `enableDepth`, `mesh` ⇐ `enableMesh`,
|
|
31
|
+
* `anchors` ⇐ `enableAnchors`); `intrinsics` / `pose` / `trackingState` are
|
|
32
|
+
* always present. Emission is gated on a TS-set enabled flag (only true
|
|
33
|
+
* when `onArFrame` is provided) and throttled to `arFrameMetaInterval` ms
|
|
34
|
+
* (default 100 ≈ 10 Hz) on the native side.
|
|
35
|
+
*/
|
|
36
|
+
export interface ARFrameMeta {
|
|
37
|
+
/** Frame-capture timestamp in NANOSECONDS (AR-framework monotonic clock). */
|
|
38
|
+
timestamp: number;
|
|
39
|
+
|
|
40
|
+
/** AR tracking quality at this frame. */
|
|
41
|
+
trackingState: 'notAvailable' | 'limited' | 'normal';
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Camera pose in world coordinates at frame-capture time.
|
|
45
|
+
*
|
|
46
|
+
* - `rotation` — quaternion `(x, y, z, w)`, matching the convention
|
|
47
|
+
* used throughout the engine + the `CameraFrame.pose` field.
|
|
48
|
+
* - `translation` — metres in world space `[x, y, z]`.
|
|
49
|
+
*/
|
|
50
|
+
pose: {
|
|
51
|
+
rotation: [number, number, number, number];
|
|
52
|
+
translation: [number, number, number];
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Camera intrinsics for this frame — focal lengths (`fx`, `fy`) and
|
|
57
|
+
* principal point (`cx`, `cy`) in PIXELS at the `imageWidth × imageHeight`
|
|
58
|
+
* capture resolution. Always attempted (not gated); `null` only when the
|
|
59
|
+
* AR framework didn't provide them for this frame.
|
|
60
|
+
*/
|
|
61
|
+
intrinsics: {
|
|
62
|
+
fx: number;
|
|
63
|
+
fy: number;
|
|
64
|
+
cx: number;
|
|
65
|
+
cy: number;
|
|
66
|
+
imageWidth: number;
|
|
67
|
+
imageHeight: number;
|
|
68
|
+
} | null;
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Depth-map summary — dimensions + whether a per-pixel confidence channel
|
|
72
|
+
* is available. NO pixel buffer is copied (that's the costly part).
|
|
73
|
+
* Present only when the `enableDepth` prop is on AND the device produced a
|
|
74
|
+
* depth map this frame; `null` otherwise.
|
|
75
|
+
*/
|
|
76
|
+
depth: {
|
|
77
|
+
width: number;
|
|
78
|
+
height: number;
|
|
79
|
+
hasConfidence: boolean;
|
|
80
|
+
} | null;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Tracked AR anchors visible in this frame (planes / images / points /
|
|
84
|
+
* mesh). Empty array when `enableAnchors` is on but nothing is tracked;
|
|
85
|
+
* effectively empty when `enableAnchors` is off. `transform` is a 4×4
|
|
86
|
+
* row-major anchor→world matrix (16 numbers).
|
|
87
|
+
*/
|
|
88
|
+
anchors: Array<{
|
|
89
|
+
id: string;
|
|
90
|
+
type: 'plane' | 'image' | 'point' | 'mesh';
|
|
91
|
+
alignment?: 'horizontal' | 'vertical';
|
|
92
|
+
extent?: [number, number];
|
|
93
|
+
classification?: string;
|
|
94
|
+
transform: number[];
|
|
95
|
+
}>;
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Scene-reconstruction mesh summary — anchor / vertex / face COUNTS only
|
|
99
|
+
* (no vertex / face byte marshaling). Present only when `enableMesh` is
|
|
100
|
+
* on; `null` otherwise.
|
|
101
|
+
*/
|
|
102
|
+
mesh: {
|
|
103
|
+
anchorCount: number;
|
|
104
|
+
vertexCount: number;
|
|
105
|
+
faceCount: number;
|
|
106
|
+
} | null;
|
|
107
|
+
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* v0.8.0 — unified frame contract for the lib's worklet processor.
|
|
5
5
|
*
|
|
6
6
|
* Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
|
|
7
|
-
* this directory) receive a `
|
|
7
|
+
* this directory) receive a `CameraFrame` regardless of capture mode.
|
|
8
8
|
* The lib-owned worklet runtime guarantees the same JS-visible shape
|
|
9
9
|
* whether the underlying source is a vision-camera `Frame` (non-AR
|
|
10
10
|
* mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
|
|
@@ -18,10 +18,10 @@
|
|
|
18
18
|
* (Phase-0 audit confirmed the iOS path). But vision-camera's
|
|
19
19
|
* **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
|
|
20
20
|
* ARCore does NOT produce `ImageProxy` instances. Forcing
|
|
21
|
-
* `
|
|
21
|
+
* `CameraFrame extends Frame` would either (a) require reverse-
|
|
22
22
|
* engineering ImageProxy on Android (intractable + fragile), or
|
|
23
23
|
* (b) make the type asymmetric per platform. Both are worse than
|
|
24
|
-
* making `
|
|
24
|
+
* making `CameraFrame` a structural sibling type that vc Frames
|
|
25
25
|
* happen to satisfy (because vc Frames carry the same width / height /
|
|
26
26
|
* orientation / pixelFormat / timestamp / toArrayBuffer surface).
|
|
27
27
|
*
|
|
@@ -38,10 +38,10 @@
|
|
|
38
38
|
* a JPEG-encode frame-processor plugin). Returning a reference and
|
|
39
39
|
* reading it later will read into freed memory.
|
|
40
40
|
*/
|
|
41
|
-
export interface
|
|
41
|
+
export interface CameraFrame {
|
|
42
42
|
// ── vision-camera-shaped fields (structural compat) ─────────────
|
|
43
43
|
// Worklets written against a vc `Frame` work unchanged against a
|
|
44
|
-
// `
|
|
44
|
+
// `CameraFrame` (the fields below are a strict subset of vc
|
|
45
45
|
// Frame's JS-visible surface).
|
|
46
46
|
|
|
47
47
|
/** Pixel width of the camera image. */
|
|
@@ -167,6 +167,28 @@ export interface StitcherFrame {
|
|
|
167
167
|
* tracking is degraded check this. Undefined in non-AR mode.
|
|
168
168
|
*/
|
|
169
169
|
arTrackingState?: 'notAvailable' | 'limited' | 'normal';
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Camera intrinsics for THIS frame — focal lengths (`fx`,`fy`) and
|
|
173
|
+
* principal point (`cx`,`cy`) in PIXELS at the `imageWidth × imageHeight`
|
|
174
|
+
* capture resolution. Needed to lift 2D image-space coordinates to 3D
|
|
175
|
+
* via pose + intrinsics (e.g. object-level reconstruction).
|
|
176
|
+
*
|
|
177
|
+
* Populated on **AR frames** (`source: 'ar'`) from ARKit
|
|
178
|
+
* `ARCamera.intrinsics` / ARCore `Camera` intrinsics. **Undefined for
|
|
179
|
+
* non-AR (vision-camera) frames** — they are raw vc `Frame`s without an
|
|
180
|
+
* intrinsics surface; read vc's own APIs there if needed. (The spec
|
|
181
|
+
* called this required; it's optional here because the non-AR frame
|
|
182
|
+
* shape genuinely can't carry it.)
|
|
183
|
+
*/
|
|
184
|
+
intrinsics?: {
|
|
185
|
+
fx: number;
|
|
186
|
+
fy: number;
|
|
187
|
+
cx: number;
|
|
188
|
+
cy: number;
|
|
189
|
+
imageWidth: number;
|
|
190
|
+
imageHeight: number;
|
|
191
|
+
};
|
|
170
192
|
}
|
|
171
193
|
|
|
172
194
|
/**
|
|
@@ -177,21 +199,67 @@ export interface StitcherFrame {
|
|
|
177
199
|
export interface ARAnchor {
|
|
178
200
|
/** Stable per-session anchor identifier. */
|
|
179
201
|
id: string;
|
|
180
|
-
/** Anchor kind. `'point'` is Android (ARCore) only. */
|
|
181
|
-
type: 'plane' | 'image' | 'point';
|
|
182
202
|
/**
|
|
183
|
-
*
|
|
184
|
-
*
|
|
203
|
+
* Anchor kind. `'point'` is Android (ARCore) only; `'mesh'` is a
|
|
204
|
+
* scene-reconstruction mesh anchor, present only when the `enableMesh`
|
|
205
|
+
* `<Camera>` prop is on (and the device supports reconstruction).
|
|
206
|
+
*/
|
|
207
|
+
type: 'plane' | 'image' | 'point' | 'mesh';
|
|
208
|
+
/**
|
|
209
|
+
* 4×4 row-major transform from anchor space to world space (16
|
|
210
|
+
* numbers). For `'mesh'` anchors, the `meshGeometry.vertices` are in
|
|
211
|
+
* this anchor's LOCAL space — multiply by `transform` for world coords.
|
|
185
212
|
*/
|
|
186
213
|
transform: number[];
|
|
214
|
+
/**
|
|
215
|
+
* Plane orientation — `'horizontal'` (floor / table / seat) vs
|
|
216
|
+
* `'vertical'` (wall / door / window). Present on `'plane'` anchors;
|
|
217
|
+
* undefined for other anchor kinds. Lets a host distinguish a shelf
|
|
218
|
+
* surface from the wall behind it.
|
|
219
|
+
*/
|
|
220
|
+
alignment?: 'horizontal' | 'vertical';
|
|
221
|
+
/**
|
|
222
|
+
* Plane size in metres along its local x / z axes (`[x, z]`). Present
|
|
223
|
+
* on `'plane'` anchors only.
|
|
224
|
+
*/
|
|
225
|
+
extent?: [number, number];
|
|
226
|
+
/**
|
|
227
|
+
* ARKit semantic classification of the plane's surface, when the
|
|
228
|
+
* framework provides it (iOS; mostly horizontal planes). Undefined
|
|
229
|
+
* when unknown / unsupported (incl. Android, which has no equivalent).
|
|
230
|
+
*/
|
|
231
|
+
classification?:
|
|
232
|
+
| 'wall'
|
|
233
|
+
| 'floor'
|
|
234
|
+
| 'ceiling'
|
|
235
|
+
| 'table'
|
|
236
|
+
| 'seat'
|
|
237
|
+
| 'door'
|
|
238
|
+
| 'window'
|
|
239
|
+
| 'none';
|
|
240
|
+
/**
|
|
241
|
+
* Scene-reconstruction geometry — present only on `type: 'mesh'`
|
|
242
|
+
* anchors. Buffers (wrap in the noted typed-array view):
|
|
243
|
+
* - `vertices` → `Float32Array`, xyz triplets in anchor-local space.
|
|
244
|
+
* - `faces` → `Uint32Array`, triangle indices into `vertices`.
|
|
245
|
+
* - `classifications` → optional `Uint8Array`, one ARKit mesh class
|
|
246
|
+
* per face (0=none, 1=wall, 2=floor, 3=ceiling, …). **iOS only**
|
|
247
|
+
* (from `ARMeshAnchor`); absent on Android, where the mesh is
|
|
248
|
+
* reconstructed from the depth map and carries no semantics.
|
|
249
|
+
*/
|
|
250
|
+
meshGeometry?: {
|
|
251
|
+
vertices: ArrayBuffer;
|
|
252
|
+
faces: ArrayBuffer;
|
|
253
|
+
classifications?: ArrayBuffer;
|
|
254
|
+
};
|
|
187
255
|
}
|
|
188
256
|
|
|
189
257
|
/**
|
|
190
258
|
* v0.8.0 — worklet function signature for the unified frame processor.
|
|
191
259
|
*
|
|
192
260
|
* Must be a `'worklet'`-prefixed function (so it can run on the
|
|
193
|
-
* worklet runtime). Receives a `
|
|
261
|
+
* worklet runtime). Receives a `CameraFrame` per camera frame; the
|
|
194
262
|
* return value is ignored (use `runOnJS` / shared values to surface
|
|
195
263
|
* results back to the JS thread).
|
|
196
264
|
*/
|
|
197
|
-
export type
|
|
265
|
+
export type CameraFrameProcessor = (frame: CameraFrame) => void;
|
|
@@ -126,18 +126,18 @@ import type {
|
|
|
126
126
|
FrameProcessorPlugin,
|
|
127
127
|
} from 'react-native-vision-camera';
|
|
128
128
|
|
|
129
|
-
import type {
|
|
129
|
+
import type { CameraFrame } from './CameraFrame';
|
|
130
130
|
|
|
131
131
|
/**
|
|
132
132
|
* Frames the lib's stitching worklet accepts. Accepting either a
|
|
133
133
|
* vc `Frame` (what the host's `useFrameProcessor` body sees) or the
|
|
134
|
-
* lib's `
|
|
134
|
+
* lib's `CameraFrame` (what the lib's `useFrameProcessor` body
|
|
135
135
|
* sees) keeps the same `useStitcherWorklet` usable from both kinds
|
|
136
136
|
* of host worklet bodies without a cast on the call site. The
|
|
137
137
|
* worklet only reads `width` / `height`; the rest of the frame
|
|
138
138
|
* object is forwarded verbatim to the native plugin.
|
|
139
139
|
*/
|
|
140
|
-
export type StitcherWorkletInput = Frame |
|
|
140
|
+
export type StitcherWorkletInput = Frame | CameraFrame;
|
|
141
141
|
|
|
142
142
|
|
|
143
143
|
export interface UseStitcherWorkletOptions {
|
|
@@ -173,7 +173,7 @@ export interface UseStitcherWorkletOptions {
|
|
|
173
173
|
|
|
174
174
|
export interface StitcherWorkletHandle {
|
|
175
175
|
/**
|
|
176
|
-
* Worklet function: pass a `
|
|
176
|
+
* Worklet function: pass a `CameraFrame` to perform one frame of
|
|
177
177
|
* the lib's first-party stitching (throttle + pose synthesis +
|
|
178
178
|
* native plugin call). Safe to call from inside another
|
|
179
179
|
* `'worklet'`-prefixed function (this is the canonical
|
|
@@ -341,7 +341,7 @@ export function useStitcherWorklet(
|
|
|
341
341
|
// party callback installed in `RNSARWorkletRuntime`). Calling
|
|
342
342
|
// the vc Frame Processor plugin here would throw
|
|
343
343
|
// `getPropertyAsObject: property '__frame' is undefined`
|
|
344
|
-
// because AR frames are `
|
|
344
|
+
// because AR frames are `CameraFrameHostObject` instances
|
|
345
345
|
// and don't carry the vc `Frame` proxy's JSI marker. The
|
|
346
346
|
// throw is caught silently by the per-worklet error handler
|
|
347
347
|
// (`RNSARWorkletRuntime.mm:284-301`) and bubbles up only to
|
|
@@ -353,13 +353,13 @@ export function useStitcherWorklet(
|
|
|
353
353
|
// hook (the AR-side stitching path runs natively, independent
|
|
354
354
|
// of the composed worklet body).
|
|
355
355
|
//
|
|
356
|
-
// The `(frame as
|
|
356
|
+
// The `(frame as CameraFrame).source` cast is safe: vc
|
|
357
357
|
// `Frame` doesn't carry a `source` property so the check
|
|
358
358
|
// returns `undefined !== 'ar'` → `true`, and the worklet
|
|
359
359
|
// proceeds normally. Only frames that explicitly tag
|
|
360
360
|
// themselves as AR-source (which our native AR dispatcher
|
|
361
|
-
// does — see `
|
|
362
|
-
if ((frame as
|
|
361
|
+
// does — see `CameraFrameHostObject.mm`) get short-circuited.
|
|
362
|
+
if ((frame as CameraFrame).source === 'ar') return;
|
|
363
363
|
|
|
364
364
|
// Throttle (verbatim from useFrameProcessorDriver).
|
|
365
365
|
sharedFrameCounter.value += 1;
|
|
@@ -388,7 +388,7 @@ export function useStitcherWorklet(
|
|
|
388
388
|
const fy = h * sharedFyNumerator.value;
|
|
389
389
|
|
|
390
390
|
// vc's `plugin.call` is typed against vc's `Frame`. The worklet
|
|
391
|
-
// accepts the union (`Frame |
|
|
391
|
+
// accepts the union (`Frame | CameraFrame`); cast through
|
|
392
392
|
// `unknown` because the union doesn't satisfy vc's interface
|
|
393
393
|
// even though structurally both members do.
|
|
394
394
|
plugin.call(frame as unknown as Frame, {
|
|
@@ -1,214 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
-
//
|
|
3
|
-
// StitcherFrameHostObject.mm — iOS-specific wrapper for the shared
|
|
4
|
-
// `retailens::StitcherFrameJsiHostObject` (defined in
|
|
5
|
-
// `cpp/stitcher_frame_jsi.{hpp,cpp}`).
|
|
6
|
-
//
|
|
7
|
-
// Owns:
|
|
8
|
-
// - The Obj-C facade callable from Swift / other Obj-C / .mm files.
|
|
9
|
-
// - The iOS-specific `PixelBufferReader` impl (wraps a
|
|
10
|
-
// `CVPixelBufferRef` from `ARFrame.capturedImage`; lock / memcpy
|
|
11
|
-
// / unlock pattern).
|
|
12
|
-
// - The Obj-C → C++ extraction logic that builds a
|
|
13
|
-
// `retailens::StitcherFrameData` from an `ARFrame` + the lib's
|
|
14
|
-
// `RNSARFramePose`.
|
|
15
|
-
//
|
|
16
|
-
// Does NOT own:
|
|
17
|
-
// - The JSI `get` / `getPropertyNames` dispatch. That lives in
|
|
18
|
-
// `cpp/stitcher_frame_jsi.cpp` and is identical to the Android
|
|
19
|
-
// implementation (DRY across platforms).
|
|
20
|
-
|
|
21
|
-
#import "StitcherFrameHostObject.h"
|
|
22
|
-
|
|
23
|
-
#import <Foundation/Foundation.h>
|
|
24
|
-
#import <CoreVideo/CVPixelBuffer.h>
|
|
25
|
-
#import <CoreMedia/CoreMedia.h>
|
|
26
|
-
#import <os/log.h>
|
|
27
|
-
|
|
28
|
-
#include <jsi/jsi.h>
|
|
29
|
-
|
|
30
|
-
#include <algorithm>
|
|
31
|
-
#include <cstring>
|
|
32
|
-
#include <memory>
|
|
33
|
-
#include <string>
|
|
34
|
-
#include <utility>
|
|
35
|
-
|
|
36
|
-
#include "stitcher_frame_data.hpp"
|
|
37
|
-
#include "stitcher_frame_jsi.hpp"
|
|
38
|
-
|
|
39
|
-
using namespace facebook;
|
|
40
|
-
|
|
41
|
-
// Forward-declare the Swift `RNSARFramePose` Obj-C surface we need.
|
|
42
|
-
// This matches the pattern in `KeyframeGateFrameProcessor.mm`
|
|
43
|
-
// (forward-declaring `IncrementalStitcher`) — avoids depending on
|
|
44
|
-
// the autogenerated `RNImageStitcher-Swift.h`, which is created at
|
|
45
|
-
// build time and not always available to .mm files in this pod.
|
|
46
|
-
//
|
|
47
|
-
// MUST stay in sync with `RNSARSession.swift::RNSARFramePose` —
|
|
48
|
-
// adding a new field there means adding it here too.
|
|
49
|
-
@class RNSARFramePose;
|
|
50
|
-
@interface RNSARFramePose : NSObject
|
|
51
|
-
@property (nonatomic, readonly) double tx;
|
|
52
|
-
@property (nonatomic, readonly) double ty;
|
|
53
|
-
@property (nonatomic, readonly) double tz;
|
|
54
|
-
@property (nonatomic, readonly) double qx;
|
|
55
|
-
@property (nonatomic, readonly) double qy;
|
|
56
|
-
@property (nonatomic, readonly) double qz;
|
|
57
|
-
@property (nonatomic, readonly) double qw;
|
|
58
|
-
@property (nonatomic, readonly) NSInteger imageWidth;
|
|
59
|
-
@property (nonatomic, readonly) NSInteger imageHeight;
|
|
60
|
-
@property (nonatomic, readonly) double timestampMs;
|
|
61
|
-
@end
|
|
62
|
-
|
|
63
|
-
#pragma mark - iOS PixelBufferReader
|
|
64
|
-
|
|
65
|
-
namespace {
|
|
66
|
-
|
|
67
|
-
/// iOS-specific `retailens::PixelBufferReader` impl. See the base
|
|
68
|
-
/// class docstring for the general contract (thread-affinity,
|
|
69
|
-
/// invalidation semantics, Y-plane-only constraint). This subclass
|
|
70
|
-
/// adds:
|
|
71
|
-
/// - `CVPixelBuffer` lock/memcpy/unlock per copyTo
|
|
72
|
-
/// - `CFBridgingRetain` of the parent `ARFrame` so ARKit's
|
|
73
|
-
/// pool can't reclaim the underlying buffer mid-read
|
|
74
|
-
class IOSPixelBufferReader : public retailens::PixelBufferReader {
|
|
75
|
-
public:
|
|
76
|
-
explicit IOSPixelBufferReader(ARFrame* arFrame) {
|
|
77
|
-
// Retain the ARFrame for our lifetime. CFBridgingRetain hands
|
|
78
|
-
// ARC ownership to our void*. Released in destructor.
|
|
79
|
-
_retainedFrame = (void*)CFBridgingRetain(arFrame);
|
|
80
|
-
CVPixelBufferRef pixelBuffer = arFrame.capturedImage;
|
|
81
|
-
if (pixelBuffer != NULL) {
|
|
82
|
-
_bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
|
|
83
|
-
_height = CVPixelBufferGetHeight(pixelBuffer);
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
~IOSPixelBufferReader() override {
|
|
88
|
-
// Transfer ownership back to ARC, which then releases.
|
|
89
|
-
if (_retainedFrame != nullptr) {
|
|
90
|
-
ARFrame* frame = CFBridgingRelease(_retainedFrame);
|
|
91
|
-
(void)frame;
|
|
92
|
-
_retainedFrame = nullptr;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
std::size_t byteSize() const override {
|
|
97
|
-
return _bytesPerRow * _height;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) override {
|
|
101
|
-
if (_retainedFrame == nullptr) return 0;
|
|
102
|
-
ARFrame* frame = (__bridge ARFrame*)_retainedFrame;
|
|
103
|
-
CVPixelBufferRef pixelBuffer = frame.capturedImage;
|
|
104
|
-
if (pixelBuffer == NULL) return 0;
|
|
105
|
-
|
|
106
|
-
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
|
107
|
-
const uint8_t* src = (const uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
|
|
108
|
-
std::size_t toCopy = std::min<std::size_t>(byteSize(), maxBytes);
|
|
109
|
-
if (src != nullptr && toCopy > 0) {
|
|
110
|
-
std::memcpy(dst, src, toCopy);
|
|
111
|
-
} else {
|
|
112
|
-
toCopy = 0;
|
|
113
|
-
}
|
|
114
|
-
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
|
|
115
|
-
return toCopy;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
private:
|
|
119
|
-
void* _retainedFrame = nullptr; // CFBridgingRetain'd ARFrame
|
|
120
|
-
std::size_t _bytesPerRow = 0;
|
|
121
|
-
std::size_t _height = 0;
|
|
122
|
-
};
|
|
123
|
-
|
|
124
|
-
} // anonymous namespace
|
|
125
|
-
|
|
126
|
-
#pragma mark - Obj-C facade
|
|
127
|
-
|
|
128
|
-
@implementation StitcherFrameHostObject {
|
|
129
|
-
std::shared_ptr<retailens::StitcherFrameJsiHostObject> _hostObject;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
+ (instancetype)fromARFrame:(ARFrame*)arFrame pose:(RNSARFramePose*)pose {
|
|
133
|
-
StitcherFrameHostObject* obj = [[self alloc] init];
|
|
134
|
-
|
|
135
|
-
retailens::StitcherFrameData data;
|
|
136
|
-
data.source = "ar";
|
|
137
|
-
data.width = static_cast<int32_t>(pose.imageWidth);
|
|
138
|
-
data.height = static_cast<int32_t>(pose.imageHeight);
|
|
139
|
-
// ARKit's `kCVPixelFormatType_420YpCbCr8BiPlanarFullRange` (NV12)
|
|
140
|
-
// is reported as "yuv". Other formats (rare in ARKit; possible if
|
|
141
|
-
// ARWorldTrackingConfiguration.videoFormat is overridden to BGRA)
|
|
142
|
-
// → "unknown" + os_log warning so worklets that gate on
|
|
143
|
-
// `pixelFormat === 'yuv'` can be debugged without a screen recording.
|
|
144
|
-
OSType pf = CVPixelBufferGetPixelFormatType(arFrame.capturedImage);
|
|
145
|
-
if (pf == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
|
|
146
|
-
pf == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
|
|
147
|
-
data.pixelFormat = "yuv";
|
|
148
|
-
} else {
|
|
149
|
-
data.pixelFormat = "unknown";
|
|
150
|
-
os_log_error(OS_LOG_DEFAULT,
|
|
151
|
-
"[StitcherFrame] unexpected ARKit pixel format 0x%x; "
|
|
152
|
-
"worklet receives pixelFormat='unknown' and toArrayBuffer() "
|
|
153
|
-
"bytes are first-plane only (layout undefined for unknown "
|
|
154
|
-
"formats). See StitcherFrame.ts docstring.", (unsigned int)pf);
|
|
155
|
-
}
|
|
156
|
-
// ARKit doesn't have a `Frame.orientation` per se; pose carries
|
|
157
|
-
// the imageWidth >= imageHeight discriminator the lib uses
|
|
158
|
-
// elsewhere (`isLandscape`). v0.8.0 ships a coarse mapping;
|
|
159
|
-
// worklets that need exact UI orientation can read it from
|
|
160
|
-
// device-orientation sensors.
|
|
161
|
-
data.orientation =
|
|
162
|
-
(pose.imageWidth >= pose.imageHeight) ? "landscape-right" : "portrait";
|
|
163
|
-
// `ARFrame.timestamp` is CFAbsoluteTime (seconds since epoch).
|
|
164
|
-
// Convert to ns to match vc Frame.timestamp.
|
|
165
|
-
data.timestampNs = arFrame.timestamp * 1e9;
|
|
166
|
-
|
|
167
|
-
data.qx = pose.qx;
|
|
168
|
-
data.qy = pose.qy;
|
|
169
|
-
data.qz = pose.qz;
|
|
170
|
-
data.qw = pose.qw;
|
|
171
|
-
data.tx = pose.tx;
|
|
172
|
-
data.ty = pose.ty;
|
|
173
|
-
data.tz = pose.tz;
|
|
174
|
-
data.hasTranslation = true; // AR mode always has translation
|
|
175
|
-
|
|
176
|
-
switch (arFrame.camera.trackingState) {
|
|
177
|
-
case ARTrackingStateNotAvailable:
|
|
178
|
-
data.arTrackingState = "notAvailable";
|
|
179
|
-
break;
|
|
180
|
-
case ARTrackingStateLimited:
|
|
181
|
-
data.arTrackingState = "limited";
|
|
182
|
-
break;
|
|
183
|
-
case ARTrackingStateNormal:
|
|
184
|
-
data.arTrackingState = "normal";
|
|
185
|
-
break;
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
data.pixelReader = std::make_shared<IOSPixelBufferReader>(arFrame);
|
|
189
|
-
|
|
190
|
-
// Use the static factory (private ctor enforces shared_ptr
|
|
191
|
-
// ownership — required for `shared_from_this()` inside the JSI
|
|
192
|
-
// `toArrayBuffer` lambda).
|
|
193
|
-
obj->_hostObject =
|
|
194
|
-
retailens::StitcherFrameJsiHostObject::create(std::move(data));
|
|
195
|
-
return obj;
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
- (void)invalidate {
|
|
199
|
-
if (_hostObject) {
|
|
200
|
-
_hostObject->invalidate();
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
- (void*)jsiHostObjectPtr {
|
|
205
|
-
if (!_hostObject) return NULL;
|
|
206
|
-
// Box a heap-allocated copy of the shared_ptr to the abstract
|
|
207
|
-
// `jsi::HostObject` base. Caller (worklet runtime) does:
|
|
208
|
-
// auto sp = static_cast<std::shared_ptr<jsi::HostObject>*>(ptr);
|
|
209
|
-
// auto jsObj = jsi::Object::createFromHostObject(rt, *sp);
|
|
210
|
-
// delete sp;
|
|
211
|
-
return new std::shared_ptr<jsi::HostObject>(_hostObject);
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
@end
|