react-native-image-stitcher 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +121 -0
- package/RNImageStitcher.podspec +1 -1
- package/android/src/main/cpp/CMakeLists.txt +4 -4
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +216 -7
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +656 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +156 -0
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +1 -1
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +84 -2
- package/cpp/{stitcher_frame_data.hpp → camera_frame_data.hpp} +96 -13
- package/cpp/{stitcher_frame_jsi.cpp → camera_frame_jsi.cpp} +154 -11
- package/cpp/{stitcher_frame_jsi.hpp → camera_frame_jsi.hpp} +12 -12
- package/cpp/stitcher_proxy_jsi.cpp +31 -0
- package/cpp/stitcher_proxy_jsi.hpp +16 -0
- package/cpp/stitcher_worklet_dispatch.cpp +5 -5
- package/cpp/stitcher_worklet_dispatch.hpp +5 -5
- package/dist/camera/ARCameraView.d.ts +60 -3
- package/dist/camera/ARCameraView.js +68 -1
- package/dist/camera/Camera.d.ts +54 -7
- package/dist/camera/Camera.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/stitching/ARFrameMeta.d.ts +100 -0
- package/dist/stitching/{StitcherFrame.js → ARFrameMeta.js} +1 -1
- package/dist/stitching/{StitcherFrame.d.ts → CameraFrame.d.ts} +70 -11
- package/dist/stitching/CameraFrame.js +4 -0
- package/dist/stitching/useStitcherWorklet.d.ts +4 -4
- package/dist/stitching/useStitcherWorklet.js +4 -4
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +23 -1
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +137 -2
- package/ios/Sources/RNImageStitcher/{StitcherFrameHostObject.h → CameraFrameHostObject.h} +26 -3
- package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm +760 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +292 -34
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +2 -2
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +4 -4
- package/package.json +1 -1
- package/src/camera/ARCameraView.tsx +165 -5
- package/src/camera/Camera.tsx +69 -7
- package/src/index.ts +7 -3
- package/src/stitching/ARFrameMeta.ts +107 -0
- package/src/stitching/{StitcherFrame.ts → CameraFrame.ts} +79 -11
- package/src/stitching/useStitcherWorklet.ts +9 -9
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
package/dist/camera/Camera.d.ts
CHANGED
|
@@ -41,7 +41,8 @@
|
|
|
41
41
|
import React from 'react';
|
|
42
42
|
import { type StyleProp, type ViewStyle } from 'react-native';
|
|
43
43
|
import type { DrawableFrameProcessor, ReadonlyFrameProcessor } from 'react-native-vision-camera';
|
|
44
|
-
import type {
|
|
44
|
+
import type { CameraFrameProcessor } from '../stitching/CameraFrame';
|
|
45
|
+
import type { ARFrameMeta } from '../stitching/ARFrameMeta';
|
|
45
46
|
import { type CaptureHeaderProps } from './CaptureHeader';
|
|
46
47
|
import { type CapturePreviewAction } from './CapturePreview';
|
|
47
48
|
import { type CaptureThumbnailItem } from './CaptureThumbnailStrip';
|
|
@@ -527,11 +528,11 @@ export interface CameraProps {
|
|
|
527
528
|
* worklet to fire on vc's Frame Processor runtime.
|
|
528
529
|
*
|
|
529
530
|
* ```tsx
|
|
530
|
-
* import { Camera, useFrameProcessor, type
|
|
531
|
+
* import { Camera, useFrameProcessor, type CameraFrame }
|
|
531
532
|
* from 'react-native-image-stitcher';
|
|
532
533
|
*
|
|
533
534
|
* function MyScreen() {
|
|
534
|
-
* const fp = useFrameProcessor((frame:
|
|
535
|
+
* const fp = useFrameProcessor((frame: CameraFrame) => {
|
|
535
536
|
* 'worklet';
|
|
536
537
|
* // ...
|
|
537
538
|
* }, []);
|
|
@@ -552,12 +553,12 @@ export interface CameraProps {
|
|
|
552
553
|
* ```tsx
|
|
553
554
|
* import {
|
|
554
555
|
* Camera, useFrameProcessor, useStitcherWorklet,
|
|
555
|
-
* type
|
|
556
|
+
* type CameraFrame,
|
|
556
557
|
* } from 'react-native-image-stitcher';
|
|
557
558
|
*
|
|
558
559
|
* function MyScreen() {
|
|
559
560
|
* const stitcher = useStitcherWorklet();
|
|
560
|
-
* const fp = useFrameProcessor((frame:
|
|
561
|
+
* const fp = useFrameProcessor((frame: CameraFrame) => {
|
|
561
562
|
* 'worklet';
|
|
562
563
|
* hostPreLogic(frame);
|
|
563
564
|
* stitcher.call(frame); // ← first-party stitching
|
|
@@ -601,14 +602,60 @@ export interface CameraProps {
|
|
|
601
602
|
/**
|
|
602
603
|
* AR-mode host worklet, invoked once per ARKit / ARCore frame
|
|
603
604
|
* ALONGSIDE the lib's first-party stitching (composition, not
|
|
604
|
-
* replacement). Receives a `
|
|
605
|
+
* replacement). Receives a `CameraFrame` tagged `source: 'ar'`
|
|
605
606
|
* with world-space `pose` + `arTrackingState`. Only fires in AR
|
|
606
607
|
* capture (`captureSource === 'ar'`); the non-AR equivalent is
|
|
607
608
|
* `frameProcessor` above (the two modes use different runtimes and
|
|
608
609
|
* frame shapes). Must be a `'worklet'`-prefixed function; if the
|
|
609
610
|
* native install is unavailable it silently never fires.
|
|
610
611
|
*/
|
|
611
|
-
arFrameProcessor?:
|
|
612
|
+
arFrameProcessor?: CameraFrameProcessor;
|
|
613
|
+
/**
|
|
614
|
+
* Opt in to per-frame AR depth on the `arFrameProcessor` frame
|
|
615
|
+
* (`CameraFrame.arDepth`). Default `false` — depth is the costliest
|
|
616
|
+
* field (a per-frame buffer copy), so it's off until you need it.
|
|
617
|
+
*/
|
|
618
|
+
enableDepth?: boolean;
|
|
619
|
+
/**
|
|
620
|
+
* Opt in to per-frame AR anchors (`CameraFrame.arAnchors` — detected
|
|
621
|
+
* planes / images). Default `false`.
|
|
622
|
+
*/
|
|
623
|
+
enableAnchors?: boolean;
|
|
624
|
+
/**
|
|
625
|
+
* Opt in to scene-reconstruction mesh anchors (`type: 'mesh'` in
|
|
626
|
+
* `arAnchors`, with `meshGeometry`). Default `false`. iOS enables
|
|
627
|
+
* ARKit `sceneReconstruction` (LiDAR); Android reconstructs a rough
|
|
628
|
+
* mesh from the depth map. Expensive — only on when needed.
|
|
629
|
+
*/
|
|
630
|
+
enableMesh?: boolean;
|
|
631
|
+
/**
|
|
632
|
+
* Which plane orientations to surface in `CameraFrame.arAnchors`
|
|
633
|
+
* (requires `enableAnchors`; AR capture only). Default `'vertical'`
|
|
634
|
+
* — the orientation the plane-projected stitch path has always used.
|
|
635
|
+
* `'horizontal'` surfaces floors / tables; `'both'` surfaces every
|
|
636
|
+
* detected plane. See `ARCameraView` for the per-platform details.
|
|
637
|
+
*/
|
|
638
|
+
planeDetection?: 'vertical' | 'horizontal' | 'both';
|
|
639
|
+
/**
|
|
640
|
+
* v0.18.0 — LIGHT per-frame AR metadata callback, invoked on the JS
|
|
641
|
+
* MAIN thread (NOT a worklet). Only fires in AR capture
|
|
642
|
+
* (`captureSource === 'ar'`). Receives an {@link ARFrameMeta} carrying
|
|
643
|
+
* pose, tracking state, intrinsics, and (when the matching `enable*`
|
|
644
|
+
* prop is on) depth dimensions, anchors, and mesh counts.
|
|
645
|
+
*
|
|
646
|
+
* This is the recommended way to read AR metadata: it sidesteps the
|
|
647
|
+
* worklet path entirely (the `arFrameProcessor` worklet can only safely
|
|
648
|
+
* surface a worklets-core shared value, because capturing a host
|
|
649
|
+
* callback crashes the worklet closure-wrap). Native builds the meta
|
|
650
|
+
* and emits a device event; `<Camera>` threads the handler through to
|
|
651
|
+
* `<ARCameraView>`, which subscribes and invokes it on the main thread.
|
|
652
|
+
*/
|
|
653
|
+
onArFrame?: (meta: ARFrameMeta) => void;
|
|
654
|
+
/**
|
|
655
|
+
* v0.18.0 — throttle interval (ms) for {@link onArFrame}. Default `100`
|
|
656
|
+
* (≈ 10 Hz). No effect unless `onArFrame` is provided.
|
|
657
|
+
*/
|
|
658
|
+
arFrameMetaInterval?: number;
|
|
612
659
|
/**
|
|
613
660
|
* Which device holds the non-AR panorama capture accepts.
|
|
614
661
|
*
|
package/dist/camera/Camera.js
CHANGED
|
@@ -310,7 +310,7 @@ function extractPanoramaOverrides(props) {
|
|
|
310
310
|
* The public `<Camera>` component.
|
|
311
311
|
*/
|
|
312
312
|
function Camera(props) {
|
|
313
|
-
const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, engine = 'batch-keyframe',
|
|
313
|
+
const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, enableDepth, enableAnchors, enableMesh, planeDetection, onArFrame, arFrameMetaInterval, engine = 'batch-keyframe',
|
|
314
314
|
// ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
|
|
315
315
|
panMode = 'vertical', panGuidance = true, maxPanDurationMs = 0, panTooFastThreshold, lateralBudgetCm = 4, rectCrop = false, showPreview = false, guidanceCopy, } = props;
|
|
316
316
|
// Derived guidance state. The landscape-only gate decision itself is
|
|
@@ -1425,7 +1425,7 @@ function Camera(props) {
|
|
|
1425
1425
|
// (V12.14.8 OOM fix). The CaptureStatusOverlay renders the
|
|
1426
1426
|
// "Stitching…" state on top, so no placeholder label is needed
|
|
1427
1427
|
// in that case — only for the camera-switch transition.
|
|
1428
|
-
react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
|
|
1428
|
+
react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor, enableDepth: enableDepth, enableAnchors: enableAnchors, enableMesh: enableMesh, planeDetection: planeDetection, onArFrame: onArFrame, arFrameMetaInterval: arFrameMetaInterval })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
|
|
1429
1429
|
// `video={true}` is REQUIRED for takeSnapshot to work on iOS.
|
|
1430
1430
|
// vision-camera v4's iOS implementation of takeSnapshot waits
|
|
1431
1431
|
// for a frame on the video pipeline; with video disabled, the
|
package/dist/index.d.ts
CHANGED
|
@@ -91,7 +91,8 @@ export { IncrementalOutcome, incrementalStitcherIsAvailable, subscribeIncrementa
|
|
|
91
91
|
export type { IncrementalState, AcceptedKeyframe } from './stitching/incremental';
|
|
92
92
|
export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
|
|
93
93
|
export { useKeyframeStream } from './stitching/useKeyframeStream';
|
|
94
|
-
export type {
|
|
94
|
+
export type { CameraFrame, CameraFrameProcessor, ARAnchor, } from './stitching/CameraFrame';
|
|
95
|
+
export type { ARFrameMeta } from './stitching/ARFrameMeta';
|
|
95
96
|
export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
|
|
96
97
|
export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
|
|
97
98
|
export { useStitcherWorklet } from './stitching/useStitcherWorklet';
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v0.18.0 — LIGHT per-frame AR metadata delivered to JS on the MAIN
|
|
3
|
+
* thread via a normal callback (`onArFrame`), bypassing worklets entirely.
|
|
4
|
+
*
|
|
5
|
+
* ## Why a callback, NOT a worklet
|
|
6
|
+
*
|
|
7
|
+
* The AR worklet path (`arFrameProcessor` + the `__stitcherProxy` JSI
|
|
8
|
+
* registry) deep-copies the worklet's whole closure into the AR worklet
|
|
9
|
+
* runtime via react-native-worklets-core's `WorkletInvoker`. When the
|
|
10
|
+
* worklet captures a host object (e.g. a `createRunOnJS` callback) that
|
|
11
|
+
* closure-wrap recurses without termination → stack overflow → SIGBUS the
|
|
12
|
+
* instant AR mode mounts (verified on device). Worklets can therefore
|
|
13
|
+
* only safely capture a worklets-core *shared value* — an awkward,
|
|
14
|
+
* poll-from-JS pattern for getting structured data back.
|
|
15
|
+
*
|
|
16
|
+
* `onArFrame` sidesteps the whole problem: native builds the metadata and
|
|
17
|
+
* emits it as a plain `RNImageStitcherARFrame` device event carrying a
|
|
18
|
+
* JSON object; the JS side subscribes via `NativeEventEmitter` and invokes
|
|
19
|
+
* the host callback on the main thread. No worklet, no closure-wrap, no
|
|
20
|
+
* shared-value polling. This is the recommended way to read AR metadata.
|
|
21
|
+
*
|
|
22
|
+
* ## Cost / gating
|
|
23
|
+
*
|
|
24
|
+
* The metadata is intentionally LIGHT — no pixel / vertex / face byte
|
|
25
|
+
* marshaling. `depth` reports only the depth map's dimensions + whether a
|
|
26
|
+
* confidence channel exists (no buffer copy); `mesh` reports only anchor /
|
|
27
|
+
* vertex / face *counts*. Native gates each costly field on the matching
|
|
28
|
+
* extraction flag (`depth` ⇐ `enableDepth`, `mesh` ⇐ `enableMesh`,
|
|
29
|
+
* `anchors` ⇐ `enableAnchors`); `intrinsics` / `pose` / `trackingState` are
|
|
30
|
+
* always present. Emission is gated on a TS-set enabled flag (only true
|
|
31
|
+
* when `onArFrame` is provided) and throttled to `arFrameMetaInterval` ms
|
|
32
|
+
* (default 100 ≈ 10 Hz) on the native side.
|
|
33
|
+
*/
|
|
34
|
+
export interface ARFrameMeta {
|
|
35
|
+
/** Frame-capture timestamp in NANOSECONDS (AR-framework monotonic clock). */
|
|
36
|
+
timestamp: number;
|
|
37
|
+
/** AR tracking quality at this frame. */
|
|
38
|
+
trackingState: 'notAvailable' | 'limited' | 'normal';
|
|
39
|
+
/**
|
|
40
|
+
* Camera pose in world coordinates at frame-capture time.
|
|
41
|
+
*
|
|
42
|
+
* - `rotation` — quaternion `(x, y, z, w)`, matching the convention
|
|
43
|
+
* used throughout the engine + the `CameraFrame.pose` field.
|
|
44
|
+
* - `translation` — metres in world space `[x, y, z]`.
|
|
45
|
+
*/
|
|
46
|
+
pose: {
|
|
47
|
+
rotation: [number, number, number, number];
|
|
48
|
+
translation: [number, number, number];
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Camera intrinsics for this frame — focal lengths (`fx`, `fy`) and
|
|
52
|
+
* principal point (`cx`, `cy`) in PIXELS at the `imageWidth × imageHeight`
|
|
53
|
+
* capture resolution. Always attempted (not gated); `null` only when the
|
|
54
|
+
* AR framework didn't provide them for this frame.
|
|
55
|
+
*/
|
|
56
|
+
intrinsics: {
|
|
57
|
+
fx: number;
|
|
58
|
+
fy: number;
|
|
59
|
+
cx: number;
|
|
60
|
+
cy: number;
|
|
61
|
+
imageWidth: number;
|
|
62
|
+
imageHeight: number;
|
|
63
|
+
} | null;
|
|
64
|
+
/**
|
|
65
|
+
* Depth-map summary — dimensions + whether a per-pixel confidence channel
|
|
66
|
+
* is available. NO pixel buffer is copied (that's the costly part).
|
|
67
|
+
* Present only when the `enableDepth` prop is on AND the device produced a
|
|
68
|
+
* depth map this frame; `null` otherwise.
|
|
69
|
+
*/
|
|
70
|
+
depth: {
|
|
71
|
+
width: number;
|
|
72
|
+
height: number;
|
|
73
|
+
hasConfidence: boolean;
|
|
74
|
+
} | null;
|
|
75
|
+
/**
|
|
76
|
+
* Tracked AR anchors visible in this frame (planes / images / points /
|
|
77
|
+
* mesh). Empty array when `enableAnchors` is on but nothing is tracked;
|
|
78
|
+
* effectively empty when `enableAnchors` is off. `transform` is a 4×4
|
|
79
|
+
* row-major anchor→world matrix (16 numbers).
|
|
80
|
+
*/
|
|
81
|
+
anchors: Array<{
|
|
82
|
+
id: string;
|
|
83
|
+
type: 'plane' | 'image' | 'point' | 'mesh';
|
|
84
|
+
alignment?: 'horizontal' | 'vertical';
|
|
85
|
+
extent?: [number, number];
|
|
86
|
+
classification?: string;
|
|
87
|
+
transform: number[];
|
|
88
|
+
}>;
|
|
89
|
+
/**
|
|
90
|
+
* Scene-reconstruction mesh summary — anchor / vertex / face COUNTS only
|
|
91
|
+
* (no vertex / face byte marshaling). Present only when `enableMesh` is
|
|
92
|
+
* on; `null` otherwise.
|
|
93
|
+
*/
|
|
94
|
+
mesh: {
|
|
95
|
+
anchorCount: number;
|
|
96
|
+
vertexCount: number;
|
|
97
|
+
faceCount: number;
|
|
98
|
+
} | null;
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=ARFrameMeta.d.ts.map
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* v0.8.0 — unified frame contract for the lib's worklet processor.
|
|
3
3
|
*
|
|
4
4
|
* Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
|
|
5
|
-
* this directory) receive a `
|
|
5
|
+
* this directory) receive a `CameraFrame` regardless of capture mode.
|
|
6
6
|
* The lib-owned worklet runtime guarantees the same JS-visible shape
|
|
7
7
|
* whether the underlying source is a vision-camera `Frame` (non-AR
|
|
8
8
|
* mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
|
|
@@ -16,10 +16,10 @@
|
|
|
16
16
|
* (Phase-0 audit confirmed the iOS path). But vision-camera's
|
|
17
17
|
* **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
|
|
18
18
|
* ARCore does NOT produce `ImageProxy` instances. Forcing
|
|
19
|
-
* `
|
|
19
|
+
* `CameraFrame extends Frame` would either (a) require reverse-
|
|
20
20
|
* engineering ImageProxy on Android (intractable + fragile), or
|
|
21
21
|
* (b) make the type asymmetric per platform. Both are worse than
|
|
22
|
-
* making `
|
|
22
|
+
* making `CameraFrame` a structural sibling type that vc Frames
|
|
23
23
|
* happen to satisfy (because vc Frames carry the same width / height /
|
|
24
24
|
* orientation / pixelFormat / timestamp / toArrayBuffer surface).
|
|
25
25
|
*
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
* a JPEG-encode frame-processor plugin). Returning a reference and
|
|
37
37
|
* reading it later will read into freed memory.
|
|
38
38
|
*/
|
|
39
|
-
export interface
|
|
39
|
+
export interface CameraFrame {
|
|
40
40
|
/** Pixel width of the camera image. */
|
|
41
41
|
width: number;
|
|
42
42
|
/** Pixel height of the camera image. */
|
|
@@ -141,6 +141,27 @@ export interface StitcherFrame {
|
|
|
141
141
|
* tracking is degraded check this. Undefined in non-AR mode.
|
|
142
142
|
*/
|
|
143
143
|
arTrackingState?: 'notAvailable' | 'limited' | 'normal';
|
|
144
|
+
/**
|
|
145
|
+
* Camera intrinsics for THIS frame — focal lengths (`fx`,`fy`) and
|
|
146
|
+
* principal point (`cx`,`cy`) in PIXELS at the `imageWidth × imageHeight`
|
|
147
|
+
* capture resolution. Needed to lift 2D image-space coordinates to 3D
|
|
148
|
+
* via pose + intrinsics (e.g. object-level reconstruction).
|
|
149
|
+
*
|
|
150
|
+
* Populated on **AR frames** (`source: 'ar'`) from ARKit
|
|
151
|
+
* `ARCamera.intrinsics` / ARCore `Camera` intrinsics. **Undefined for
|
|
152
|
+
* non-AR (vision-camera) frames** — they are raw vc `Frame`s without an
|
|
153
|
+
* intrinsics surface; read vc's own APIs there if needed. (The spec
|
|
154
|
+
* called this required; it's optional here because the non-AR frame
|
|
155
|
+
* shape genuinely can't carry it.)
|
|
156
|
+
*/
|
|
157
|
+
intrinsics?: {
|
|
158
|
+
fx: number;
|
|
159
|
+
fy: number;
|
|
160
|
+
cx: number;
|
|
161
|
+
cy: number;
|
|
162
|
+
imageWidth: number;
|
|
163
|
+
imageHeight: number;
|
|
164
|
+
};
|
|
144
165
|
}
|
|
145
166
|
/**
|
|
146
167
|
* v0.8.0 — public AR anchor type. Subset of ARKit/ARCore anchor info
|
|
@@ -150,21 +171,59 @@ export interface StitcherFrame {
|
|
|
150
171
|
export interface ARAnchor {
|
|
151
172
|
/** Stable per-session anchor identifier. */
|
|
152
173
|
id: string;
|
|
153
|
-
/** Anchor kind. `'point'` is Android (ARCore) only. */
|
|
154
|
-
type: 'plane' | 'image' | 'point';
|
|
155
174
|
/**
|
|
156
|
-
*
|
|
157
|
-
*
|
|
175
|
+
* Anchor kind. `'point'` is Android (ARCore) only; `'mesh'` is a
|
|
176
|
+
* scene-reconstruction mesh anchor, present only when the `enableMesh`
|
|
177
|
+
* `<Camera>` prop is on (and the device supports reconstruction).
|
|
178
|
+
*/
|
|
179
|
+
type: 'plane' | 'image' | 'point' | 'mesh';
|
|
180
|
+
/**
|
|
181
|
+
* 4×4 row-major transform from anchor space to world space (16
|
|
182
|
+
* numbers). For `'mesh'` anchors, the `meshGeometry.vertices` are in
|
|
183
|
+
* this anchor's LOCAL space — multiply by `transform` for world coords.
|
|
158
184
|
*/
|
|
159
185
|
transform: number[];
|
|
186
|
+
/**
|
|
187
|
+
* Plane orientation — `'horizontal'` (floor / table / seat) vs
|
|
188
|
+
* `'vertical'` (wall / door / window). Present on `'plane'` anchors;
|
|
189
|
+
* undefined for other anchor kinds. Lets a host distinguish a shelf
|
|
190
|
+
* surface from the wall behind it.
|
|
191
|
+
*/
|
|
192
|
+
alignment?: 'horizontal' | 'vertical';
|
|
193
|
+
/**
|
|
194
|
+
* Plane size in metres along its local x / z axes (`[x, z]`). Present
|
|
195
|
+
* on `'plane'` anchors only.
|
|
196
|
+
*/
|
|
197
|
+
extent?: [number, number];
|
|
198
|
+
/**
|
|
199
|
+
* ARKit semantic classification of the plane's surface, when the
|
|
200
|
+
* framework provides it (iOS; mostly horizontal planes). Undefined
|
|
201
|
+
* when unknown / unsupported (incl. Android, which has no equivalent).
|
|
202
|
+
*/
|
|
203
|
+
classification?: 'wall' | 'floor' | 'ceiling' | 'table' | 'seat' | 'door' | 'window' | 'none';
|
|
204
|
+
/**
|
|
205
|
+
* Scene-reconstruction geometry — present only on `type: 'mesh'`
|
|
206
|
+
* anchors. Buffers (wrap in the noted typed-array view):
|
|
207
|
+
* - `vertices` → `Float32Array`, xyz triplets in anchor-local space.
|
|
208
|
+
* - `faces` → `Uint32Array`, triangle indices into `vertices`.
|
|
209
|
+
* - `classifications` → optional `Uint8Array`, one ARKit mesh class
|
|
210
|
+
* per face (0=none, 1=wall, 2=floor, 3=ceiling, …). **iOS only**
|
|
211
|
+
* (from `ARMeshAnchor`); absent on Android, where the mesh is
|
|
212
|
+
* reconstructed from the depth map and carries no semantics.
|
|
213
|
+
*/
|
|
214
|
+
meshGeometry?: {
|
|
215
|
+
vertices: ArrayBuffer;
|
|
216
|
+
faces: ArrayBuffer;
|
|
217
|
+
classifications?: ArrayBuffer;
|
|
218
|
+
};
|
|
160
219
|
}
|
|
161
220
|
/**
|
|
162
221
|
* v0.8.0 — worklet function signature for the unified frame processor.
|
|
163
222
|
*
|
|
164
223
|
* Must be a `'worklet'`-prefixed function (so it can run on the
|
|
165
|
-
* worklet runtime). Receives a `
|
|
224
|
+
* worklet runtime). Receives a `CameraFrame` per camera frame; the
|
|
166
225
|
* return value is ignored (use `runOnJS` / shared values to surface
|
|
167
226
|
* results back to the JS thread).
|
|
168
227
|
*/
|
|
169
|
-
export type
|
|
170
|
-
//# sourceMappingURL=
|
|
228
|
+
export type CameraFrameProcessor = (frame: CameraFrame) => void;
|
|
229
|
+
//# sourceMappingURL=CameraFrame.d.ts.map
|
|
@@ -106,17 +106,17 @@
|
|
|
106
106
|
* their own worklet via this hook must do the wiring themselves.
|
|
107
107
|
*/
|
|
108
108
|
import type { Frame } from 'react-native-vision-camera';
|
|
109
|
-
import type {
|
|
109
|
+
import type { CameraFrame } from './CameraFrame';
|
|
110
110
|
/**
|
|
111
111
|
* Frames the lib's stitching worklet accepts. Accepting either a
|
|
112
112
|
* vc `Frame` (what the host's `useFrameProcessor` body sees) or the
|
|
113
|
-
* lib's `
|
|
113
|
+
* lib's `CameraFrame` (what the lib's `useFrameProcessor` body
|
|
114
114
|
* sees) keeps the same `useStitcherWorklet` usable from both kinds
|
|
115
115
|
* of host worklet bodies without a cast on the call site. The
|
|
116
116
|
* worklet only reads `width` / `height`; the rest of the frame
|
|
117
117
|
* object is forwarded verbatim to the native plugin.
|
|
118
118
|
*/
|
|
119
|
-
export type StitcherWorkletInput = Frame |
|
|
119
|
+
export type StitcherWorkletInput = Frame | CameraFrame;
|
|
120
120
|
export interface UseStitcherWorkletOptions {
|
|
121
121
|
/**
|
|
122
122
|
* Gyro sample interval in ms (~30 Hz default). Drives the JS-
|
|
@@ -145,7 +145,7 @@ export interface UseStitcherWorkletOptions {
|
|
|
145
145
|
}
|
|
146
146
|
export interface StitcherWorkletHandle {
|
|
147
147
|
/**
|
|
148
|
-
* Worklet function: pass a `
|
|
148
|
+
* Worklet function: pass a `CameraFrame` to perform one frame of
|
|
149
149
|
* the lib's first-party stitching (throttle + pose synthesis +
|
|
150
150
|
* native plugin call). Safe to call from inside another
|
|
151
151
|
* `'worklet'`-prefixed function (this is the canonical
|
|
@@ -228,7 +228,7 @@ function useStitcherWorklet(options = {}) {
|
|
|
228
228
|
// party callback installed in `RNSARWorkletRuntime`). Calling
|
|
229
229
|
// the vc Frame Processor plugin here would throw
|
|
230
230
|
// `getPropertyAsObject: property '__frame' is undefined`
|
|
231
|
-
// because AR frames are `
|
|
231
|
+
// because AR frames are `CameraFrameHostObject` instances
|
|
232
232
|
// and don't carry the vc `Frame` proxy's JSI marker. The
|
|
233
233
|
// throw is caught silently by the per-worklet error handler
|
|
234
234
|
// (`RNSARWorkletRuntime.mm:284-301`) and bubbles up only to
|
|
@@ -240,12 +240,12 @@ function useStitcherWorklet(options = {}) {
|
|
|
240
240
|
// hook (the AR-side stitching path runs natively, independent
|
|
241
241
|
// of the composed worklet body).
|
|
242
242
|
//
|
|
243
|
-
// The `(frame as
|
|
243
|
+
// The `(frame as CameraFrame).source` cast is safe: vc
|
|
244
244
|
// `Frame` doesn't carry a `source` property so the check
|
|
245
245
|
// returns `undefined !== 'ar'` → `true`, and the worklet
|
|
246
246
|
// proceeds normally. Only frames that explicitly tag
|
|
247
247
|
// themselves as AR-source (which our native AR dispatcher
|
|
248
|
-
// does — see `
|
|
248
|
+
// does — see `CameraFrameHostObject.mm`) get short-circuited.
|
|
249
249
|
if (frame.source === 'ar')
|
|
250
250
|
return;
|
|
251
251
|
// Throttle (verbatim from useFrameProcessorDriver).
|
|
@@ -273,7 +273,7 @@ function useStitcherWorklet(options = {}) {
|
|
|
273
273
|
const fx = w * sharedFxNumerator.value;
|
|
274
274
|
const fy = h * sharedFyNumerator.value;
|
|
275
275
|
// vc's `plugin.call` is typed against vc's `Frame`. The worklet
|
|
276
|
-
// accepts the union (`Frame |
|
|
276
|
+
// accepts the union (`Frame | CameraFrame`); cast through
|
|
277
277
|
// `unknown` because the union doesn't satisfy vc's interface
|
|
278
278
|
// even though structurally both members do.
|
|
279
279
|
plugin.call(frame, {
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// imports as `NativeModules.RNSARSession`.
|
|
6
6
|
|
|
7
7
|
#import <React/RCTBridgeModule.h>
|
|
8
|
+
#import <React/RCTEventEmitter.h>
|
|
8
9
|
|
|
9
10
|
// REMAP form, NOT EXTERN_MODULE. The Swift singleton in
|
|
10
11
|
// RNSARSession.swift takes the @objc name "RNSARSession"
|
|
@@ -20,7 +21,14 @@
|
|
|
20
21
|
// `RNSARSessionBridge` and dispatch methods against THAT
|
|
21
22
|
// class — where takePhoto / startRecording / stopRecording etc.
|
|
22
23
|
// actually live.
|
|
23
|
-
|
|
24
|
+
//
|
|
25
|
+
// v0.18.0 — base class is now `RCTEventEmitter` (was `NSObject`) so the
|
|
26
|
+
// "RNSARSession" module can emit the `RNImageStitcherARFrame` device
|
|
27
|
+
// event for the `onArFrame` channel. RN auto-provides the
|
|
28
|
+
// `addListener:` / `removeListeners:` emitter selectors for a module
|
|
29
|
+
// whose remap base is RCTEventEmitter; the Swift class supplies
|
|
30
|
+
// `supportedEvents` / `startObserving` / `stopObserving`.
|
|
31
|
+
@interface RCT_EXTERN_REMAP_MODULE(RNSARSession, RNSARSessionBridge, RCTEventEmitter)
|
|
24
32
|
|
|
25
33
|
RCT_EXTERN_METHOD(isSupported:(RCTPromiseResolveBlock)resolver
|
|
26
34
|
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
@@ -34,6 +42,20 @@ RCT_EXTERN_METHOD(stop:(RCTPromiseResolveBlock)resolver
|
|
|
34
42
|
RCT_EXTERN_METHOD(getState:(RCTPromiseResolveBlock)resolver
|
|
35
43
|
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
36
44
|
|
|
45
|
+
RCT_EXTERN_METHOD(setSceneReconstructionEnabled:(nonnull NSNumber *)enabled
|
|
46
|
+
resolver:(RCTPromiseResolveBlock)resolver
|
|
47
|
+
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
48
|
+
|
|
49
|
+
RCT_EXTERN_METHOD(setPlaneDetection:(nonnull NSString *)mode
|
|
50
|
+
resolver:(RCTPromiseResolveBlock)resolver
|
|
51
|
+
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
52
|
+
|
|
53
|
+
// v0.18.0 — toggle the onArFrame LIGHT-metadata channel + its throttle.
|
|
54
|
+
RCT_EXTERN_METHOD(setArFrameMetaEnabled:(nonnull NSNumber *)enabled
|
|
55
|
+
intervalMs:(nonnull NSNumber *)intervalMs
|
|
56
|
+
resolver:(RCTPromiseResolveBlock)resolver
|
|
57
|
+
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
58
|
+
|
|
37
59
|
RCT_EXTERN_METHOD(snapshotPoseLog:(RCTPromiseResolveBlock)resolver
|
|
38
60
|
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
39
61
|
|
|
@@ -17,15 +17,109 @@
|
|
|
17
17
|
import Foundation
|
|
18
18
|
import React
|
|
19
19
|
|
|
20
|
+
// v0.18.0 — `RNSARSessionBridge` is now an `RCTEventEmitter` (was a
|
|
21
|
+
// plain `NSObject`) so it can deliver the `onArFrame` LIGHT-metadata
|
|
22
|
+
// channel as the JS `RNImageStitcherARFrame` device event. The JS side
|
|
23
|
+
// subscribes via `new NativeEventEmitter(NativeModules.RNSARSession)` —
|
|
24
|
+
// the same module name this bridge is remapped to (see ARSessionBridge.m).
|
|
25
|
+
//
|
|
26
|
+
// Pattern mirrors `IncrementalStitcherBridge`: observe a NotificationCenter
|
|
27
|
+
// post from the framework-free `RNSARSession` engine, then re-emit on the
|
|
28
|
+
// main queue via `bridge.enqueueJSCall("RCTDeviceEventEmitter", "emit", …)`
|
|
29
|
+
// rather than `RCTEventEmitter.sendEvent(…)`, because under RN bridgeless
|
|
30
|
+
// interop `sendEvent` silently no-ops for some event-body shapes (see the
|
|
31
|
+
// IncrementalStitcherBridge.handleStateUpdate docstring).
|
|
20
32
|
@objc(RNSARSessionBridge)
|
|
21
|
-
public final class RNSARSessionBridge:
|
|
33
|
+
public final class RNSARSessionBridge: RCTEventEmitter {
|
|
34
|
+
|
|
35
|
+
/// Whether at least one JS listener is attached to the AR-frame event.
|
|
36
|
+
/// RN's EventEmitter contract: don't emit when no listeners are
|
|
37
|
+
/// registered. Toggled by `startObserving` / `stopObserving`.
|
|
38
|
+
private var hasListeners: Bool = false
|
|
39
|
+
|
|
40
|
+
private static let arFrameEvent = "RNImageStitcherARFrame"
|
|
41
|
+
|
|
42
|
+
public override init() {
|
|
43
|
+
super.init()
|
|
44
|
+
// Defensively de-dupe the observer: under RN bridgeless interop a
|
|
45
|
+
// bridge's init() can run twice on the same instance. Remove any
|
|
46
|
+
// prior registration for this notification before adding, so the
|
|
47
|
+
// observer fires at most once per post regardless.
|
|
48
|
+
NotificationCenter.default.removeObserver(
|
|
49
|
+
self,
|
|
50
|
+
name: .retailensARFrameMeta,
|
|
51
|
+
object: nil
|
|
52
|
+
)
|
|
53
|
+
NotificationCenter.default.addObserver(
|
|
54
|
+
self,
|
|
55
|
+
selector: #selector(handleArFrameMeta(_:)),
|
|
56
|
+
name: .retailensARFrameMeta,
|
|
57
|
+
object: nil
|
|
58
|
+
)
|
|
59
|
+
}
|
|
22
60
|
|
|
23
|
-
|
|
61
|
+
deinit {
|
|
62
|
+
NotificationCenter.default.removeObserver(self)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// MARK: - RCTEventEmitter protocol
|
|
66
|
+
|
|
67
|
+
public override static func requiresMainQueueSetup() -> Bool {
|
|
24
68
|
// ARSession.start() must be called on the main thread —
|
|
25
69
|
// ARKit needs to attach to the active CVDisplayLink.
|
|
26
70
|
return true
|
|
27
71
|
}
|
|
28
72
|
|
|
73
|
+
public override func supportedEvents() -> [String]! {
|
|
74
|
+
return [Self.arFrameEvent]
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
public override func startObserving() {
|
|
78
|
+
hasListeners = true
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
public override func stopObserving() {
|
|
82
|
+
hasListeners = false
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/// Forward a posted `ARFrameMeta` dictionary to JS as the
|
|
86
|
+
/// `RNImageStitcherARFrame` device event. Dropped when no JS listener
|
|
87
|
+
/// is attached. Emits via `enqueueJSCall` on the main queue (see the
|
|
88
|
+
/// class docstring for why not `sendEvent`).
|
|
89
|
+
@objc private func handleArFrameMeta(_ notification: Notification) {
|
|
90
|
+
guard hasListeners else { return }
|
|
91
|
+
guard let userInfo = notification.userInfo else { return }
|
|
92
|
+
DispatchQueue.main.async { [weak self] in
|
|
93
|
+
guard let self = self, let bridge = self.bridge else { return }
|
|
94
|
+
bridge.enqueueJSCall(
|
|
95
|
+
"RCTDeviceEventEmitter",
|
|
96
|
+
method: "emit",
|
|
97
|
+
args: [Self.arFrameEvent, userInfo],
|
|
98
|
+
completion: nil
|
|
99
|
+
)
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// MARK: - Module methods
|
|
104
|
+
|
|
105
|
+
/// v0.18.0 — toggle the `onArFrame` LIGHT-metadata channel. Called
|
|
106
|
+
/// from JS with `true` + the throttle interval (ms) when a host
|
|
107
|
+
/// supplies `<Camera onArFrame={...}>`, and `false` on
|
|
108
|
+
/// unmount / prop-removal. Resolves with no value.
|
|
109
|
+
@objc(setArFrameMetaEnabled:intervalMs:resolver:rejecter:)
|
|
110
|
+
public func setArFrameMetaEnabled(
|
|
111
|
+
enabled: NSNumber,
|
|
112
|
+
intervalMs: NSNumber,
|
|
113
|
+
resolver: @escaping RCTPromiseResolveBlock,
|
|
114
|
+
rejecter: @escaping RCTPromiseRejectBlock
|
|
115
|
+
) {
|
|
116
|
+
RNSARSession.shared.setArFrameMetaEnabled(
|
|
117
|
+
enabled.boolValue,
|
|
118
|
+
intervalMs: intervalMs.doubleValue
|
|
119
|
+
)
|
|
120
|
+
resolver(nil)
|
|
121
|
+
}
|
|
122
|
+
|
|
29
123
|
@objc(isSupported:rejecter:)
|
|
30
124
|
public func isSupported(
|
|
31
125
|
resolver: @escaping RCTPromiseResolveBlock,
|
|
@@ -68,6 +162,47 @@ public final class RNSARSessionBridge: NSObject {
|
|
|
68
162
|
])
|
|
69
163
|
}
|
|
70
164
|
|
|
165
|
+
/// Toggle ARKit scene reconstruction (LiDAR mesh / `ARMeshAnchor`s).
|
|
166
|
+
/// Driven by the <Camera> `enableMesh` prop; gates the
|
|
167
|
+
/// StitcherFrame `meshGeometry` extraction at the SESSION level
|
|
168
|
+
/// (the per-frame `__stitcherProxy.setExtractionConfig(...mesh)`
|
|
169
|
+
/// gates the marshaling — both must be on for a host to receive
|
|
170
|
+
/// mesh). Resolves with no value.
|
|
171
|
+
///
|
|
172
|
+
/// Hops to the main queue: `setSceneReconstructionEnabled` may call
|
|
173
|
+
/// `arSession.run(config)` to reconfigure a live session, and
|
|
174
|
+
/// ARKit session lifecycle must run on the main thread (same
|
|
175
|
+
/// constraint as `start`).
|
|
176
|
+
@objc(setSceneReconstructionEnabled:resolver:rejecter:)
|
|
177
|
+
public func setSceneReconstructionEnabled(
|
|
178
|
+
enabled: NSNumber,
|
|
179
|
+
resolver: @escaping RCTPromiseResolveBlock,
|
|
180
|
+
rejecter: @escaping RCTPromiseRejectBlock
|
|
181
|
+
) {
|
|
182
|
+
let on = enabled.boolValue
|
|
183
|
+
DispatchQueue.main.async {
|
|
184
|
+
RNSARSession.shared.setSceneReconstructionEnabled(on)
|
|
185
|
+
resolver(nil)
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/// Hops to the main queue: `setPlaneDetection` may call
|
|
190
|
+
/// `arSession.run(config)` to reconfigure a live session, and ARKit
|
|
191
|
+
/// session lifecycle must run on the main thread (same constraint as
|
|
192
|
+
/// `start` / `setSceneReconstructionEnabled`).
|
|
193
|
+
@objc(setPlaneDetection:resolver:rejecter:)
|
|
194
|
+
public func setPlaneDetection(
|
|
195
|
+
mode: NSString,
|
|
196
|
+
resolver: @escaping RCTPromiseResolveBlock,
|
|
197
|
+
rejecter: @escaping RCTPromiseRejectBlock
|
|
198
|
+
) {
|
|
199
|
+
let m = mode as String
|
|
200
|
+
DispatchQueue.main.async {
|
|
201
|
+
RNSARSession.shared.setPlaneDetection(m)
|
|
202
|
+
resolver(nil)
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
71
206
|
@objc(snapshotPoseLog:rejecter:)
|
|
72
207
|
public func snapshotPoseLog(
|
|
73
208
|
resolver: @escaping RCTPromiseResolveBlock,
|