react-native-image-stitcher 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +151 -0
- package/RNImageStitcher.podspec +1 -1
- package/android/src/main/cpp/CMakeLists.txt +4 -4
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +216 -7
- package/android/src/main/java/io/imagestitcher/rn/ARFrameContext.kt +89 -0
- package/android/src/main/java/io/imagestitcher/rn/ARFramePlugin.kt +57 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +831 -6
- package/android/src/main/java/io/imagestitcher/rn/RNSARPluginRegistry.kt +109 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +184 -0
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +1 -1
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +84 -2
- package/cpp/{stitcher_frame_data.hpp → camera_frame_data.hpp} +96 -13
- package/cpp/{stitcher_frame_jsi.cpp → camera_frame_jsi.cpp} +154 -11
- package/cpp/{stitcher_frame_jsi.hpp → camera_frame_jsi.hpp} +12 -12
- package/cpp/stitcher_proxy_jsi.cpp +31 -0
- package/cpp/stitcher_proxy_jsi.hpp +16 -0
- package/cpp/stitcher_worklet_dispatch.cpp +5 -5
- package/cpp/stitcher_worklet_dispatch.hpp +5 -5
- package/dist/camera/ARCameraView.d.ts +81 -3
- package/dist/camera/ARCameraView.js +103 -1
- package/dist/camera/Camera.d.ts +73 -7
- package/dist/camera/Camera.js +2 -2
- package/dist/index.d.ts +3 -1
- package/dist/stitching/ARFrameMeta.d.ts +149 -0
- package/dist/stitching/{StitcherFrame.js → ARFrameMeta.js} +1 -1
- package/dist/stitching/{StitcherFrame.d.ts → CameraFrame.d.ts} +70 -11
- package/dist/stitching/CameraFrame.js +4 -0
- package/dist/stitching/useStitcherWorklet.d.ts +4 -4
- package/dist/stitching/useStitcherWorklet.js +4 -4
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +23 -1
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +172 -2
- package/ios/Sources/RNImageStitcher/CameraFrameHostObject.h +108 -0
- package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm +772 -0
- package/ios/Sources/RNImageStitcher/RNISARFramePlugin.swift +247 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +418 -34
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +2 -2
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +4 -4
- package/package.json +1 -1
- package/src/camera/ARCameraView.tsx +230 -5
- package/src/camera/Camera.tsx +91 -7
- package/src/index.ts +12 -3
- package/src/stitching/ARFrameMeta.ts +157 -0
- package/src/stitching/{StitcherFrame.ts → CameraFrame.ts} +79 -11
- package/src/stitching/useStitcherWorklet.ts +9 -9
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
package/dist/camera/Camera.d.ts
CHANGED
|
@@ -41,7 +41,8 @@
|
|
|
41
41
|
import React from 'react';
|
|
42
42
|
import { type StyleProp, type ViewStyle } from 'react-native';
|
|
43
43
|
import type { DrawableFrameProcessor, ReadonlyFrameProcessor } from 'react-native-vision-camera';
|
|
44
|
-
import type {
|
|
44
|
+
import type { CameraFrameProcessor } from '../stitching/CameraFrame';
|
|
45
|
+
import type { ARFrameMeta, ARPluginResult } from '../stitching/ARFrameMeta';
|
|
45
46
|
import { type CaptureHeaderProps } from './CaptureHeader';
|
|
46
47
|
import { type CapturePreviewAction } from './CapturePreview';
|
|
47
48
|
import { type CaptureThumbnailItem } from './CaptureThumbnailStrip';
|
|
@@ -527,11 +528,11 @@ export interface CameraProps {
|
|
|
527
528
|
* worklet to fire on vc's Frame Processor runtime.
|
|
528
529
|
*
|
|
529
530
|
* ```tsx
|
|
530
|
-
* import { Camera, useFrameProcessor, type
|
|
531
|
+
* import { Camera, useFrameProcessor, type CameraFrame }
|
|
531
532
|
* from 'react-native-image-stitcher';
|
|
532
533
|
*
|
|
533
534
|
* function MyScreen() {
|
|
534
|
-
* const fp = useFrameProcessor((frame:
|
|
535
|
+
* const fp = useFrameProcessor((frame: CameraFrame) => {
|
|
535
536
|
* 'worklet';
|
|
536
537
|
* // ...
|
|
537
538
|
* }, []);
|
|
@@ -552,12 +553,12 @@ export interface CameraProps {
|
|
|
552
553
|
* ```tsx
|
|
553
554
|
* import {
|
|
554
555
|
* Camera, useFrameProcessor, useStitcherWorklet,
|
|
555
|
-
* type
|
|
556
|
+
* type CameraFrame,
|
|
556
557
|
* } from 'react-native-image-stitcher';
|
|
557
558
|
*
|
|
558
559
|
* function MyScreen() {
|
|
559
560
|
* const stitcher = useStitcherWorklet();
|
|
560
|
-
* const fp = useFrameProcessor((frame:
|
|
561
|
+
* const fp = useFrameProcessor((frame: CameraFrame) => {
|
|
561
562
|
* 'worklet';
|
|
562
563
|
* hostPreLogic(frame);
|
|
563
564
|
* stitcher.call(frame); // ← first-party stitching
|
|
@@ -601,14 +602,79 @@ export interface CameraProps {
|
|
|
601
602
|
/**
|
|
602
603
|
* AR-mode host worklet, invoked once per ARKit / ARCore frame
|
|
603
604
|
* ALONGSIDE the lib's first-party stitching (composition, not
|
|
604
|
-
* replacement). Receives a `
|
|
605
|
+
* replacement). Receives a `CameraFrame` tagged `source: 'ar'`
|
|
605
606
|
* with world-space `pose` + `arTrackingState`. Only fires in AR
|
|
606
607
|
* capture (`captureSource === 'ar'`); the non-AR equivalent is
|
|
607
608
|
* `frameProcessor` above (the two modes use different runtimes and
|
|
608
609
|
* frame shapes). Must be a `'worklet'`-prefixed function; if the
|
|
609
610
|
* native install is unavailable it silently never fires.
|
|
610
611
|
*/
|
|
611
|
-
arFrameProcessor?:
|
|
612
|
+
arFrameProcessor?: CameraFrameProcessor;
|
|
613
|
+
/**
|
|
614
|
+
* Opt in to per-frame AR depth on the `arFrameProcessor` frame
|
|
615
|
+
* (`CameraFrame.arDepth`). Default `false` — depth is the costliest
|
|
616
|
+
* field (a per-frame buffer copy), so it's off until you need it.
|
|
617
|
+
*/
|
|
618
|
+
enableDepth?: boolean;
|
|
619
|
+
/**
|
|
620
|
+
* Opt in to per-frame AR anchors (`CameraFrame.arAnchors` — detected
|
|
621
|
+
* planes / images). Default `false`.
|
|
622
|
+
*/
|
|
623
|
+
enableAnchors?: boolean;
|
|
624
|
+
/**
|
|
625
|
+
* Opt in to scene-reconstruction mesh anchors (`type: 'mesh'` in
|
|
626
|
+
* `arAnchors`, with `meshGeometry`). Default `false`. iOS enables
|
|
627
|
+
* ARKit `sceneReconstruction` (LiDAR); Android reconstructs a rough
|
|
628
|
+
* mesh from the depth map. Expensive — only on when needed.
|
|
629
|
+
*/
|
|
630
|
+
enableMesh?: boolean;
|
|
631
|
+
/**
|
|
632
|
+
* Which plane orientations to surface in `CameraFrame.arAnchors`
|
|
633
|
+
* (requires `enableAnchors`; AR capture only). Default `'vertical'`
|
|
634
|
+
* — the orientation the plane-projected stitch path has always used.
|
|
635
|
+
* `'horizontal'` surfaces floors / tables; `'both'` surfaces every
|
|
636
|
+
* detected plane. See `ARCameraView` for the per-platform details.
|
|
637
|
+
*/
|
|
638
|
+
planeDetection?: 'vertical' | 'horizontal' | 'both';
|
|
639
|
+
/**
|
|
640
|
+
* v0.18.0 — LIGHT per-frame AR metadata callback, invoked on the JS
|
|
641
|
+
* MAIN thread (NOT a worklet). Only fires in AR capture
|
|
642
|
+
* (`captureSource === 'ar'`). Receives an {@link ARFrameMeta} carrying
|
|
643
|
+
* pose, tracking state, intrinsics, and (when the matching `enable*`
|
|
644
|
+
* prop is on) depth dimensions, anchors, and mesh counts.
|
|
645
|
+
*
|
|
646
|
+
* This is the recommended way to read AR metadata: it sidesteps the
|
|
647
|
+
* worklet path entirely (the `arFrameProcessor` worklet can only safely
|
|
648
|
+
* surface a worklets-core shared value, because capturing a host
|
|
649
|
+
* callback crashes the worklet closure-wrap). Native builds the meta
|
|
650
|
+
* and emits a device event; `<Camera>` threads the handler through to
|
|
651
|
+
* `<ARCameraView>`, which subscribes and invokes it on the main thread.
|
|
652
|
+
*/
|
|
653
|
+
onArFrame?: (meta: ARFrameMeta) => void;
|
|
654
|
+
/**
|
|
655
|
+
* v0.18.0 — throttle interval (ms) for {@link onArFrame}. Default `100`
|
|
656
|
+
* (≈ 10 Hz). No effect unless `onArFrame` is provided.
|
|
657
|
+
*/
|
|
658
|
+
arFrameMetaInterval?: number;
|
|
659
|
+
/**
|
|
660
|
+
* v0.19.0 — ASYNCHRONOUS AR-plugin result callback (the AR plugin
|
|
661
|
+
* framework), invoked on the JS MAIN thread (NOT a worklet). Only fires in
|
|
662
|
+
* AR capture (`captureSource === 'ar'`). Host-registered native plugins
|
|
663
|
+
* (see `RNISARPluginRegistry` / `RNSARPluginRegistry`) that offload heavy
|
|
664
|
+
* per-frame work to their own queue push results via
|
|
665
|
+
* `registry.emit(name, result)`; `<Camera>` threads this handler to
|
|
666
|
+
* `<ARCameraView>`, which subscribes to the `RNImageStitcherARPluginResult`
|
|
667
|
+
* device event and invokes it with `{ plugin, result }`.
|
|
668
|
+
*
|
|
669
|
+
* SYNCHRONOUS plugin results (computed inline on the AR thread) instead ride
|
|
670
|
+
* the throttled {@link onArFrame} event on {@link ARFrameMeta.plugins}.
|
|
671
|
+
* Use `onArFrame` for the in-band sync channel and `onArPluginResult` for
|
|
672
|
+
* the out-of-band async channel — a host can wire either or both.
|
|
673
|
+
*
|
|
674
|
+
* The SDK ships ONLY the generic plugin framework; there are no built-in
|
|
675
|
+
* plugins, so this never fires unless the host registers native plugins.
|
|
676
|
+
*/
|
|
677
|
+
onArPluginResult?: (e: ARPluginResult) => void;
|
|
612
678
|
/**
|
|
613
679
|
* Which device holds the non-AR panorama capture accepts.
|
|
614
680
|
*
|
package/dist/camera/Camera.js
CHANGED
|
@@ -310,7 +310,7 @@ function extractPanoramaOverrides(props) {
|
|
|
310
310
|
* The public `<Camera>` component.
|
|
311
311
|
*/
|
|
312
312
|
function Camera(props) {
|
|
313
|
-
const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, engine = 'batch-keyframe',
|
|
313
|
+
const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, enableDepth, enableAnchors, enableMesh, planeDetection, onArFrame, arFrameMetaInterval, onArPluginResult, engine = 'batch-keyframe',
|
|
314
314
|
// ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
|
|
315
315
|
panMode = 'vertical', panGuidance = true, maxPanDurationMs = 0, panTooFastThreshold, lateralBudgetCm = 4, rectCrop = false, showPreview = false, guidanceCopy, } = props;
|
|
316
316
|
// Derived guidance state. The landscape-only gate decision itself is
|
|
@@ -1425,7 +1425,7 @@ function Camera(props) {
|
|
|
1425
1425
|
// (V12.14.8 OOM fix). The CaptureStatusOverlay renders the
|
|
1426
1426
|
// "Stitching…" state on top, so no placeholder label is needed
|
|
1427
1427
|
// in that case — only for the camera-switch transition.
|
|
1428
|
-
react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
|
|
1428
|
+
react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor, enableDepth: enableDepth, enableAnchors: enableAnchors, enableMesh: enableMesh, planeDetection: planeDetection, onArFrame: onArFrame, arFrameMetaInterval: arFrameMetaInterval, onArPluginResult: onArPluginResult })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
|
|
1429
1429
|
// `video={true}` is REQUIRED for takeSnapshot to work on iOS.
|
|
1430
1430
|
// vision-camera v4's iOS implementation of takeSnapshot waits
|
|
1431
1431
|
// for a frame on the video pipeline; with video disabled, the
|
package/dist/index.d.ts
CHANGED
|
@@ -91,7 +91,9 @@ export { IncrementalOutcome, incrementalStitcherIsAvailable, subscribeIncrementa
|
|
|
91
91
|
export type { IncrementalState, AcceptedKeyframe } from './stitching/incremental';
|
|
92
92
|
export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
|
|
93
93
|
export { useKeyframeStream } from './stitching/useKeyframeStream';
|
|
94
|
-
export type {
|
|
94
|
+
export type { CameraFrame, CameraFrameProcessor, ARAnchor, } from './stitching/CameraFrame';
|
|
95
|
+
export type { ARFrameMeta } from './stitching/ARFrameMeta';
|
|
96
|
+
export type { ARPluginResult } from './stitching/ARFrameMeta';
|
|
95
97
|
export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
|
|
96
98
|
export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
|
|
97
99
|
export { useStitcherWorklet } from './stitching/useStitcherWorklet';
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v0.18.0 — LIGHT per-frame AR metadata delivered to JS on the MAIN
|
|
3
|
+
* thread via a normal callback (`onArFrame`), bypassing worklets entirely.
|
|
4
|
+
*
|
|
5
|
+
* ## Why a callback, NOT a worklet
|
|
6
|
+
*
|
|
7
|
+
* The AR worklet path (`arFrameProcessor` + the `__stitcherProxy` JSI
|
|
8
|
+
* registry) deep-copies the worklet's whole closure into the AR worklet
|
|
9
|
+
* runtime via react-native-worklets-core's `WorkletInvoker`. When the
|
|
10
|
+
* worklet captures a host object (e.g. a `createRunOnJS` callback) that
|
|
11
|
+
* closure-wrap recurses without termination → stack overflow → SIGBUS the
|
|
12
|
+
* instant AR mode mounts (verified on device). Worklets can therefore
|
|
13
|
+
* only safely capture a worklets-core *shared value* — an awkward,
|
|
14
|
+
* poll-from-JS pattern for getting structured data back.
|
|
15
|
+
*
|
|
16
|
+
* `onArFrame` sidesteps the whole problem: native builds the metadata and
|
|
17
|
+
* emits it as a plain `RNImageStitcherARFrame` device event carrying a
|
|
18
|
+
* JSON object; the JS side subscribes via `NativeEventEmitter` and invokes
|
|
19
|
+
* the host callback on the main thread. No worklet, no closure-wrap, no
|
|
20
|
+
* shared-value polling. This is the recommended way to read AR metadata.
|
|
21
|
+
*
|
|
22
|
+
* ## Cost / gating
|
|
23
|
+
*
|
|
24
|
+
* The metadata is intentionally LIGHT — no pixel / vertex / face byte
|
|
25
|
+
* marshaling. `depth` reports only the depth map's dimensions + whether a
|
|
26
|
+
* confidence channel exists (no buffer copy); `mesh` reports only anchor /
|
|
27
|
+
* vertex / face *counts*. Native gates each costly field on the matching
|
|
28
|
+
* extraction flag (`depth` ⇐ `enableDepth`, `mesh` ⇐ `enableMesh`,
|
|
29
|
+
* `anchors` ⇐ `enableAnchors`); `intrinsics` / `pose` / `trackingState` are
|
|
30
|
+
* always present. Emission is gated on a TS-set enabled flag (only true
|
|
31
|
+
* when `onArFrame` is provided) and throttled to `arFrameMetaInterval` ms
|
|
32
|
+
* (default 100 ≈ 10 Hz) on the native side.
|
|
33
|
+
*/
|
|
34
|
+
export interface ARFrameMeta {
|
|
35
|
+
/** Frame-capture timestamp in NANOSECONDS (AR-framework monotonic clock). */
|
|
36
|
+
timestamp: number;
|
|
37
|
+
/** AR tracking quality at this frame. */
|
|
38
|
+
trackingState: 'notAvailable' | 'limited' | 'normal';
|
|
39
|
+
/**
|
|
40
|
+
* Camera pose in world coordinates at frame-capture time.
|
|
41
|
+
*
|
|
42
|
+
* - `rotation` — quaternion `(x, y, z, w)`, matching the convention
|
|
43
|
+
* used throughout the engine + the `CameraFrame.pose` field.
|
|
44
|
+
* - `translation` — metres in world space `[x, y, z]`.
|
|
45
|
+
*/
|
|
46
|
+
pose: {
|
|
47
|
+
rotation: [number, number, number, number];
|
|
48
|
+
translation: [number, number, number];
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Camera intrinsics for this frame — focal lengths (`fx`, `fy`) and
|
|
52
|
+
* principal point (`cx`, `cy`) in PIXELS at the `imageWidth × imageHeight`
|
|
53
|
+
* capture resolution. Always attempted (not gated); `null` only when the
|
|
54
|
+
* AR framework didn't provide them for this frame.
|
|
55
|
+
*/
|
|
56
|
+
intrinsics: {
|
|
57
|
+
fx: number;
|
|
58
|
+
fy: number;
|
|
59
|
+
cx: number;
|
|
60
|
+
cy: number;
|
|
61
|
+
imageWidth: number;
|
|
62
|
+
imageHeight: number;
|
|
63
|
+
} | null;
|
|
64
|
+
/**
|
|
65
|
+
* Depth-map summary — dimensions + whether a per-pixel confidence channel
|
|
66
|
+
* is available. NO pixel buffer is copied (that's the costly part).
|
|
67
|
+
* Present only when the `enableDepth` prop is on AND the device produced a
|
|
68
|
+
* depth map this frame; `null` otherwise.
|
|
69
|
+
*/
|
|
70
|
+
depth: {
|
|
71
|
+
width: number;
|
|
72
|
+
height: number;
|
|
73
|
+
hasConfidence: boolean;
|
|
74
|
+
} | null;
|
|
75
|
+
/**
|
|
76
|
+
* Tracked AR anchors visible in this frame (planes / images / points /
|
|
77
|
+
* mesh). Empty array when `enableAnchors` is on but nothing is tracked;
|
|
78
|
+
* effectively empty when `enableAnchors` is off. `transform` is a 4×4
|
|
79
|
+
* row-major anchor→world matrix (16 numbers).
|
|
80
|
+
*/
|
|
81
|
+
anchors: Array<{
|
|
82
|
+
id: string;
|
|
83
|
+
type: 'plane' | 'image' | 'point' | 'mesh';
|
|
84
|
+
alignment?: 'horizontal' | 'vertical';
|
|
85
|
+
extent?: [number, number];
|
|
86
|
+
classification?: string;
|
|
87
|
+
transform: number[];
|
|
88
|
+
}>;
|
|
89
|
+
/**
|
|
90
|
+
* Scene-reconstruction mesh summary — anchor / vertex / face COUNTS only
|
|
91
|
+
* (no vertex / face byte marshaling). Present only when `enableMesh` is
|
|
92
|
+
* on; `null` otherwise.
|
|
93
|
+
*/
|
|
94
|
+
mesh: {
|
|
95
|
+
anchorCount: number;
|
|
96
|
+
vertexCount: number;
|
|
97
|
+
faceCount: number;
|
|
98
|
+
} | null;
|
|
99
|
+
/**
|
|
100
|
+
* v0.19.0 — SYNCHRONOUS results from host-registered AR frame plugins
|
|
101
|
+
* (the AR plugin framework). Keyed by each plugin's `name()`; the value
|
|
102
|
+
* is the light JSON result the plugin's `process(ctx)` returned on the AR
|
|
103
|
+
* thread (`nil`/`null`-returning plugins are omitted). Only present when
|
|
104
|
+
* the native plugin registry is non-empty AND at least one plugin returned
|
|
105
|
+
* a sync result for this frame; otherwise omitted entirely (zero-plugin
|
|
106
|
+
* apps pay nothing — native skips building the context).
|
|
107
|
+
*
|
|
108
|
+
* The SDK ships ONLY the generic framework — there are no built-in
|
|
109
|
+
* plugins. Hosts register native plugins via `RNISARPluginRegistry`
|
|
110
|
+
* (iOS) / `RNSARPluginRegistry` (Android) at startup; each plugin is
|
|
111
|
+
* called once per AR frame while the registry is non-empty. Result
|
|
112
|
+
* values are `unknown` because each plugin defines its own shape — cast
|
|
113
|
+
* after reading the entry you care about (e.g.
|
|
114
|
+
* `meta.plugins?.brightness as number`).
|
|
115
|
+
*
|
|
116
|
+
* ## Sync vs async results
|
|
117
|
+
*
|
|
118
|
+
* This field carries only the LIGHT, in-band SYNC results (computed fast
|
|
119
|
+
* enough to ride the throttled `onArFrame` event). Plugins that offload
|
|
120
|
+
* heavy work to their own queue deliver results out-of-band via
|
|
121
|
+
* `registry.emit(name, result)`, which surfaces through the separate
|
|
122
|
+
* `onArPluginResult` callback (the `RNImageStitcherARPluginResult`
|
|
123
|
+
* event) — NOT here.
|
|
124
|
+
*/
|
|
125
|
+
plugins?: {
|
|
126
|
+
[name: string]: unknown;
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* v0.19.0 — an ASYNCHRONOUS result from a host-registered AR frame plugin,
|
|
131
|
+
* delivered via the `onArPluginResult` callback.
|
|
132
|
+
*
|
|
133
|
+
* Unlike the in-band SYNC results carried on {@link ARFrameMeta.plugins}
|
|
134
|
+
* (which ride the throttled `onArFrame` event), a plugin produces an async
|
|
135
|
+
* result by offloading heavy work to its own queue and later calling
|
|
136
|
+
* `registry.emit(name, result)` on the native side. The SDK routes that to
|
|
137
|
+
* JS as a `RNImageStitcherARPluginResult` device event; `<ARCameraView>`
|
|
138
|
+
* subscribes and invokes `onArPluginResult` on the JS MAIN thread.
|
|
139
|
+
*
|
|
140
|
+
* `result` is `unknown` because each plugin defines its own result shape —
|
|
141
|
+
* branch on `plugin` (the emitting plugin's `name()`) and cast accordingly.
|
|
142
|
+
*/
|
|
143
|
+
export interface ARPluginResult {
|
|
144
|
+
/** The `name()` of the plugin that emitted this result. */
|
|
145
|
+
plugin: string;
|
|
146
|
+
/** The plugin-defined result payload (cast after branching on `plugin`). */
|
|
147
|
+
result: unknown;
|
|
148
|
+
}
|
|
149
|
+
//# sourceMappingURL=ARFrameMeta.d.ts.map
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* v0.8.0 — unified frame contract for the lib's worklet processor.
|
|
3
3
|
*
|
|
4
4
|
* Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
|
|
5
|
-
* this directory) receive a `
|
|
5
|
+
* this directory) receive a `CameraFrame` regardless of capture mode.
|
|
6
6
|
* The lib-owned worklet runtime guarantees the same JS-visible shape
|
|
7
7
|
* whether the underlying source is a vision-camera `Frame` (non-AR
|
|
8
8
|
* mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
|
|
@@ -16,10 +16,10 @@
|
|
|
16
16
|
* (Phase-0 audit confirmed the iOS path). But vision-camera's
|
|
17
17
|
* **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
|
|
18
18
|
* ARCore does NOT produce `ImageProxy` instances. Forcing
|
|
19
|
-
* `
|
|
19
|
+
* `CameraFrame extends Frame` would either (a) require reverse-
|
|
20
20
|
* engineering ImageProxy on Android (intractable + fragile), or
|
|
21
21
|
* (b) make the type asymmetric per platform. Both are worse than
|
|
22
|
-
* making `
|
|
22
|
+
* making `CameraFrame` a structural sibling type that vc Frames
|
|
23
23
|
* happen to satisfy (because vc Frames carry the same width / height /
|
|
24
24
|
* orientation / pixelFormat / timestamp / toArrayBuffer surface).
|
|
25
25
|
*
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
* a JPEG-encode frame-processor plugin). Returning a reference and
|
|
37
37
|
* reading it later will read into freed memory.
|
|
38
38
|
*/
|
|
39
|
-
export interface
|
|
39
|
+
export interface CameraFrame {
|
|
40
40
|
/** Pixel width of the camera image. */
|
|
41
41
|
width: number;
|
|
42
42
|
/** Pixel height of the camera image. */
|
|
@@ -141,6 +141,27 @@ export interface StitcherFrame {
|
|
|
141
141
|
* tracking is degraded check this. Undefined in non-AR mode.
|
|
142
142
|
*/
|
|
143
143
|
arTrackingState?: 'notAvailable' | 'limited' | 'normal';
|
|
144
|
+
/**
|
|
145
|
+
* Camera intrinsics for THIS frame — focal lengths (`fx`,`fy`) and
|
|
146
|
+
* principal point (`cx`,`cy`) in PIXELS at the `imageWidth × imageHeight`
|
|
147
|
+
* capture resolution. Needed to lift 2D image-space coordinates to 3D
|
|
148
|
+
* via pose + intrinsics (e.g. object-level reconstruction).
|
|
149
|
+
*
|
|
150
|
+
* Populated on **AR frames** (`source: 'ar'`) from ARKit
|
|
151
|
+
* `ARCamera.intrinsics` / ARCore `Camera` intrinsics. **Undefined for
|
|
152
|
+
* non-AR (vision-camera) frames** — they are raw vc `Frame`s without an
|
|
153
|
+
* intrinsics surface; read vc's own APIs there if needed. (The spec
|
|
154
|
+
* called this required; it's optional here because the non-AR frame
|
|
155
|
+
* shape genuinely can't carry it.)
|
|
156
|
+
*/
|
|
157
|
+
intrinsics?: {
|
|
158
|
+
fx: number;
|
|
159
|
+
fy: number;
|
|
160
|
+
cx: number;
|
|
161
|
+
cy: number;
|
|
162
|
+
imageWidth: number;
|
|
163
|
+
imageHeight: number;
|
|
164
|
+
};
|
|
144
165
|
}
|
|
145
166
|
/**
|
|
146
167
|
* v0.8.0 — public AR anchor type. Subset of ARKit/ARCore anchor info
|
|
@@ -150,21 +171,59 @@ export interface StitcherFrame {
|
|
|
150
171
|
export interface ARAnchor {
|
|
151
172
|
/** Stable per-session anchor identifier. */
|
|
152
173
|
id: string;
|
|
153
|
-
/** Anchor kind. `'point'` is Android (ARCore) only. */
|
|
154
|
-
type: 'plane' | 'image' | 'point';
|
|
155
174
|
/**
|
|
156
|
-
*
|
|
157
|
-
*
|
|
175
|
+
* Anchor kind. `'point'` is Android (ARCore) only; `'mesh'` is a
|
|
176
|
+
* scene-reconstruction mesh anchor, present only when the `enableMesh`
|
|
177
|
+
* `<Camera>` prop is on (and the device supports reconstruction).
|
|
178
|
+
*/
|
|
179
|
+
type: 'plane' | 'image' | 'point' | 'mesh';
|
|
180
|
+
/**
|
|
181
|
+
* 4×4 row-major transform from anchor space to world space (16
|
|
182
|
+
* numbers). For `'mesh'` anchors, the `meshGeometry.vertices` are in
|
|
183
|
+
* this anchor's LOCAL space — multiply by `transform` for world coords.
|
|
158
184
|
*/
|
|
159
185
|
transform: number[];
|
|
186
|
+
/**
|
|
187
|
+
* Plane orientation — `'horizontal'` (floor / table / seat) vs
|
|
188
|
+
* `'vertical'` (wall / door / window). Present on `'plane'` anchors;
|
|
189
|
+
* undefined for other anchor kinds. Lets a host distinguish a shelf
|
|
190
|
+
* surface from the wall behind it.
|
|
191
|
+
*/
|
|
192
|
+
alignment?: 'horizontal' | 'vertical';
|
|
193
|
+
/**
|
|
194
|
+
* Plane size in metres along its local x / z axes (`[x, z]`). Present
|
|
195
|
+
* on `'plane'` anchors only.
|
|
196
|
+
*/
|
|
197
|
+
extent?: [number, number];
|
|
198
|
+
/**
|
|
199
|
+
* ARKit semantic classification of the plane's surface, when the
|
|
200
|
+
* framework provides it (iOS; mostly horizontal planes). Undefined
|
|
201
|
+
* when unknown / unsupported (incl. Android, which has no equivalent).
|
|
202
|
+
*/
|
|
203
|
+
classification?: 'wall' | 'floor' | 'ceiling' | 'table' | 'seat' | 'door' | 'window' | 'none';
|
|
204
|
+
/**
|
|
205
|
+
* Scene-reconstruction geometry — present only on `type: 'mesh'`
|
|
206
|
+
* anchors. Buffers (wrap in the noted typed-array view):
|
|
207
|
+
* - `vertices` → `Float32Array`, xyz triplets in anchor-local space.
|
|
208
|
+
* - `faces` → `Uint32Array`, triangle indices into `vertices`.
|
|
209
|
+
* - `classifications` → optional `Uint8Array`, one ARKit mesh class
|
|
210
|
+
* per face (0=none, 1=wall, 2=floor, 3=ceiling, …). **iOS only**
|
|
211
|
+
* (from `ARMeshAnchor`); absent on Android, where the mesh is
|
|
212
|
+
* reconstructed from the depth map and carries no semantics.
|
|
213
|
+
*/
|
|
214
|
+
meshGeometry?: {
|
|
215
|
+
vertices: ArrayBuffer;
|
|
216
|
+
faces: ArrayBuffer;
|
|
217
|
+
classifications?: ArrayBuffer;
|
|
218
|
+
};
|
|
160
219
|
}
|
|
161
220
|
/**
|
|
162
221
|
* v0.8.0 — worklet function signature for the unified frame processor.
|
|
163
222
|
*
|
|
164
223
|
* Must be a `'worklet'`-prefixed function (so it can run on the
|
|
165
|
-
* worklet runtime). Receives a `
|
|
224
|
+
* worklet runtime). Receives a `CameraFrame` per camera frame; the
|
|
166
225
|
* return value is ignored (use `runOnJS` / shared values to surface
|
|
167
226
|
* results back to the JS thread).
|
|
168
227
|
*/
|
|
169
|
-
export type
|
|
170
|
-
//# sourceMappingURL=
|
|
228
|
+
export type CameraFrameProcessor = (frame: CameraFrame) => void;
|
|
229
|
+
//# sourceMappingURL=CameraFrame.d.ts.map
|
|
@@ -106,17 +106,17 @@
|
|
|
106
106
|
* their own worklet via this hook must do the wiring themselves.
|
|
107
107
|
*/
|
|
108
108
|
import type { Frame } from 'react-native-vision-camera';
|
|
109
|
-
import type {
|
|
109
|
+
import type { CameraFrame } from './CameraFrame';
|
|
110
110
|
/**
|
|
111
111
|
* Frames the lib's stitching worklet accepts. Accepting either a
|
|
112
112
|
* vc `Frame` (what the host's `useFrameProcessor` body sees) or the
|
|
113
|
-
* lib's `
|
|
113
|
+
* lib's `CameraFrame` (what the lib's `useFrameProcessor` body
|
|
114
114
|
* sees) keeps the same `useStitcherWorklet` usable from both kinds
|
|
115
115
|
* of host worklet bodies without a cast on the call site. The
|
|
116
116
|
* worklet only reads `width` / `height`; the rest of the frame
|
|
117
117
|
* object is forwarded verbatim to the native plugin.
|
|
118
118
|
*/
|
|
119
|
-
export type StitcherWorkletInput = Frame |
|
|
119
|
+
export type StitcherWorkletInput = Frame | CameraFrame;
|
|
120
120
|
export interface UseStitcherWorkletOptions {
|
|
121
121
|
/**
|
|
122
122
|
* Gyro sample interval in ms (~30 Hz default). Drives the JS-
|
|
@@ -145,7 +145,7 @@ export interface UseStitcherWorkletOptions {
|
|
|
145
145
|
}
|
|
146
146
|
export interface StitcherWorkletHandle {
|
|
147
147
|
/**
|
|
148
|
-
* Worklet function: pass a `
|
|
148
|
+
* Worklet function: pass a `CameraFrame` to perform one frame of
|
|
149
149
|
* the lib's first-party stitching (throttle + pose synthesis +
|
|
150
150
|
* native plugin call). Safe to call from inside another
|
|
151
151
|
* `'worklet'`-prefixed function (this is the canonical
|
|
@@ -228,7 +228,7 @@ function useStitcherWorklet(options = {}) {
|
|
|
228
228
|
// party callback installed in `RNSARWorkletRuntime`). Calling
|
|
229
229
|
// the vc Frame Processor plugin here would throw
|
|
230
230
|
// `getPropertyAsObject: property '__frame' is undefined`
|
|
231
|
-
// because AR frames are `
|
|
231
|
+
// because AR frames are `CameraFrameHostObject` instances
|
|
232
232
|
// and don't carry the vc `Frame` proxy's JSI marker. The
|
|
233
233
|
// throw is caught silently by the per-worklet error handler
|
|
234
234
|
// (`RNSARWorkletRuntime.mm:284-301`) and bubbles up only to
|
|
@@ -240,12 +240,12 @@ function useStitcherWorklet(options = {}) {
|
|
|
240
240
|
// hook (the AR-side stitching path runs natively, independent
|
|
241
241
|
// of the composed worklet body).
|
|
242
242
|
//
|
|
243
|
-
// The `(frame as
|
|
243
|
+
// The `(frame as CameraFrame).source` cast is safe: vc
|
|
244
244
|
// `Frame` doesn't carry a `source` property so the check
|
|
245
245
|
// returns `undefined !== 'ar'` → `true`, and the worklet
|
|
246
246
|
// proceeds normally. Only frames that explicitly tag
|
|
247
247
|
// themselves as AR-source (which our native AR dispatcher
|
|
248
|
-
// does — see `
|
|
248
|
+
// does — see `CameraFrameHostObject.mm`) get short-circuited.
|
|
249
249
|
if (frame.source === 'ar')
|
|
250
250
|
return;
|
|
251
251
|
// Throttle (verbatim from useFrameProcessorDriver).
|
|
@@ -273,7 +273,7 @@ function useStitcherWorklet(options = {}) {
|
|
|
273
273
|
const fx = w * sharedFxNumerator.value;
|
|
274
274
|
const fy = h * sharedFyNumerator.value;
|
|
275
275
|
// vc's `plugin.call` is typed against vc's `Frame`. The worklet
|
|
276
|
-
// accepts the union (`Frame |
|
|
276
|
+
// accepts the union (`Frame | CameraFrame`); cast through
|
|
277
277
|
// `unknown` because the union doesn't satisfy vc's interface
|
|
278
278
|
// even though structurally both members do.
|
|
279
279
|
plugin.call(frame, {
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// imports as `NativeModules.RNSARSession`.
|
|
6
6
|
|
|
7
7
|
#import <React/RCTBridgeModule.h>
|
|
8
|
+
#import <React/RCTEventEmitter.h>
|
|
8
9
|
|
|
9
10
|
// REMAP form, NOT EXTERN_MODULE. The Swift singleton in
|
|
10
11
|
// RNSARSession.swift takes the @objc name "RNSARSession"
|
|
@@ -20,7 +21,14 @@
|
|
|
20
21
|
// `RNSARSessionBridge` and dispatch methods against THAT
|
|
21
22
|
// class — where takePhoto / startRecording / stopRecording etc.
|
|
22
23
|
// actually live.
|
|
23
|
-
|
|
24
|
+
//
|
|
25
|
+
// v0.18.0 — base class is now `RCTEventEmitter` (was `NSObject`) so the
|
|
26
|
+
// "RNSARSession" module can emit the `RNImageStitcherARFrame` device
|
|
27
|
+
// event for the `onArFrame` channel. RN auto-provides the
|
|
28
|
+
// `addListener:` / `removeListeners:` emitter selectors for a module
|
|
29
|
+
// whose remap base is RCTEventEmitter; the Swift class supplies
|
|
30
|
+
// `supportedEvents` / `startObserving` / `stopObserving`.
|
|
31
|
+
@interface RCT_EXTERN_REMAP_MODULE(RNSARSession, RNSARSessionBridge, RCTEventEmitter)
|
|
24
32
|
|
|
25
33
|
RCT_EXTERN_METHOD(isSupported:(RCTPromiseResolveBlock)resolver
|
|
26
34
|
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
@@ -34,6 +42,20 @@ RCT_EXTERN_METHOD(stop:(RCTPromiseResolveBlock)resolver
|
|
|
34
42
|
RCT_EXTERN_METHOD(getState:(RCTPromiseResolveBlock)resolver
|
|
35
43
|
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
36
44
|
|
|
45
|
+
RCT_EXTERN_METHOD(setSceneReconstructionEnabled:(nonnull NSNumber *)enabled
|
|
46
|
+
resolver:(RCTPromiseResolveBlock)resolver
|
|
47
|
+
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
48
|
+
|
|
49
|
+
RCT_EXTERN_METHOD(setPlaneDetection:(nonnull NSString *)mode
|
|
50
|
+
resolver:(RCTPromiseResolveBlock)resolver
|
|
51
|
+
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
52
|
+
|
|
53
|
+
// v0.18.0 — toggle the onArFrame LIGHT-metadata channel + its throttle.
|
|
54
|
+
RCT_EXTERN_METHOD(setArFrameMetaEnabled:(nonnull NSNumber *)enabled
|
|
55
|
+
intervalMs:(nonnull NSNumber *)intervalMs
|
|
56
|
+
resolver:(RCTPromiseResolveBlock)resolver
|
|
57
|
+
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
58
|
+
|
|
37
59
|
RCT_EXTERN_METHOD(snapshotPoseLog:(RCTPromiseResolveBlock)resolver
|
|
38
60
|
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
39
61
|
|