react-native-image-stitcher 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/android/src/main/java/io/imagestitcher/rn/ARFrameContext.kt +89 -0
- package/android/src/main/java/io/imagestitcher/rn/ARFramePlugin.kt +57 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +183 -14
- package/android/src/main/java/io/imagestitcher/rn/RNSARPluginRegistry.kt +109 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +28 -0
- package/dist/camera/ARCameraView.d.ts +22 -1
- package/dist/camera/ARCameraView.js +36 -1
- package/dist/camera/Camera.d.ts +20 -1
- package/dist/camera/Camera.js +2 -2
- package/dist/index.d.ts +1 -0
- package/dist/stitching/ARFrameMeta.d.ts +49 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +36 -1
- package/ios/Sources/RNImageStitcher/CameraFrameHostObject.h +25 -0
- package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm +66 -54
- package/ios/Sources/RNImageStitcher/RNISARFramePlugin.swift +247 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +127 -1
- package/package.json +1 -1
- package/src/camera/ARCameraView.tsx +66 -1
- package/src/camera/Camera.tsx +23 -1
- package/src/index.ts +6 -1
- package/src/stitching/ARFrameMeta.ts +50 -0
package/dist/camera/Camera.d.ts
CHANGED
|
@@ -42,7 +42,7 @@ import React from 'react';
|
|
|
42
42
|
import { type StyleProp, type ViewStyle } from 'react-native';
|
|
43
43
|
import type { DrawableFrameProcessor, ReadonlyFrameProcessor } from 'react-native-vision-camera';
|
|
44
44
|
import type { CameraFrameProcessor } from '../stitching/CameraFrame';
|
|
45
|
-
import type { ARFrameMeta } from '../stitching/ARFrameMeta';
|
|
45
|
+
import type { ARFrameMeta, ARPluginResult } from '../stitching/ARFrameMeta';
|
|
46
46
|
import { type CaptureHeaderProps } from './CaptureHeader';
|
|
47
47
|
import { type CapturePreviewAction } from './CapturePreview';
|
|
48
48
|
import { type CaptureThumbnailItem } from './CaptureThumbnailStrip';
|
|
@@ -656,6 +656,25 @@ export interface CameraProps {
|
|
|
656
656
|
* (≈ 10 Hz). No effect unless `onArFrame` is provided.
|
|
657
657
|
*/
|
|
658
658
|
arFrameMetaInterval?: number;
|
|
659
|
+
/**
|
|
660
|
+
* v0.19.0 — ASYNCHRONOUS AR-plugin result callback (the AR plugin
|
|
661
|
+
* framework), invoked on the JS MAIN thread (NOT a worklet). Only fires in
|
|
662
|
+
* AR capture (`captureSource === 'ar'`). Host-registered native plugins
|
|
663
|
+
* (see `RNISARPluginRegistry` / `RNSARPluginRegistry`) that offload heavy
|
|
664
|
+
* per-frame work to their own queue push results via
|
|
665
|
+
* `registry.emit(name, result)`; `<Camera>` threads this handler to
|
|
666
|
+
* `<ARCameraView>`, which subscribes to the `RNImageStitcherARPluginResult`
|
|
667
|
+
* device event and invokes it with `{ plugin, result }`.
|
|
668
|
+
*
|
|
669
|
+
* SYNCHRONOUS plugin results (computed inline on the AR thread) instead ride
|
|
670
|
+
* the throttled {@link onArFrame} event on {@link ARFrameMeta.plugins}.
|
|
671
|
+
* Use `onArFrame` for the in-band sync channel and `onArPluginResult` for
|
|
672
|
+
* the out-of-band async channel — a host can wire either or both.
|
|
673
|
+
*
|
|
674
|
+
* The SDK ships ONLY the generic plugin framework; there are no built-in
|
|
675
|
+
* plugins, so this never fires unless the host registers native plugins.
|
|
676
|
+
*/
|
|
677
|
+
onArPluginResult?: (e: ARPluginResult) => void;
|
|
659
678
|
/**
|
|
660
679
|
* Which device holds the non-AR panorama capture accepts.
|
|
661
680
|
*
|
package/dist/camera/Camera.js
CHANGED
|
@@ -310,7 +310,7 @@ function extractPanoramaOverrides(props) {
|
|
|
310
310
|
* The public `<Camera>` component.
|
|
311
311
|
*/
|
|
312
312
|
function Camera(props) {
|
|
313
|
-
const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, enableDepth, enableAnchors, enableMesh, planeDetection, onArFrame, arFrameMetaInterval, engine = 'batch-keyframe',
|
|
313
|
+
const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, enableDepth, enableAnchors, enableMesh, planeDetection, onArFrame, arFrameMetaInterval, onArPluginResult, engine = 'batch-keyframe',
|
|
314
314
|
// ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
|
|
315
315
|
panMode = 'vertical', panGuidance = true, maxPanDurationMs = 0, panTooFastThreshold, lateralBudgetCm = 4, rectCrop = false, showPreview = false, guidanceCopy, } = props;
|
|
316
316
|
// Derived guidance state. The landscape-only gate decision itself is
|
|
@@ -1425,7 +1425,7 @@ function Camera(props) {
|
|
|
1425
1425
|
// (V12.14.8 OOM fix). The CaptureStatusOverlay renders the
|
|
1426
1426
|
// "Stitching…" state on top, so no placeholder label is needed
|
|
1427
1427
|
// in that case — only for the camera-switch transition.
|
|
1428
|
-
react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor, enableDepth: enableDepth, enableAnchors: enableAnchors, enableMesh: enableMesh, planeDetection: planeDetection, onArFrame: onArFrame, arFrameMetaInterval: arFrameMetaInterval })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
|
|
1428
|
+
react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor, enableDepth: enableDepth, enableAnchors: enableAnchors, enableMesh: enableMesh, planeDetection: planeDetection, onArFrame: onArFrame, arFrameMetaInterval: arFrameMetaInterval, onArPluginResult: onArPluginResult })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
|
|
1429
1429
|
// `video={true}` is REQUIRED for takeSnapshot to work on iOS.
|
|
1430
1430
|
// vision-camera v4's iOS implementation of takeSnapshot waits
|
|
1431
1431
|
// for a frame on the video pipeline; with video disabled, the
|
package/dist/index.d.ts
CHANGED
|
@@ -93,6 +93,7 @@ export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
|
|
|
93
93
|
export { useKeyframeStream } from './stitching/useKeyframeStream';
|
|
94
94
|
export type { CameraFrame, CameraFrameProcessor, ARAnchor, } from './stitching/CameraFrame';
|
|
95
95
|
export type { ARFrameMeta } from './stitching/ARFrameMeta';
|
|
96
|
+
export type { ARPluginResult } from './stitching/ARFrameMeta';
|
|
96
97
|
export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
|
|
97
98
|
export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
|
|
98
99
|
export { useStitcherWorklet } from './stitching/useStitcherWorklet';
|
|
@@ -96,5 +96,54 @@ export interface ARFrameMeta {
|
|
|
96
96
|
vertexCount: number;
|
|
97
97
|
faceCount: number;
|
|
98
98
|
} | null;
|
|
99
|
+
/**
|
|
100
|
+
* v0.19.0 — SYNCHRONOUS results from host-registered AR frame plugins
|
|
101
|
+
* (the AR plugin framework). Keyed by each plugin's `name()`; the value
|
|
102
|
+
* is the light JSON result the plugin's `process(ctx)` returned on the AR
|
|
103
|
+
* thread (`nil`/`null`-returning plugins are omitted). Only present when
|
|
104
|
+
* the native plugin registry is non-empty AND at least one plugin returned
|
|
105
|
+
* a sync result for this frame; otherwise omitted entirely (zero-plugin
|
|
106
|
+
* apps pay nothing — native skips building the context).
|
|
107
|
+
*
|
|
108
|
+
* The SDK ships ONLY the generic framework — there are no built-in
|
|
109
|
+
* plugins. Hosts register native plugins via `RNISARPluginRegistry`
|
|
110
|
+
* (iOS) / `RNSARPluginRegistry` (Android) at startup; each plugin is
|
|
111
|
+
* called once per AR frame while the registry is non-empty. Result
|
|
112
|
+
* values are `unknown` because each plugin defines its own shape — cast
|
|
113
|
+
* after reading the entry you care about (e.g.
|
|
114
|
+
* `meta.plugins?.brightness as number`).
|
|
115
|
+
*
|
|
116
|
+
* ## Sync vs async results
|
|
117
|
+
*
|
|
118
|
+
* This field carries only the LIGHT, in-band SYNC results (computed fast
|
|
119
|
+
* enough to ride the throttled `onArFrame` event). Plugins that offload
|
|
120
|
+
* heavy work to their own queue deliver results out-of-band via
|
|
121
|
+
* `registry.emit(name, result)`, which surfaces through the separate
|
|
122
|
+
* `onArPluginResult` callback (the `RNImageStitcherARPluginResult`
|
|
123
|
+
* event) — NOT here.
|
|
124
|
+
*/
|
|
125
|
+
plugins?: {
|
|
126
|
+
[name: string]: unknown;
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* v0.19.0 — an ASYNCHRONOUS result from a host-registered AR frame plugin,
|
|
131
|
+
* delivered via the `onArPluginResult` callback.
|
|
132
|
+
*
|
|
133
|
+
* Unlike the in-band SYNC results carried on {@link ARFrameMeta.plugins}
|
|
134
|
+
* (which ride the throttled `onArFrame` event), a plugin produces an async
|
|
135
|
+
* result by offloading heavy work to its own queue and later calling
|
|
136
|
+
* `registry.emit(name, result)` on the native side. The SDK routes that to
|
|
137
|
+
* JS as a `RNImageStitcherARPluginResult` device event; `<ARCameraView>`
|
|
138
|
+
* subscribes and invokes `onArPluginResult` on the JS MAIN thread.
|
|
139
|
+
*
|
|
140
|
+
* `result` is `unknown` because each plugin defines its own result shape —
|
|
141
|
+
* branch on `plugin` (the emitting plugin's `name()`) and cast accordingly.
|
|
142
|
+
*/
|
|
143
|
+
export interface ARPluginResult {
|
|
144
|
+
/** The `name()` of the plugin that emitted this result. */
|
|
145
|
+
plugin: string;
|
|
146
|
+
/** The plugin-defined result payload (cast after branching on `plugin`). */
|
|
147
|
+
result: unknown;
|
|
99
148
|
}
|
|
100
149
|
//# sourceMappingURL=ARFrameMeta.d.ts.map
|
|
@@ -38,6 +38,8 @@ public final class RNSARSessionBridge: RCTEventEmitter {
|
|
|
38
38
|
private var hasListeners: Bool = false
|
|
39
39
|
|
|
40
40
|
private static let arFrameEvent = "RNImageStitcherARFrame"
|
|
41
|
+
// v0.19.0 — async AR-plugin result channel (RNISARPluginRegistry.emit).
|
|
42
|
+
private static let arPluginResultEvent = "RNImageStitcherARPluginResult"
|
|
41
43
|
|
|
42
44
|
public override init() {
|
|
43
45
|
super.init()
|
|
@@ -56,6 +58,20 @@ public final class RNSARSessionBridge: RCTEventEmitter {
|
|
|
56
58
|
name: .retailensARFrameMeta,
|
|
57
59
|
object: nil
|
|
58
60
|
)
|
|
61
|
+
// v0.19.0 — observe the async AR-plugin result channel (posted by
|
|
62
|
+
// `RNISARPluginRegistry.emit`) and re-emit as a JS device event.
|
|
63
|
+
// Same de-dupe rationale as the onArFrame observer above.
|
|
64
|
+
NotificationCenter.default.removeObserver(
|
|
65
|
+
self,
|
|
66
|
+
name: .retailensARPluginResult,
|
|
67
|
+
object: nil
|
|
68
|
+
)
|
|
69
|
+
NotificationCenter.default.addObserver(
|
|
70
|
+
self,
|
|
71
|
+
selector: #selector(handleArPluginResult(_:)),
|
|
72
|
+
name: .retailensARPluginResult,
|
|
73
|
+
object: nil
|
|
74
|
+
)
|
|
59
75
|
}
|
|
60
76
|
|
|
61
77
|
deinit {
|
|
@@ -71,7 +87,7 @@ public final class RNSARSessionBridge: RCTEventEmitter {
|
|
|
71
87
|
}
|
|
72
88
|
|
|
73
89
|
public override func supportedEvents() -> [String]! {
|
|
74
|
-
return [Self.arFrameEvent]
|
|
90
|
+
return [Self.arFrameEvent, Self.arPluginResultEvent]
|
|
75
91
|
}
|
|
76
92
|
|
|
77
93
|
public override func startObserving() {
|
|
@@ -100,6 +116,25 @@ public final class RNSARSessionBridge: RCTEventEmitter {
|
|
|
100
116
|
}
|
|
101
117
|
}
|
|
102
118
|
|
|
119
|
+
/// v0.19.0 — forward a posted async AR-plugin result
|
|
120
|
+
/// (`{ plugin, result }`, from `RNISARPluginRegistry.emit`) to JS as
|
|
121
|
+
/// the `RNImageStitcherARPluginResult` device event. Dropped when no
|
|
122
|
+
/// JS listener is attached. Emits via `enqueueJSCall` on the main
|
|
123
|
+
/// queue (same `sendEvent`-avoidance rationale as `handleArFrameMeta`).
|
|
124
|
+
@objc private func handleArPluginResult(_ notification: Notification) {
|
|
125
|
+
guard hasListeners else { return }
|
|
126
|
+
guard let userInfo = notification.userInfo else { return }
|
|
127
|
+
DispatchQueue.main.async { [weak self] in
|
|
128
|
+
guard let self = self, let bridge = self.bridge else { return }
|
|
129
|
+
bridge.enqueueJSCall(
|
|
130
|
+
"RCTDeviceEventEmitter",
|
|
131
|
+
method: "emit",
|
|
132
|
+
args: [Self.arPluginResultEvent, userInfo],
|
|
133
|
+
completion: nil
|
|
134
|
+
)
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
103
138
|
// MARK: - Module methods
|
|
104
139
|
|
|
105
140
|
/// v0.18.0 — toggle the `onArFrame` LIGHT-metadata channel. Called
|
|
@@ -78,6 +78,31 @@ NS_SWIFT_NAME(CameraFrameHostObject)
|
|
|
78
78
|
pose:(RNSARFramePose *)pose
|
|
79
79
|
NS_SWIFT_NAME(lightArFrameMeta(from:pose:));
|
|
80
80
|
|
|
81
|
+
/// Build the gated anchor-dictionary array for an ARFrame — the SAME
|
|
82
|
+
/// `[{ id, type, alignment?, extent?, classification?, transform }]`
|
|
83
|
+
/// shape the `onArFrame` LIGHT meta surfaces (mesh anchors excluded;
|
|
84
|
+
/// they're summarised under `mesh`). Returns an EMPTY array when the
|
|
85
|
+
/// JS `enableAnchors` flag is off (read from the shared C++ extraction
|
|
86
|
+
/// config) — matching the TS contract's `Array<...>` (never null).
|
|
87
|
+
///
|
|
88
|
+
/// Extracted so BOTH the `onArFrame` light-meta path AND the v0.19.0 AR
|
|
89
|
+
/// plugin context (`RNISARFrameContext.anchors`) build anchors from one
|
|
90
|
+
/// source of truth (DRY).
|
|
91
|
+
///
|
|
92
|
+
/// Thread: safe to call from the ARSession delegate queue (reads the
|
|
93
|
+
/// frame synchronously; copies nothing that outlives the call).
|
|
94
|
+
+ (NSArray<NSDictionary *> *)arAnchorDictsFromFrame:(ARFrame *)arFrame
|
|
95
|
+
NS_SWIFT_NAME(arAnchorDicts(from:));
|
|
96
|
+
|
|
97
|
+
/// Whether the shared C++ extraction config currently has `enableDepth`
|
|
98
|
+
/// on (the `<Camera enableDepth>` prop, set via JS
|
|
99
|
+
/// `__stitcherProxy.setExtractionConfig`). The v0.19.0 AR plugin
|
|
100
|
+
/// context uses this to decide whether to expose the raw
|
|
101
|
+
/// `ARFrame.sceneDepth` depthBuffer to plugins — gating the config read
|
|
102
|
+
/// (C++) in the .mm so Swift doesn't need the C++ header.
|
|
103
|
+
+ (BOOL)arExtractionDepthEnabled
|
|
104
|
+
NS_SWIFT_NAME(arExtractionDepthEnabled());
|
|
105
|
+
|
|
81
106
|
@end
|
|
82
107
|
|
|
83
108
|
NS_ASSUME_NONNULL_END
|
|
@@ -653,60 +653,15 @@ void ExtractARMesh(ARFrame* arFrame, retailens::CameraFrameData& data) {
|
|
|
653
653
|
meta[@"depth"] = depthValue;
|
|
654
654
|
|
|
655
655
|
// anchors: id / coarse type / row-major 4x4 transform, plus plane
|
|
656
|
-
// alignment + extent + (capable-device) classification.
|
|
657
|
-
//
|
|
658
|
-
//
|
|
659
|
-
//
|
|
660
|
-
//
|
|
661
|
-
//
|
|
662
|
-
//
|
|
663
|
-
//
|
|
664
|
-
|
|
665
|
-
if (cfg.anchors) {
|
|
666
|
-
for (ARAnchor *a in arFrame.anchors) {
|
|
667
|
-
// ARMeshAnchors are summarised under `mesh`; skip here.
|
|
668
|
-
if ([a isKindOfClass:[ARMeshAnchor class]]) continue;
|
|
669
|
-
|
|
670
|
-
NSMutableDictionary *anchor = [NSMutableDictionary dictionary];
|
|
671
|
-
anchor[@"id"] = a.identifier.UUIDString;
|
|
672
|
-
|
|
673
|
-
if ([a isKindOfClass:[ARPlaneAnchor class]]) {
|
|
674
|
-
anchor[@"type"] = @"plane";
|
|
675
|
-
ARPlaneAnchor *plane = (ARPlaneAnchor *)a;
|
|
676
|
-
anchor[@"alignment"] =
|
|
677
|
-
(plane.alignment == ARPlaneAnchorAlignmentVertical) ? @"vertical"
|
|
678
|
-
: @"horizontal";
|
|
679
|
-
// [extentX, extentZ] in plane-local metres (deprecated `extent`
|
|
680
|
-
// for iOS-15 parity, same as ExtractARAnchors).
|
|
681
|
-
anchor[@"extent"] = @[ @(plane.extent.x), @(plane.extent.z) ];
|
|
682
|
-
if (ARPlaneAnchor.isClassificationSupported &&
|
|
683
|
-
plane.classificationStatus == ARPlaneClassificationStatusKnown) {
|
|
684
|
-
std::string cls = PlaneClassificationString(plane.classification);
|
|
685
|
-
if (!cls.empty()) {
|
|
686
|
-
anchor[@"classification"] =
|
|
687
|
-
[NSString stringWithUTF8String:cls.c_str()];
|
|
688
|
-
}
|
|
689
|
-
}
|
|
690
|
-
} else if ([a isKindOfClass:[ARImageAnchor class]]) {
|
|
691
|
-
anchor[@"type"] = @"image";
|
|
692
|
-
} else {
|
|
693
|
-
anchor[@"type"] = @"point";
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
// Row-major anchor->world (transpose ARKit's column-major matrix),
|
|
697
|
-
// 16 NSNumbers — same transpose as ExtractARAnchors.
|
|
698
|
-
const simd_float4x4 m = a.transform;
|
|
699
|
-
NSMutableArray *transform = [NSMutableArray arrayWithCapacity:16];
|
|
700
|
-
for (int r = 0; r < 4; ++r) {
|
|
701
|
-
for (int c = 0; c < 4; ++c) {
|
|
702
|
-
[transform addObject:@((double)m.columns[c][r])];
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
anchor[@"transform"] = transform;
|
|
706
|
-
[anchorsOut addObject:anchor];
|
|
707
|
-
}
|
|
708
|
-
}
|
|
709
|
-
meta[@"anchors"] = anchorsOut;
|
|
656
|
+
// alignment + extent + (capable-device) classification. Built by the
|
|
657
|
+
// shared `arAnchorDictsFromFrame:` helper (DRY — same source the
|
|
658
|
+
// v0.19.0 AR plugin context reuses). Empty array when the prop is off
|
|
659
|
+
// (cheap + JSON-stable, matching the TS contract's `Array<...>` rather
|
|
660
|
+
// than null). Mesh anchors are summarised under `mesh` (counts) below,
|
|
661
|
+
// NOT listed individually — to match Android's collectTrackingAnchors
|
|
662
|
+
// which surfaces plane/image anchors and emits mesh as a separate
|
|
663
|
+
// summary.
|
|
664
|
+
meta[@"anchors"] = [self arAnchorDictsFromFrame:arFrame];
|
|
710
665
|
|
|
711
666
|
// mesh: anchor / vertex / face COUNTS only (no vertex/face byte
|
|
712
667
|
// marshaling). null when the prop is off. Counts are read from each
|
|
@@ -741,6 +696,63 @@ void ExtractARMesh(ARFrame* arFrame, retailens::CameraFrameData& data) {
|
|
|
741
696
|
return meta;
|
|
742
697
|
}
|
|
743
698
|
|
|
699
|
+
+ (NSArray<NSDictionary *> *)arAnchorDictsFromFrame:(ARFrame *)arFrame {
|
|
700
|
+
// Gate on the shared C++ extraction config's `anchors` flag — when the
|
|
701
|
+
// <Camera enableAnchors> prop is off, return an empty array (JSON-
|
|
702
|
+
// stable, matches the TS `Array<...>` contract; never null).
|
|
703
|
+
NSMutableArray<NSDictionary *> *anchorsOut = [NSMutableArray array];
|
|
704
|
+
const retailens::ExtractionConfig cfg = retailens::getExtractionConfig();
|
|
705
|
+
if (!cfg.anchors || arFrame == nil) return anchorsOut;
|
|
706
|
+
|
|
707
|
+
for (ARAnchor *a in arFrame.anchors) {
|
|
708
|
+
// ARMeshAnchors are summarised under `mesh` (counts), not listed here.
|
|
709
|
+
if ([a isKindOfClass:[ARMeshAnchor class]]) continue;
|
|
710
|
+
|
|
711
|
+
NSMutableDictionary *anchor = [NSMutableDictionary dictionary];
|
|
712
|
+
anchor[@"id"] = a.identifier.UUIDString;
|
|
713
|
+
|
|
714
|
+
if ([a isKindOfClass:[ARPlaneAnchor class]]) {
|
|
715
|
+
anchor[@"type"] = @"plane";
|
|
716
|
+
ARPlaneAnchor *plane = (ARPlaneAnchor *)a;
|
|
717
|
+
anchor[@"alignment"] =
|
|
718
|
+
(plane.alignment == ARPlaneAnchorAlignmentVertical) ? @"vertical"
|
|
719
|
+
: @"horizontal";
|
|
720
|
+
// [extentX, extentZ] in plane-local metres (deprecated `extent` for
|
|
721
|
+
// iOS-15 parity, same as ExtractARAnchors).
|
|
722
|
+
anchor[@"extent"] = @[ @(plane.extent.x), @(plane.extent.z) ];
|
|
723
|
+
if (ARPlaneAnchor.isClassificationSupported &&
|
|
724
|
+
plane.classificationStatus == ARPlaneClassificationStatusKnown) {
|
|
725
|
+
std::string cls = PlaneClassificationString(plane.classification);
|
|
726
|
+
if (!cls.empty()) {
|
|
727
|
+
anchor[@"classification"] =
|
|
728
|
+
[NSString stringWithUTF8String:cls.c_str()];
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
} else if ([a isKindOfClass:[ARImageAnchor class]]) {
|
|
732
|
+
anchor[@"type"] = @"image";
|
|
733
|
+
} else {
|
|
734
|
+
anchor[@"type"] = @"point";
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
// Row-major anchor->world (transpose ARKit's column-major matrix),
|
|
738
|
+
// 16 NSNumbers — same transpose as ExtractARAnchors.
|
|
739
|
+
const simd_float4x4 m = a.transform;
|
|
740
|
+
NSMutableArray *transform = [NSMutableArray arrayWithCapacity:16];
|
|
741
|
+
for (int r = 0; r < 4; ++r) {
|
|
742
|
+
for (int c = 0; c < 4; ++c) {
|
|
743
|
+
[transform addObject:@((double)m.columns[c][r])];
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
anchor[@"transform"] = transform;
|
|
747
|
+
[anchorsOut addObject:anchor];
|
|
748
|
+
}
|
|
749
|
+
return anchorsOut;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
+ (BOOL)arExtractionDepthEnabled {
|
|
753
|
+
return retailens::getExtractionConfig().depth ? YES : NO;
|
|
754
|
+
}
|
|
755
|
+
|
|
744
756
|
- (void)invalidate {
|
|
745
757
|
if (_hostObject) {
|
|
746
758
|
_hostObject->invalidate();
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// RNISARFramePlugin — v0.19.0 native AR plugin framework (iOS).
|
|
4
|
+
//
|
|
5
|
+
// The SDK owns the ARSession and drives one per-frame callback path
|
|
6
|
+
// (`RNSARSession.session(_:didUpdate:)`). Host apps that need to run
|
|
7
|
+
// heavier native per-frame analysis (OCR, barcode reading, ML
|
|
8
|
+
// inference, …) register a *native* plugin against this framework — the
|
|
9
|
+
// SDK ships ONLY the generic plumbing; no OCR or other concrete plugin.
|
|
10
|
+
//
|
|
11
|
+
// The ergonomics mirror vision-camera's FrameProcessorPlugin
|
|
12
|
+
// registration: the host conforms a class to `RNISARFramePlugin`,
|
|
13
|
+
// registers it once at startup via `RNISARPluginRegistry.shared`, and
|
|
14
|
+
// the SDK calls `process(_:)` on the AR thread for every ARFrame while
|
|
15
|
+
// the registry is non-empty.
|
|
16
|
+
//
|
|
17
|
+
// Two result channels:
|
|
18
|
+
// 1. SYNC — `process(_:)` returns a light `[String: Any]?`. Non-nil
|
|
19
|
+
// results are folded into the throttled `onArFrame` `ARFrameMeta`
|
|
20
|
+
// under `plugins: { [name]: result }`, riding the existing
|
|
21
|
+
// `RNImageStitcherARFrame` device event. Use for cheap, per-frame
|
|
22
|
+
// scalars (brightness, a quick blur score, …).
|
|
23
|
+
// 2. ASYNC — the plugin offloads heavy work to its own queue and later
|
|
24
|
+
// calls `RNISARPluginRegistry.shared.emit(name, result)`, which the
|
|
25
|
+
// SDK re-emits as the `RNImageStitcherARPluginResult` device event
|
|
26
|
+
// `{ plugin, result }`. Use for OCR / ML whose latency exceeds a
|
|
27
|
+
// frame interval.
|
|
28
|
+
//
|
|
29
|
+
// PERFORMANCE CONTRACT: the SDK only builds the `RNISARFrameContext` and
|
|
30
|
+
// calls plugins when the registry is NON-EMPTY, so a zero-plugin app
|
|
31
|
+
// pays nothing on the AR hot path. Plugins MUST self-throttle (the SDK
|
|
32
|
+
// calls `process(_:)` on every ARFrame) and MUST offload anything
|
|
33
|
+
// heavier than a few hundred microseconds.
|
|
34
|
+
|
|
35
|
+
import Foundation
|
|
36
|
+
import ARKit
|
|
37
|
+
import CoreVideo
|
|
38
|
+
|
|
39
|
+
// MARK: - Async result event channel
|
|
40
|
+
|
|
41
|
+
/// v0.19.0 — async AR-plugin result channel. `RNISARPluginRegistry.emit`
|
|
42
|
+
/// posts this notification (carrying `{ plugin, result }`); the
|
|
43
|
+
/// `RNSARSessionBridge` (an RCTEventEmitter) observes it and re-emits as
|
|
44
|
+
/// the JS `RNImageStitcherARPluginResult` device event. We route via
|
|
45
|
+
/// NotificationCenter — rather than the registry holding a bridge
|
|
46
|
+
/// reference — mirroring the `.retailensARFrameMeta` (`onArFrame`)
|
|
47
|
+
/// channel, so the framework-free engine pattern is preserved.
|
|
48
|
+
public extension Notification.Name {
|
|
49
|
+
static let retailensARPluginResult =
|
|
50
|
+
Notification.Name("RNImageStitcherARPluginResult")
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
// MARK: - Plugin protocol
|
|
55
|
+
|
|
56
|
+
/// A native AR frame plugin. Host apps conform a class to this and
|
|
57
|
+
/// register it once via `RNISARPluginRegistry.shared.register(_:)`.
|
|
58
|
+
///
|
|
59
|
+
/// The SDK calls `process(_:)` once per ARFrame on the AR (ARSession
|
|
60
|
+
/// delegate) thread while the registry is non-empty. Return a light
|
|
61
|
+
/// `[String: Any]?` for the SYNC channel, or `nil`; for heavy work,
|
|
62
|
+
/// offload to your own queue and later call
|
|
63
|
+
/// `RNISARPluginRegistry.shared.emit(name(), result)` for the ASYNC
|
|
64
|
+
/// channel.
|
|
65
|
+
@objc public protocol RNISARFramePlugin: AnyObject {
|
|
66
|
+
/// Stable identifier for this plugin. Used as the key in the
|
|
67
|
+
/// `onArFrame` meta's `plugins` map AND as the `plugin` field of the
|
|
68
|
+
/// async `RNImageStitcherARPluginResult` event. Keep it constant for
|
|
69
|
+
/// the plugin's lifetime; the registry stores plugins keyed by name
|
|
70
|
+
/// (a second `register` with the same name replaces the first).
|
|
71
|
+
func name() -> String
|
|
72
|
+
|
|
73
|
+
/// Called on the AR thread once per ARFrame while the registry is
|
|
74
|
+
/// non-empty. Return a light JSON-safe result for the SYNC channel
|
|
75
|
+
/// (NSNumber / NSString / NSArray / NSDictionary leaves) or `nil`.
|
|
76
|
+
///
|
|
77
|
+
/// LIFETIME: `context.pixelBuffer` (and `context.depthBuffer`) are the
|
|
78
|
+
/// live ARFrame buffers — VALID ONLY for the duration of this call.
|
|
79
|
+
/// ARKit recycles them once `process(_:)` returns. If you offload
|
|
80
|
+
/// work to another thread/queue, you MUST copy the bytes you need
|
|
81
|
+
/// BEFORE returning. Do NOT retain the CVPixelBuffer expecting the
|
|
82
|
+
/// pixels to survive — a CF retain does not protect against ARKit's
|
|
83
|
+
/// pool reuse.
|
|
84
|
+
func process(_ context: RNISARFrameContext) -> [String: Any]?
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
// MARK: - Per-frame context
|
|
89
|
+
|
|
90
|
+
/// Zero-copy native view of one ARFrame, handed to each plugin's
|
|
91
|
+
/// `process(_:)`. Exposes the live capture buffer + pose + intrinsics +
|
|
92
|
+
/// (opt-in) depth + (opt-in) anchors. Nothing here is copied for the
|
|
93
|
+
/// plugin's benefit; the buffers belong to ARKit and are valid only
|
|
94
|
+
/// during the synchronous `process(_:)` call (see the lifetime note on
|
|
95
|
+
/// `RNISARFramePlugin.process(_:)`).
|
|
96
|
+
@objc(RNISARFrameContext)
|
|
97
|
+
public final class RNISARFrameContext: NSObject {
|
|
98
|
+
|
|
99
|
+
/// The ARFrame's `capturedImage` (BGRA/YUV `CVPixelBuffer`). VALID
|
|
100
|
+
/// ONLY during `process(_:)` — copy before offloading (see the
|
|
101
|
+
/// lifetime note on `RNISARFramePlugin.process(_:)`).
|
|
102
|
+
@objc public let pixelBuffer: CVPixelBuffer
|
|
103
|
+
|
|
104
|
+
/// Frame timestamp in NANOSECONDS (AR-framework monotonic clock) —
|
|
105
|
+
/// matches `CameraFrame.timestampNs` and the `ARFrameMeta.timestamp`
|
|
106
|
+
/// contract.
|
|
107
|
+
@objc public let timestampNs: Double
|
|
108
|
+
|
|
109
|
+
/// Camera intrinsics (pixels). `imageWidth`/`imageHeight` are the
|
|
110
|
+
/// capture resolution the intrinsics are expressed against.
|
|
111
|
+
@objc public let fx: Double
|
|
112
|
+
@objc public let fy: Double
|
|
113
|
+
@objc public let cx: Double
|
|
114
|
+
@objc public let cy: Double
|
|
115
|
+
@objc public let imageWidth: Int
|
|
116
|
+
@objc public let imageHeight: Int
|
|
117
|
+
|
|
118
|
+
/// World-space camera pose: rotation as a unit quaternion
|
|
119
|
+
/// `[x, y, z, w]` and translation `[x, y, z]` in metres (ARKit's
|
|
120
|
+
/// right-handed, Y-up, -Z-forward world frame).
|
|
121
|
+
@objc public let poseRotation: [Double]
|
|
122
|
+
@objc public let poseTranslation: [Double]
|
|
123
|
+
|
|
124
|
+
/// Tracking quality: `"notAvailable"` / `"limited"` / `"normal"`.
|
|
125
|
+
@objc public let trackingState: String
|
|
126
|
+
|
|
127
|
+
/// The ARFrame's `sceneDepth` (or `smoothedSceneDepth`) depth map
|
|
128
|
+
/// `CVPixelBuffer` — `nil` unless the `<Camera enableDepth>` prop is
|
|
129
|
+
/// on AND the device produced depth this frame. VALID ONLY during
|
|
130
|
+
/// `process(_:)`; copy before offloading.
|
|
131
|
+
@objc public let depthBuffer: CVPixelBuffer?
|
|
132
|
+
|
|
133
|
+
/// Tracking anchors as the same light dicts the `onArFrame` meta
|
|
134
|
+
/// surfaces (`{ id, type, alignment?, extent?, classification?,
|
|
135
|
+
/// transform }`; mesh anchors excluded). EMPTY unless the
|
|
136
|
+
/// `<Camera enableAnchors>` prop is on.
|
|
137
|
+
@objc public let anchors: [[String: Any]]
|
|
138
|
+
|
|
139
|
+
@objc public init(
|
|
140
|
+
pixelBuffer: CVPixelBuffer,
|
|
141
|
+
timestampNs: Double,
|
|
142
|
+
fx: Double, fy: Double, cx: Double, cy: Double,
|
|
143
|
+
imageWidth: Int, imageHeight: Int,
|
|
144
|
+
poseRotation: [Double],
|
|
145
|
+
poseTranslation: [Double],
|
|
146
|
+
trackingState: String,
|
|
147
|
+
depthBuffer: CVPixelBuffer?,
|
|
148
|
+
anchors: [[String: Any]]
|
|
149
|
+
) {
|
|
150
|
+
self.pixelBuffer = pixelBuffer
|
|
151
|
+
self.timestampNs = timestampNs
|
|
152
|
+
self.fx = fx; self.fy = fy; self.cx = cx; self.cy = cy
|
|
153
|
+
self.imageWidth = imageWidth
|
|
154
|
+
self.imageHeight = imageHeight
|
|
155
|
+
self.poseRotation = poseRotation
|
|
156
|
+
self.poseTranslation = poseTranslation
|
|
157
|
+
self.trackingState = trackingState
|
|
158
|
+
self.depthBuffer = depthBuffer
|
|
159
|
+
self.anchors = anchors
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
// MARK: - Registry
|
|
165
|
+
|
|
166
|
+
/// Process-wide registry of native AR plugins + the async result router.
|
|
167
|
+
///
|
|
168
|
+
/// The host registers plugins at startup (e.g. in the AppDelegate); the
|
|
169
|
+
/// SDK reads `plugins()` on the AR thread each frame. Also the entry
|
|
170
|
+
/// point for the ASYNC channel: a plugin calls `emit(name, result)` from
|
|
171
|
+
/// its own queue and the SDK re-emits a `RNImageStitcherARPluginResult`
|
|
172
|
+
/// JS event.
|
|
173
|
+
///
|
|
174
|
+
/// THREAD SAFETY: registration (host/startup thread) and reads (AR
|
|
175
|
+
/// thread) are serialised by an internal lock. `plugins()` returns a
|
|
176
|
+
/// snapshot array so the AR thread can iterate without holding the lock.
|
|
177
|
+
@objc(RNISARPluginRegistry)
|
|
178
|
+
public final class RNISARPluginRegistry: NSObject {
|
|
179
|
+
|
|
180
|
+
/// Shared instance — the only way hosts register plugins.
|
|
181
|
+
@objc public static let shared = RNISARPluginRegistry()
|
|
182
|
+
|
|
183
|
+
/// Registered plugins, keyed by `name()` for O(1) replace/unregister.
|
|
184
|
+
/// Insertion order is preserved for deterministic `process(_:)`
|
|
185
|
+
/// ordering via a parallel ordered key list.
|
|
186
|
+
private var pluginsByName: [String: RNISARFramePlugin] = [:]
|
|
187
|
+
private var order: [String] = []
|
|
188
|
+
private let lock = NSLock()
|
|
189
|
+
|
|
190
|
+
private override init() { super.init() }
|
|
191
|
+
|
|
192
|
+
/// Register (or replace) a plugin. Keyed by `plugin.name()`: a
|
|
193
|
+
/// second register with the same name replaces the first (and keeps
|
|
194
|
+
/// its position in the ordering). Idempotent for the same instance.
|
|
195
|
+
@objc public func register(_ plugin: RNISARFramePlugin) {
|
|
196
|
+
let key = plugin.name()
|
|
197
|
+
lock.lock()
|
|
198
|
+
defer { lock.unlock() }
|
|
199
|
+
if pluginsByName[key] == nil {
|
|
200
|
+
order.append(key)
|
|
201
|
+
}
|
|
202
|
+
pluginsByName[key] = plugin
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/// Remove the plugin registered under `name`. No-op if absent.
|
|
206
|
+
@objc public func unregister(_ name: String) {
|
|
207
|
+
lock.lock()
|
|
208
|
+
defer { lock.unlock() }
|
|
209
|
+
pluginsByName.removeValue(forKey: name)
|
|
210
|
+
order.removeAll { $0 == name }
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/// Snapshot of registered plugins in registration order. Returns a
|
|
214
|
+
/// copy so the AR thread can iterate without holding the lock.
|
|
215
|
+
@objc public func plugins() -> [RNISARFramePlugin] {
|
|
216
|
+
lock.lock()
|
|
217
|
+
defer { lock.unlock() }
|
|
218
|
+
return order.compactMap { pluginsByName[$0] }
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/// Whether any plugin is registered. Cheap gate the SDK checks per
|
|
222
|
+
/// ARFrame before building the (per-frame) `RNISARFrameContext`.
|
|
223
|
+
@objc public var isEmpty: Bool {
|
|
224
|
+
lock.lock()
|
|
225
|
+
defer { lock.unlock() }
|
|
226
|
+
return order.isEmpty
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/// ASYNC channel — route a plugin's later-computed result to JS as the
|
|
230
|
+
/// `RNImageStitcherARPluginResult` device event `{ plugin, result }`.
|
|
231
|
+
/// Safe to call from any thread (the plugin's own queue); posts on
|
|
232
|
+
/// NotificationCenter, which the bridge observes + re-emits on the
|
|
233
|
+
/// main queue.
|
|
234
|
+
///
|
|
235
|
+
/// `pluginName` should match the plugin's `name()` so JS can correlate
|
|
236
|
+
/// the result with its source.
|
|
237
|
+
@objc public func emit(_ pluginName: String, _ result: [String: Any]) {
|
|
238
|
+
NotificationCenter.default.post(
|
|
239
|
+
name: .retailensARPluginResult,
|
|
240
|
+
object: nil,
|
|
241
|
+
userInfo: [
|
|
242
|
+
"plugin": pluginName,
|
|
243
|
+
"result": result,
|
|
244
|
+
]
|
|
245
|
+
)
|
|
246
|
+
}
|
|
247
|
+
}
|