react-native-image-stitcher 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,7 +42,7 @@ import React from 'react';
42
42
  import { type StyleProp, type ViewStyle } from 'react-native';
43
43
  import type { DrawableFrameProcessor, ReadonlyFrameProcessor } from 'react-native-vision-camera';
44
44
  import type { CameraFrameProcessor } from '../stitching/CameraFrame';
45
- import type { ARFrameMeta } from '../stitching/ARFrameMeta';
45
+ import type { ARFrameMeta, ARPluginResult } from '../stitching/ARFrameMeta';
46
46
  import { type CaptureHeaderProps } from './CaptureHeader';
47
47
  import { type CapturePreviewAction } from './CapturePreview';
48
48
  import { type CaptureThumbnailItem } from './CaptureThumbnailStrip';
@@ -656,6 +656,25 @@ export interface CameraProps {
656
656
  * (≈ 10 Hz). No effect unless `onArFrame` is provided.
657
657
  */
658
658
  arFrameMetaInterval?: number;
659
+ /**
660
+ * v0.19.0 — ASYNCHRONOUS AR-plugin result callback (the AR plugin
661
+ * framework), invoked on the JS MAIN thread (NOT a worklet). Only fires in
662
+ * AR capture (`captureSource === 'ar'`). Host-registered native plugins
663
+ * (see `RNISARPluginRegistry` / `RNSARPluginRegistry`) that offload heavy
664
+ * per-frame work to their own queue push results via
665
+ * `registry.emit(name, result)`; `<Camera>` threads this handler to
666
+ * `<ARCameraView>`, which subscribes to the `RNImageStitcherARPluginResult`
667
+ * device event and invokes it with `{ plugin, result }`.
668
+ *
669
+ * SYNCHRONOUS plugin results (computed inline on the AR thread) instead ride
670
+ * the throttled {@link onArFrame} event on {@link ARFrameMeta.plugins}.
671
+ * Use `onArFrame` for the in-band sync channel and `onArPluginResult` for
672
+ * the out-of-band async channel — a host can wire either or both.
673
+ *
674
+ * The SDK ships ONLY the generic plugin framework; there are no built-in
675
+ * plugins, so this never fires unless the host registers native plugins.
676
+ */
677
+ onArPluginResult?: (e: ARPluginResult) => void;
659
678
  /**
660
679
  * Which device holds the non-AR panorama capture accepts.
661
680
  *
@@ -310,7 +310,7 @@ function extractPanoramaOverrides(props) {
310
310
  * The public `<Camera>` component.
311
311
  */
312
312
  function Camera(props) {
313
- const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, enableDepth, enableAnchors, enableMesh, planeDetection, onArFrame, arFrameMetaInterval, engine = 'batch-keyframe',
313
+ const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, enableDepth, enableAnchors, enableMesh, planeDetection, onArFrame, arFrameMetaInterval, onArPluginResult, engine = 'batch-keyframe',
314
314
  // ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
315
315
  panMode = 'vertical', panGuidance = true, maxPanDurationMs = 0, panTooFastThreshold, lateralBudgetCm = 4, rectCrop = false, showPreview = false, guidanceCopy, } = props;
316
316
  // Derived guidance state. The landscape-only gate decision itself is
@@ -1425,7 +1425,7 @@ function Camera(props) {
1425
1425
  // (V12.14.8 OOM fix). The CaptureStatusOverlay renders the
1426
1426
  // "Stitching…" state on top, so no placeholder label is needed
1427
1427
  // in that case — only for the camera-switch transition.
1428
- react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor, enableDepth: enableDepth, enableAnchors: enableAnchors, enableMesh: enableMesh, planeDetection: planeDetection, onArFrame: onArFrame, arFrameMetaInterval: arFrameMetaInterval })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
1428
+ react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor, enableDepth: enableDepth, enableAnchors: enableAnchors, enableMesh: enableMesh, planeDetection: planeDetection, onArFrame: onArFrame, arFrameMetaInterval: arFrameMetaInterval, onArPluginResult: onArPluginResult })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
1429
1429
  // `video={true}` is REQUIRED for takeSnapshot to work on iOS.
1430
1430
  // vision-camera v4's iOS implementation of takeSnapshot waits
1431
1431
  // for a frame on the video pipeline; with video disabled, the
package/dist/index.d.ts CHANGED
@@ -93,6 +93,7 @@ export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
93
93
  export { useKeyframeStream } from './stitching/useKeyframeStream';
94
94
  export type { CameraFrame, CameraFrameProcessor, ARAnchor, } from './stitching/CameraFrame';
95
95
  export type { ARFrameMeta } from './stitching/ARFrameMeta';
96
+ export type { ARPluginResult } from './stitching/ARFrameMeta';
96
97
  export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
97
98
  export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
98
99
  export { useStitcherWorklet } from './stitching/useStitcherWorklet';
@@ -96,5 +96,54 @@ export interface ARFrameMeta {
96
96
  vertexCount: number;
97
97
  faceCount: number;
98
98
  } | null;
99
+ /**
100
+ * v0.19.0 — SYNCHRONOUS results from host-registered AR frame plugins
101
+ * (the AR plugin framework). Keyed by each plugin's `name()`; the value
102
+ * is the light JSON result the plugin's `process(ctx)` returned on the AR
103
+ * thread (`nil`/`null`-returning plugins are omitted). Only present when
104
+ * the native plugin registry is non-empty AND at least one plugin returned
105
+ * a sync result for this frame; otherwise omitted entirely (zero-plugin
106
+ * apps pay nothing — native skips building the context).
107
+ *
108
+ * The SDK ships ONLY the generic framework — there are no built-in
109
+ * plugins. Hosts register native plugins via `RNISARPluginRegistry`
110
+ * (iOS) / `RNSARPluginRegistry` (Android) at startup; each plugin is
111
+ * called once per AR frame while the registry is non-empty. Result
112
+ * values are `unknown` because each plugin defines its own shape — cast
113
+ * after reading the entry you care about (e.g.
114
+ * `meta.plugins?.brightness as number`).
115
+ *
116
+ * ## Sync vs async results
117
+ *
118
+ * This field carries only the LIGHT, in-band SYNC results (computed fast
119
+ * enough to ride the throttled `onArFrame` event). Plugins that offload
120
+ * heavy work to their own queue deliver results out-of-band via
121
+ * `registry.emit(name, result)`, which surfaces through the separate
122
+ * `onArPluginResult` callback (the `RNImageStitcherARPluginResult`
123
+ * event) — NOT here.
124
+ */
125
+ plugins?: {
126
+ [name: string]: unknown;
127
+ };
128
+ }
129
+ /**
130
+ * v0.19.0 — an ASYNCHRONOUS result from a host-registered AR frame plugin,
131
+ * delivered via the `onArPluginResult` callback.
132
+ *
133
+ * Unlike the in-band SYNC results carried on {@link ARFrameMeta.plugins}
134
+ * (which ride the throttled `onArFrame` event), a plugin produces an async
135
+ * result by offloading heavy work to its own queue and later calling
136
+ * `registry.emit(name, result)` on the native side. The SDK routes that to
137
+ * JS as a `RNImageStitcherARPluginResult` device event; `<ARCameraView>`
138
+ * subscribes and invokes `onArPluginResult` on the JS MAIN thread.
139
+ *
140
+ * `result` is `unknown` because each plugin defines its own result shape —
141
+ * branch on `plugin` (the emitting plugin's `name()`) and cast accordingly.
142
+ */
143
+ export interface ARPluginResult {
144
+ /** The `name()` of the plugin that emitted this result. */
145
+ plugin: string;
146
+ /** The plugin-defined result payload (cast after branching on `plugin`). */
147
+ result: unknown;
99
148
  }
100
149
  //# sourceMappingURL=ARFrameMeta.d.ts.map
@@ -38,6 +38,8 @@ public final class RNSARSessionBridge: RCTEventEmitter {
38
38
  private var hasListeners: Bool = false
39
39
 
40
40
  private static let arFrameEvent = "RNImageStitcherARFrame"
41
+ // v0.19.0 — async AR-plugin result channel (RNISARPluginRegistry.emit).
42
+ private static let arPluginResultEvent = "RNImageStitcherARPluginResult"
41
43
 
42
44
  public override init() {
43
45
  super.init()
@@ -56,6 +58,20 @@ public final class RNSARSessionBridge: RCTEventEmitter {
56
58
  name: .retailensARFrameMeta,
57
59
  object: nil
58
60
  )
61
+ // v0.19.0 — observe the async AR-plugin result channel (posted by
62
+ // `RNISARPluginRegistry.emit`) and re-emit as a JS device event.
63
+ // Same de-dupe rationale as the onArFrame observer above.
64
+ NotificationCenter.default.removeObserver(
65
+ self,
66
+ name: .retailensARPluginResult,
67
+ object: nil
68
+ )
69
+ NotificationCenter.default.addObserver(
70
+ self,
71
+ selector: #selector(handleArPluginResult(_:)),
72
+ name: .retailensARPluginResult,
73
+ object: nil
74
+ )
59
75
  }
60
76
 
61
77
  deinit {
@@ -71,7 +87,7 @@ public final class RNSARSessionBridge: RCTEventEmitter {
71
87
  }
72
88
 
73
89
  public override func supportedEvents() -> [String]! {
74
- return [Self.arFrameEvent]
90
+ return [Self.arFrameEvent, Self.arPluginResultEvent]
75
91
  }
76
92
 
77
93
  public override func startObserving() {
@@ -100,6 +116,25 @@ public final class RNSARSessionBridge: RCTEventEmitter {
100
116
  }
101
117
  }
102
118
 
119
+ /// v0.19.0 — forward a posted async AR-plugin result
120
+ /// (`{ plugin, result }`, from `RNISARPluginRegistry.emit`) to JS as
121
+ /// the `RNImageStitcherARPluginResult` device event. Dropped when no
122
+ /// JS listener is attached. Emits via `enqueueJSCall` on the main
123
+ /// queue (same `sendEvent`-avoidance rationale as `handleArFrameMeta`).
124
+ @objc private func handleArPluginResult(_ notification: Notification) {
125
+ guard hasListeners else { return }
126
+ guard let userInfo = notification.userInfo else { return }
127
+ DispatchQueue.main.async { [weak self] in
128
+ guard let self = self, let bridge = self.bridge else { return }
129
+ bridge.enqueueJSCall(
130
+ "RCTDeviceEventEmitter",
131
+ method: "emit",
132
+ args: [Self.arPluginResultEvent, userInfo],
133
+ completion: nil
134
+ )
135
+ }
136
+ }
137
+
103
138
  // MARK: - Module methods
104
139
 
105
140
  /// v0.18.0 — toggle the `onArFrame` LIGHT-metadata channel. Called
@@ -78,6 +78,31 @@ NS_SWIFT_NAME(CameraFrameHostObject)
78
78
  pose:(RNSARFramePose *)pose
79
79
  NS_SWIFT_NAME(lightArFrameMeta(from:pose:));
80
80
 
81
+ /// Build the gated anchor-dictionary array for an ARFrame — the SAME
82
+ /// `[{ id, type, alignment?, extent?, classification?, transform }]`
83
+ /// shape the `onArFrame` LIGHT meta surfaces (mesh anchors excluded;
84
+ /// they're summarised under `mesh`). Returns an EMPTY array when the
85
+ /// JS `enableAnchors` flag is off (read from the shared C++ extraction
86
+ /// config) — matching the TS contract's `Array<...>` (never null).
87
+ ///
88
+ /// Extracted so BOTH the `onArFrame` light-meta path AND the v0.19.0 AR
89
+ /// plugin context (`RNISARFrameContext.anchors`) build anchors from one
90
+ /// source of truth (DRY).
91
+ ///
92
+ /// Thread: safe to call from the ARSession delegate queue (reads the
93
+ /// frame synchronously; copies nothing that outlives the call).
94
+ + (NSArray<NSDictionary *> *)arAnchorDictsFromFrame:(ARFrame *)arFrame
95
+ NS_SWIFT_NAME(arAnchorDicts(from:));
96
+
97
+ /// Whether the shared C++ extraction config currently has `enableDepth`
98
+ /// on (the `<Camera enableDepth>` prop, set via JS
99
+ /// `__stitcherProxy.setExtractionConfig`). The v0.19.0 AR plugin
100
+ /// context uses this to decide whether to expose the raw
101
+ /// `ARFrame.sceneDepth` depthBuffer to plugins — gating the config read
102
+ /// (C++) in the .mm so Swift doesn't need the C++ header.
103
+ + (BOOL)arExtractionDepthEnabled
104
+ NS_SWIFT_NAME(arExtractionDepthEnabled());
105
+
81
106
  @end
82
107
 
83
108
  NS_ASSUME_NONNULL_END
@@ -653,60 +653,15 @@ void ExtractARMesh(ARFrame* arFrame, retailens::CameraFrameData& data) {
653
653
  meta[@"depth"] = depthValue;
654
654
 
655
655
  // anchors: id / coarse type / row-major 4x4 transform, plus plane
656
- // alignment + extent + (capable-device) classification. Mirrors
657
- // ExtractARAnchors above but into an NSDictionary array (no byte
658
- // marshaling that path is for the full host object). Empty array
659
- // when the prop is off (cheap + JSON-stable, matching the TS contract's
660
- // `Array<...>` rather than null). Mesh anchors are summarised under
661
- // `mesh` (counts) below, NOT listed individually here unless enableMesh
662
- // is off to match Android's collectTrackingAnchors which surfaces
663
- // plane/image anchors and emits mesh as a separate summary.
664
- NSMutableArray *anchorsOut = [NSMutableArray array];
665
- if (cfg.anchors) {
666
- for (ARAnchor *a in arFrame.anchors) {
667
- // ARMeshAnchors are summarised under `mesh`; skip here.
668
- if ([a isKindOfClass:[ARMeshAnchor class]]) continue;
669
-
670
- NSMutableDictionary *anchor = [NSMutableDictionary dictionary];
671
- anchor[@"id"] = a.identifier.UUIDString;
672
-
673
- if ([a isKindOfClass:[ARPlaneAnchor class]]) {
674
- anchor[@"type"] = @"plane";
675
- ARPlaneAnchor *plane = (ARPlaneAnchor *)a;
676
- anchor[@"alignment"] =
677
- (plane.alignment == ARPlaneAnchorAlignmentVertical) ? @"vertical"
678
- : @"horizontal";
679
- // [extentX, extentZ] in plane-local metres (deprecated `extent`
680
- // for iOS-15 parity, same as ExtractARAnchors).
681
- anchor[@"extent"] = @[ @(plane.extent.x), @(plane.extent.z) ];
682
- if (ARPlaneAnchor.isClassificationSupported &&
683
- plane.classificationStatus == ARPlaneClassificationStatusKnown) {
684
- std::string cls = PlaneClassificationString(plane.classification);
685
- if (!cls.empty()) {
686
- anchor[@"classification"] =
687
- [NSString stringWithUTF8String:cls.c_str()];
688
- }
689
- }
690
- } else if ([a isKindOfClass:[ARImageAnchor class]]) {
691
- anchor[@"type"] = @"image";
692
- } else {
693
- anchor[@"type"] = @"point";
694
- }
695
-
696
- // Row-major anchor->world (transpose ARKit's column-major matrix),
697
- // 16 NSNumbers — same transpose as ExtractARAnchors.
698
- const simd_float4x4 m = a.transform;
699
- NSMutableArray *transform = [NSMutableArray arrayWithCapacity:16];
700
- for (int r = 0; r < 4; ++r) {
701
- for (int c = 0; c < 4; ++c) {
702
- [transform addObject:@((double)m.columns[c][r])];
703
- }
704
- }
705
- anchor[@"transform"] = transform;
706
- [anchorsOut addObject:anchor];
707
- }
708
- }
709
- meta[@"anchors"] = anchorsOut;
656
+ // alignment + extent + (capable-device) classification. Built by the
657
+ // shared `arAnchorDictsFromFrame:` helper (DRY same source the
658
+ // v0.19.0 AR plugin context reuses). Empty array when the prop is off
659
+ // (cheap + JSON-stable, matching the TS contract's `Array<...>` rather
660
+ // than null). Mesh anchors are summarised under `mesh` (counts) below,
661
+ // NOT listed individually to match Android's collectTrackingAnchors
662
+ // which surfaces plane/image anchors and emits mesh as a separate
663
+ // summary.
664
+ meta[@"anchors"] = [self arAnchorDictsFromFrame:arFrame];
710
665
 
711
666
  // mesh: anchor / vertex / face COUNTS only (no vertex/face byte
712
667
  // marshaling). null when the prop is off. Counts are read from each
@@ -741,6 +696,63 @@ void ExtractARMesh(ARFrame* arFrame, retailens::CameraFrameData& data) {
741
696
  return meta;
742
697
  }
743
698
 
699
+ + (NSArray<NSDictionary *> *)arAnchorDictsFromFrame:(ARFrame *)arFrame {
700
+ // Gate on the shared C++ extraction config's `anchors` flag — when the
701
+ // <Camera enableAnchors> prop is off, return an empty array (JSON-
702
+ // stable, matches the TS `Array<...>` contract; never null).
703
+ NSMutableArray<NSDictionary *> *anchorsOut = [NSMutableArray array];
704
+ const retailens::ExtractionConfig cfg = retailens::getExtractionConfig();
705
+ if (!cfg.anchors || arFrame == nil) return anchorsOut;
706
+
707
+ for (ARAnchor *a in arFrame.anchors) {
708
+ // ARMeshAnchors are summarised under `mesh` (counts), not listed here.
709
+ if ([a isKindOfClass:[ARMeshAnchor class]]) continue;
710
+
711
+ NSMutableDictionary *anchor = [NSMutableDictionary dictionary];
712
+ anchor[@"id"] = a.identifier.UUIDString;
713
+
714
+ if ([a isKindOfClass:[ARPlaneAnchor class]]) {
715
+ anchor[@"type"] = @"plane";
716
+ ARPlaneAnchor *plane = (ARPlaneAnchor *)a;
717
+ anchor[@"alignment"] =
718
+ (plane.alignment == ARPlaneAnchorAlignmentVertical) ? @"vertical"
719
+ : @"horizontal";
720
+ // [extentX, extentZ] in plane-local metres (deprecated `extent` for
721
+ // iOS-15 parity, same as ExtractARAnchors).
722
+ anchor[@"extent"] = @[ @(plane.extent.x), @(plane.extent.z) ];
723
+ if (ARPlaneAnchor.isClassificationSupported &&
724
+ plane.classificationStatus == ARPlaneClassificationStatusKnown) {
725
+ std::string cls = PlaneClassificationString(plane.classification);
726
+ if (!cls.empty()) {
727
+ anchor[@"classification"] =
728
+ [NSString stringWithUTF8String:cls.c_str()];
729
+ }
730
+ }
731
+ } else if ([a isKindOfClass:[ARImageAnchor class]]) {
732
+ anchor[@"type"] = @"image";
733
+ } else {
734
+ anchor[@"type"] = @"point";
735
+ }
736
+
737
+ // Row-major anchor->world (transpose ARKit's column-major matrix),
738
+ // 16 NSNumbers — same transpose as ExtractARAnchors.
739
+ const simd_float4x4 m = a.transform;
740
+ NSMutableArray *transform = [NSMutableArray arrayWithCapacity:16];
741
+ for (int r = 0; r < 4; ++r) {
742
+ for (int c = 0; c < 4; ++c) {
743
+ [transform addObject:@((double)m.columns[c][r])];
744
+ }
745
+ }
746
+ anchor[@"transform"] = transform;
747
+ [anchorsOut addObject:anchor];
748
+ }
749
+ return anchorsOut;
750
+ }
751
+
752
+ + (BOOL)arExtractionDepthEnabled {
753
+ return retailens::getExtractionConfig().depth ? YES : NO;
754
+ }
755
+
744
756
  - (void)invalidate {
745
757
  if (_hostObject) {
746
758
  _hostObject->invalidate();
@@ -0,0 +1,247 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ //
3
+ // RNISARFramePlugin — v0.19.0 native AR plugin framework (iOS).
4
+ //
5
+ // The SDK owns the ARSession and drives one per-frame callback path
6
+ // (`RNSARSession.session(_:didUpdate:)`). Host apps that need to run
7
+ // heavier native per-frame analysis (OCR, barcode reading, ML
8
+ // inference, …) register a *native* plugin against this framework — the
9
+ // SDK ships ONLY the generic plumbing; no OCR or other concrete plugin.
10
+ //
11
+ // The ergonomics mirror vision-camera's FrameProcessorPlugin
12
+ // registration: the host conforms a class to `RNISARFramePlugin`,
13
+ // registers it once at startup via `RNISARPluginRegistry.shared`, and
14
+ // the SDK calls `process(_:)` on the AR thread for every ARFrame while
15
+ // the registry is non-empty.
16
+ //
17
+ // Two result channels:
18
+ // 1. SYNC — `process(_:)` returns a light `[String: Any]?`. Non-nil
19
+ // results are folded into the throttled `onArFrame` `ARFrameMeta`
20
+ // under `plugins: { [name]: result }`, riding the existing
21
+ // `RNImageStitcherARFrame` device event. Use for cheap, per-frame
22
+ // scalars (brightness, a quick blur score, …).
23
+ // 2. ASYNC — the plugin offloads heavy work to its own queue and later
24
+ // calls `RNISARPluginRegistry.shared.emit(name, result)`, which the
25
+ // SDK re-emits as the `RNImageStitcherARPluginResult` device event
26
+ // `{ plugin, result }`. Use for OCR / ML whose latency exceeds a
27
+ // frame interval.
28
+ //
29
+ // PERFORMANCE CONTRACT: the SDK only builds the `RNISARFrameContext` and
30
+ // calls plugins when the registry is NON-EMPTY, so a zero-plugin app
31
+ // pays nothing on the AR hot path. Plugins MUST self-throttle (the SDK
32
+ // calls `process(_:)` on every ARFrame) and MUST offload anything
33
+ // heavier than a few hundred microseconds.
34
+
35
+ import Foundation
36
+ import ARKit
37
+ import CoreVideo
38
+
39
+ // MARK: - Async result event channel
40
+
41
+ /// v0.19.0 — async AR-plugin result channel. `RNISARPluginRegistry.emit`
42
+ /// posts this notification (carrying `{ plugin, result }`); the
43
+ /// `RNSARSessionBridge` (an RCTEventEmitter) observes it and re-emits as
44
+ /// the JS `RNImageStitcherARPluginResult` device event. We route via
45
+ /// NotificationCenter — rather than the registry holding a bridge
46
+ /// reference — mirroring the `.retailensARFrameMeta` (`onArFrame`)
47
+ /// channel, so the framework-free engine pattern is preserved.
48
+ public extension Notification.Name {
49
+ static let retailensARPluginResult =
50
+ Notification.Name("RNImageStitcherARPluginResult")
51
+ }
52
+
53
+
54
+ // MARK: - Plugin protocol
55
+
56
+ /// A native AR frame plugin. Host apps conform a class to this and
57
+ /// register it once via `RNISARPluginRegistry.shared.register(_:)`.
58
+ ///
59
+ /// The SDK calls `process(_:)` once per ARFrame on the AR (ARSession
60
+ /// delegate) thread while the registry is non-empty. Return a light
61
+ /// `[String: Any]?` for the SYNC channel, or `nil`; for heavy work,
62
+ /// offload to your own queue and later call
63
+ /// `RNISARPluginRegistry.shared.emit(name(), result)` for the ASYNC
64
+ /// channel.
65
+ @objc public protocol RNISARFramePlugin: AnyObject {
66
+ /// Stable identifier for this plugin. Used as the key in the
67
+ /// `onArFrame` meta's `plugins` map AND as the `plugin` field of the
68
+ /// async `RNImageStitcherARPluginResult` event. Keep it constant for
69
+ /// the plugin's lifetime; the registry stores plugins keyed by name
70
+ /// (a second `register` with the same name replaces the first).
71
+ func name() -> String
72
+
73
+ /// Called on the AR thread once per ARFrame while the registry is
74
+ /// non-empty. Return a light JSON-safe result for the SYNC channel
75
+ /// (NSNumber / NSString / NSArray / NSDictionary leaves) or `nil`.
76
+ ///
77
+ /// LIFETIME: `context.pixelBuffer` (and `context.depthBuffer`) are the
78
+ /// live ARFrame buffers — VALID ONLY for the duration of this call.
79
+ /// ARKit recycles them once `process(_:)` returns. If you offload
80
+ /// work to another thread/queue, you MUST copy the bytes you need
81
+ /// BEFORE returning. Do NOT retain the CVPixelBuffer expecting the
82
+ /// pixels to survive — a CF retain does not protect against ARKit's
83
+ /// pool reuse.
84
+ func process(_ context: RNISARFrameContext) -> [String: Any]?
85
+ }
86
+
87
+
88
+ // MARK: - Per-frame context
89
+
90
+ /// Zero-copy native view of one ARFrame, handed to each plugin's
91
+ /// `process(_:)`. Exposes the live capture buffer + pose + intrinsics +
92
+ /// (opt-in) depth + (opt-in) anchors. Nothing here is copied for the
93
+ /// plugin's benefit; the buffers belong to ARKit and are valid only
94
+ /// during the synchronous `process(_:)` call (see the lifetime note on
95
+ /// `RNISARFramePlugin.process(_:)`).
96
+ @objc(RNISARFrameContext)
97
+ public final class RNISARFrameContext: NSObject {
98
+
99
+ /// The ARFrame's `capturedImage` (BGRA/YUV `CVPixelBuffer`). VALID
100
+ /// ONLY during `process(_:)` — copy before offloading (see the
101
+ /// lifetime note on `RNISARFramePlugin.process(_:)`).
102
+ @objc public let pixelBuffer: CVPixelBuffer
103
+
104
+ /// Frame timestamp in NANOSECONDS (AR-framework monotonic clock) —
105
+ /// matches `CameraFrame.timestampNs` and the `ARFrameMeta.timestamp`
106
+ /// contract.
107
+ @objc public let timestampNs: Double
108
+
109
+ /// Camera intrinsics (pixels). `imageWidth`/`imageHeight` are the
110
+ /// capture resolution the intrinsics are expressed against.
111
+ @objc public let fx: Double
112
+ @objc public let fy: Double
113
+ @objc public let cx: Double
114
+ @objc public let cy: Double
115
+ @objc public let imageWidth: Int
116
+ @objc public let imageHeight: Int
117
+
118
+ /// World-space camera pose: rotation as a unit quaternion
119
+ /// `[x, y, z, w]` and translation `[x, y, z]` in metres (ARKit's
120
+ /// right-handed, Y-up, -Z-forward world frame).
121
+ @objc public let poseRotation: [Double]
122
+ @objc public let poseTranslation: [Double]
123
+
124
+ /// Tracking quality: `"notAvailable"` / `"limited"` / `"normal"`.
125
+ @objc public let trackingState: String
126
+
127
+ /// The ARFrame's `sceneDepth` (or `smoothedSceneDepth`) depth map
128
+ /// `CVPixelBuffer` — `nil` unless the `<Camera enableDepth>` prop is
129
+ /// on AND the device produced depth this frame. VALID ONLY during
130
+ /// `process(_:)`; copy before offloading.
131
+ @objc public let depthBuffer: CVPixelBuffer?
132
+
133
+ /// Tracking anchors as the same light dicts the `onArFrame` meta
134
+ /// surfaces (`{ id, type, alignment?, extent?, classification?,
135
+ /// transform }`; mesh anchors excluded). EMPTY unless the
136
+ /// `<Camera enableAnchors>` prop is on.
137
+ @objc public let anchors: [[String: Any]]
138
+
139
+ @objc public init(
140
+ pixelBuffer: CVPixelBuffer,
141
+ timestampNs: Double,
142
+ fx: Double, fy: Double, cx: Double, cy: Double,
143
+ imageWidth: Int, imageHeight: Int,
144
+ poseRotation: [Double],
145
+ poseTranslation: [Double],
146
+ trackingState: String,
147
+ depthBuffer: CVPixelBuffer?,
148
+ anchors: [[String: Any]]
149
+ ) {
150
+ self.pixelBuffer = pixelBuffer
151
+ self.timestampNs = timestampNs
152
+ self.fx = fx; self.fy = fy; self.cx = cx; self.cy = cy
153
+ self.imageWidth = imageWidth
154
+ self.imageHeight = imageHeight
155
+ self.poseRotation = poseRotation
156
+ self.poseTranslation = poseTranslation
157
+ self.trackingState = trackingState
158
+ self.depthBuffer = depthBuffer
159
+ self.anchors = anchors
160
+ }
161
+ }
162
+
163
+
164
+ // MARK: - Registry
165
+
166
+ /// Process-wide registry of native AR plugins + the async result router.
167
+ ///
168
+ /// The host registers plugins at startup (e.g. in the AppDelegate); the
169
+ /// SDK reads `plugins()` on the AR thread each frame. Also the entry
170
+ /// point for the ASYNC channel: a plugin calls `emit(name, result)` from
171
+ /// its own queue and the SDK re-emits a `RNImageStitcherARPluginResult`
172
+ /// JS event.
173
+ ///
174
+ /// THREAD SAFETY: registration (host/startup thread) and reads (AR
175
+ /// thread) are serialised by an internal lock. `plugins()` returns a
176
+ /// snapshot array so the AR thread can iterate without holding the lock.
177
+ @objc(RNISARPluginRegistry)
178
+ public final class RNISARPluginRegistry: NSObject {
179
+
180
+ /// Shared instance — the only way hosts register plugins.
181
+ @objc public static let shared = RNISARPluginRegistry()
182
+
183
+ /// Registered plugins, keyed by `name()` for O(1) replace/unregister.
184
+ /// Insertion order is preserved for deterministic `process(_:)`
185
+ /// ordering via a parallel ordered key list.
186
+ private var pluginsByName: [String: RNISARFramePlugin] = [:]
187
+ private var order: [String] = []
188
+ private let lock = NSLock()
189
+
190
+ private override init() { super.init() }
191
+
192
+ /// Register (or replace) a plugin. Keyed by `plugin.name()`: a
193
+ /// second register with the same name replaces the first (and keeps
194
+ /// its position in the ordering). Idempotent for the same instance.
195
+ @objc public func register(_ plugin: RNISARFramePlugin) {
196
+ let key = plugin.name()
197
+ lock.lock()
198
+ defer { lock.unlock() }
199
+ if pluginsByName[key] == nil {
200
+ order.append(key)
201
+ }
202
+ pluginsByName[key] = plugin
203
+ }
204
+
205
+ /// Remove the plugin registered under `name`. No-op if absent.
206
+ @objc public func unregister(_ name: String) {
207
+ lock.lock()
208
+ defer { lock.unlock() }
209
+ pluginsByName.removeValue(forKey: name)
210
+ order.removeAll { $0 == name }
211
+ }
212
+
213
+ /// Snapshot of registered plugins in registration order. Returns a
214
+ /// copy so the AR thread can iterate without holding the lock.
215
+ @objc public func plugins() -> [RNISARFramePlugin] {
216
+ lock.lock()
217
+ defer { lock.unlock() }
218
+ return order.compactMap { pluginsByName[$0] }
219
+ }
220
+
221
+ /// Whether any plugin is registered. Cheap gate the SDK checks per
222
+ /// ARFrame before building the (per-frame) `RNISARFrameContext`.
223
+ @objc public var isEmpty: Bool {
224
+ lock.lock()
225
+ defer { lock.unlock() }
226
+ return order.isEmpty
227
+ }
228
+
229
+ /// ASYNC channel — route a plugin's later-computed result to JS as the
230
+ /// `RNImageStitcherARPluginResult` device event `{ plugin, result }`.
231
+ /// Safe to call from any thread (the plugin's own queue); posts on
232
+ /// NotificationCenter, which the bridge observes + re-emits on the
233
+ /// main queue.
234
+ ///
235
+ /// `pluginName` should match the plugin's `name()` so JS can correlate
236
+ /// the result with its source.
237
+ @objc public func emit(_ pluginName: String, _ result: [String: Any]) {
238
+ NotificationCenter.default.post(
239
+ name: .retailensARPluginResult,
240
+ object: nil,
241
+ userInfo: [
242
+ "plugin": pluginName,
243
+ "result": result,
244
+ ]
245
+ )
246
+ }
247
+ }