react-native-image-stitcher 0.18.0 → 0.20.0

package/ios/Sources/RNImageStitcher/RNSARCameraView.swift

@@ -31,17 +31,37 @@
 
 import Foundation
 import ARKit
+import SceneKit
 import UIKit
+import simd
 
 
 @objc(RNSARCameraView)
-public final class RNSARCameraView: UIView {
+public final class RNSARCameraView: UIView, ARSCNViewDelegate {
 
     /// The ARSCNView that does the actual rendering.  Bound to the
     /// singleton's ARSession so all preview surfaces share the same
     /// session (and the same pose log that the stitcher consumes).
     private var arSCNView: ARSCNView!
 
+    /// v0.20.0 — world-anchored overlays backed by REAL `ARAnchor`s.  Each
+    /// overlay becomes an `ARAnchor` added to the session; ARSCNView creates a
+    /// node per anchor (via `renderer(_:nodeFor:)`) and keeps its transform
+    /// synced to the anchor every frame — and crucially ARKit *refines* the
+    /// anchor as its world understanding improves (drift / re-localization),
+    /// so the marker stays glued to the real-world spot across a long session,
+    /// not just a short one (which a fixed world coordinate would not survive).
+    /// Diffed against the overlay store each render pass (`RNISAROverlay` is
+    /// `Equatable`): add new ids, rebuild changed ones, remove gone ones.
+    ///
+    /// Two maps, guarded by `anchorLock` because `nodeFor` may run on a
+    /// different (render) thread than the `updateAtTime` diff:
+    ///   - `overlayAnchors`: overlay id → (its anchor, the overlay it came from)
+    ///   - `anchorOverlays`: anchor UUID → overlay (so `nodeFor` can build it)
+    private var overlayAnchors: [String: (anchor: ARAnchor, overlay: RNISAROverlay)] = [:]
+    private var anchorOverlays: [UUID: RNISAROverlay] = [:]
+    private let anchorLock = NSLock()
+
     public override init(frame: CGRect) {
         super.init(frame: frame)
         setupView()
@@ -68,6 +88,14 @@ public final class RNSARCameraView: UIView {
         //     a renderer.
         arSCNView.session = RNSARSession.shared.arSession
 
+        // v0.20.0 — become the ARSCNView's render delegate so
+        // `renderer(_:updateAtTime:)` fires once per render pass (display
+        // rate) on the MAIN thread.  This is our per-frame overlay redraw
+        // hook: cheap (a handful of overlays), already on the main thread,
+        // and gives us smooth display-rate tracking without touching the
+        // ARSession delegate (which `RNSARSession` owns for pose logging).
+        arSCNView.delegate = self
+
         // We don't draw any 3D content in Phase 4.4.  Disable
         // SceneKit's automatic statistics overlay and lighting model
         // — we just want the camera feed.
@@ -78,6 +106,39 @@ public final class RNSARCameraView: UIView {
         // this view outside ARSCNView's letterboxed sub-rect).
         backgroundColor = .black
         addSubview(arSCNView)
+
+        // v0.20.0 — overlays render as world-anchored SceneKit nodes inside
+        // `arSCNView.scene` (see `syncOverlayNodes`), so there is no separate
+        // 2D overlay UIView to add here.
+    }
+
+    // MARK: - v0.20.0 — declarative `overlays` prop (KVC from RN)
+
+    /// Declarative `overlays` prop.  RN sets this via KVC
+    /// (`RCT_EXPORT_VIEW_PROPERTY(overlays, NSArray)`) whenever the prop
+    /// changes; we replace the JS overlay namespace wholesale (declarative
+    /// = the full set each render).  Forwarded to the global store rather
+    /// than held per-view because the overlay world is global to the single
+    /// AR session.  `@objc` + `NSArray` so KVC can set it.
+    @objc public var overlays: NSArray = [] {
+        didSet {
+            Self.applyJSSetOverlays(overlays)
+        }
+    }
+
+    /// Parse a JS overlay-dictionary array and replace the JS overlay
+    /// namespace in the shared store.  Shared by the declarative prop
+    /// setter (above) and the imperative `setOverlays` view command (in
+    /// `RNSARCameraViewManager`).
+    static func applyJSSetOverlays(_ overlays: NSArray) {
+        var parsed: [RNISAROverlay] = []
+        parsed.reserveCapacity(overlays.count)
+        for item in overlays {
+            guard let dict = item as? [String: Any],
+                  let o = RNISAROverlay.from(dictionary: dict) else { continue }
+            parsed.append(o)
+        }
+        RNISAROverlayStore.shared.setJSOverlays(parsed)
     }
 
     public override func layoutSubviews() {
@@ -96,7 +157,12 @@ public final class RNSARCameraView: UIView {
         // ARSCNView fills a same-AR sub-rect, there is nothing to crop
         // and the user sees the full captured scene.  The parent view's
         // black background fills the remainder.
-        arSCNView.frame = letterboxedFrame()
+        let lb = letterboxedFrame()
+        arSCNView.frame = lb
+        // Overlay nodes live in `arSCNView.scene` and are rendered by the
+        // same (letterboxed) ARSCNView against the live AR camera, so they
+        // need no separate framing — they align with the camera feed
+        // automatically.
     }
 
     /// Returns the largest `CGRect` inside `bounds` that matches the
@@ -184,6 +250,218 @@ public final class RNSARCameraView: UIView {
             }
         }
     }
-}
 
+    // MARK: - ARSCNViewDelegate (v0.20.0 overlay redraw hook)
+
+    /// Called once per ARSCNView render pass.  Keeps the set of overlay
+    /// ARAnchors in sync with the overlay store (add / rebuild / remove).
+    /// ARKit then tracks each anchor and ARSCNView positions its node — we do
+    /// NOT project or position anything by hand.  Cheap: an `==` diff over a
+    /// handful of overlays.
+    ///
+    /// We deliberately do NOT touch `RNSARSession`'s pose log / plugin /
+    /// onArFrame paths — those ride the ARSession delegate which the singleton
+    /// owns.  This delegate is purely the overlay-sync hook.
+    public func renderer(
+        _ renderer: SCNSceneRenderer,
+        updateAtTime time: TimeInterval
+    ) {
+        syncOverlayAnchors()
+    }
+
+    /// ARSCNViewDelegate: vend the node for one of our overlay anchors.  May
+    /// run on the render thread; ARSCNView keeps the returned node's transform
+    /// synced to the (ARKit-refined) anchor.  We build the visual RELATIVE to
+    /// the anchor origin — the anchor carries the world position.
+    public func renderer(
+        _ renderer: SCNSceneRenderer,
+        nodeFor anchor: ARAnchor
+    ) -> SCNNode? {
+        anchorLock.lock()
+        let overlay = anchorOverlays[anchor.identifier]
+        anchorLock.unlock()
+        guard let overlay = overlay else { return nil }
+
+        if let quad = overlay.worldQuad, quad.count >= 3 {
+            // Anchor sits at the centroid; draw the loop relative to it.
+            var c = simd_float3(0, 0, 0)
+            for v in quad { c += v }
+            c /= Float(quad.count)
+            return Self.makeQuadOutlineNode(
+                relCorners: quad.map { $0 - c },
+                color: overlay.color, label: overlay.label)
+        }
+        return Self.makeBillboardNode(
+            sizeMeters: overlay.sizeMeters, color: overlay.color,
+            label: overlay.label)
+    }
+
+    /// Diff the overlay store against the live anchors: add an ARAnchor for
+    /// each new/changed overlay, remove anchors whose overlay is gone.
+    private func syncOverlayAnchors() {
+        let session = arSCNView.session
+        let overlays = RNISAROverlayStore.shared.snapshot()
+        var liveIDs = Set<String>()
+        liveIDs.reserveCapacity(overlays.count)
+
+        for overlay in overlays {
+            liveIDs.insert(overlay.id)
+            anchorLock.lock()
+            let existing = overlayAnchors[overlay.id]
+            anchorLock.unlock()
+            if let existing = existing, existing.overlay == overlay { continue }
+
+            // New or changed → drop the old anchor, add a fresh one.
+            if let existing = existing {
+                session.remove(anchor: existing.anchor)
+                anchorLock.lock()
+                anchorOverlays[existing.anchor.identifier] = nil
+                overlayAnchors[overlay.id] = nil
+                anchorLock.unlock()
+            }
+            guard let pose = Self.anchorPose(for: overlay) else { continue }
+            let anchor = ARAnchor(transform: pose)
+            anchorLock.lock()
+            anchorOverlays[anchor.identifier] = overlay
+            overlayAnchors[overlay.id] = (anchor, overlay)
+            anchorLock.unlock()
+            session.add(anchor: anchor)
+        }
+
+        // Remove anchors whose overlay id is gone (snapshot under the lock,
+        // call session.remove outside it).
+        anchorLock.lock()
+        let goneIDs = overlayAnchors.keys.filter { !liveIDs.contains($0) }
+        let goneAnchors = goneIDs.compactMap { overlayAnchors[$0]?.anchor }
+        for id in goneIDs {
+            if let a = overlayAnchors[id]?.anchor { anchorOverlays[a.identifier] = nil }
+            overlayAnchors[id] = nil
+        }
+        anchorLock.unlock()
+        for a in goneAnchors { session.remove(anchor: a) }
+    }
+
+    // MARK: - v0.20.0 overlay node builders (RELATIVE to the anchor origin)
+
+    /// World transform for an overlay's anchor: a translation-only pose at the
+    /// `worldPosition`, or at the centroid of `worldQuad`.  nil if no geometry.
+    private static func anchorPose(for overlay: RNISAROverlay) -> simd_float4x4? {
+        let p: simd_float3
+        if let quad = overlay.worldQuad, quad.count >= 3 {
+            var c = simd_float3(0, 0, 0)
+            for v in quad { c += v }
+            p = c / Float(quad.count)
+        } else if let center = overlay.worldPosition {
+            p = center
+        } else {
+            return nil
+        }
+        var m = matrix_identity_float4x4
+        m.columns.3 = simd_float4(p.x, p.y, p.z, 1)
+        return m
+    }
+
+    /// A camera-facing billboard plane (at the anchor origin), sized in metres,
+    /// textured with a stroked outline + optional centred label.  Always drawn
+    /// on top (depth read/write off) so an annotation is never hidden.
+    private static func makeBillboardNode(
+        sizeMeters: CGSize?,
+        color: UIColor,
+        label: String?
+    ) -> SCNNode {
+        let w = sizeMeters?.width ?? RNISAROverlay.defaultMarkerExtent
+        let h = sizeMeters?.height ?? RNISAROverlay.defaultMarkerExtent
+        let plane = SCNPlane(width: w, height: h)
+        let mat = SCNMaterial()
+        mat.diffuse.contents = overlayImage(color: color, label: label)
+        mat.isDoubleSided = true
+        mat.lightingModel = .constant      // unlit — show the texture as-is
+        mat.writesToDepthBuffer = false
+        mat.readsFromDepthBuffer = false   // never occluded by the scene
+        plane.firstMaterial = mat
+
+        let node = SCNNode(geometry: plane)
+        node.renderingOrder = 1000         // draw after the camera background
+        let billboard = SCNBillboardConstraint()
+        billboard.freeAxes = .all          // always face the camera, flat
+        node.constraints = [billboard]
+        return node
+    }
+
+    /// A 3D line-loop through corners expressed RELATIVE to the anchor origin
+    /// (the anchor is at the quad centroid), with an optional camera-facing
+    /// label at the centre.
+    private static func makeQuadOutlineNode(
+        relCorners: [simd_float3],
+        color: UIColor,
+        label: String?
+    ) -> SCNNode {
+        let vertices = relCorners.map { SCNVector3($0.x, $0.y, $0.z) }
+        var indices: [Int32] = []
+        let n = vertices.count
+        indices.reserveCapacity(n * 2)
+        for i in 0..<n {
+            indices.append(Int32(i))
+            indices.append(Int32((i + 1) % n))   // close the loop
+        }
+        let src = SCNGeometrySource(vertices: vertices)
+        let elem = SCNGeometryElement(indices: indices, primitiveType: .line)
+        let geo = SCNGeometry(sources: [src], elements: [elem])
+        let mat = SCNMaterial()
+        mat.diffuse.contents = color
+        mat.lightingModel = .constant
+        mat.writesToDepthBuffer = false
+        mat.readsFromDepthBuffer = false
+        geo.firstMaterial = mat
+
+        let node = SCNNode(geometry: geo)
+        node.renderingOrder = 1000
+
+        if let label = label, !label.isEmpty {
+            // Label at the centroid (≈ local origin in relative space).
+            let labelNode = makeBillboardNode(
+                sizeMeters: CGSize(width: 0.12, height: 0.12),
+                color: color, label: label)
+            node.addChildNode(labelNode)
+        }
+        return node
+    }
+
+    /// Render a stroked rounded-rect outline + optional centred label chip to
+    /// a square image, used as the billboard plane's texture.  Transparent
+    /// background so only the outline + chip show over the camera feed.
+    private static func overlayImage(color: UIColor, label: String?) -> UIImage {
+        let px: CGFloat = 512
+        let renderer = UIGraphicsImageRenderer(size: CGSize(width: px, height: px))
+        return renderer.image { ctx in
+            let cg = ctx.cgContext
+            let inset = px * 0.07
+            let rect = CGRect(x: inset, y: inset,
+                              width: px - 2 * inset, height: px - 2 * inset)
+            let path = UIBezierPath(roundedRect: rect, cornerRadius: px * 0.05)
+            cg.setStrokeColor(color.cgColor)
+            cg.setLineWidth(px * 0.03)
+            path.stroke()
+
+            guard let label = label, !label.isEmpty else { return }
+            let fontSize = px * 0.13
+            let font = UIFont.systemFont(ofSize: fontSize, weight: .bold)
+            let attrs: [NSAttributedString.Key: Any] = [
+                .font: font, .foregroundColor: UIColor.white,
+            ]
+            let ts = (label as NSString).size(withAttributes: attrs)
+            let pad = fontSize * 0.35
+            let chip = CGRect(x: (px - ts.width) / 2 - pad,
+                              y: (px - ts.height) / 2 - pad,
+                              width: ts.width + 2 * pad,
+                              height: ts.height + 2 * pad)
+            color.withAlphaComponent(0.9).setFill()
+            UIBezierPath(roundedRect: chip, cornerRadius: pad).fill()
+            (label as NSString).draw(
+                at: CGPoint(x: (px - ts.width) / 2, y: (px - ts.height) / 2),
+                withAttributes: attrs)
+        }
+    }
+
+}
 

package/ios/Sources/RNImageStitcher/RNSARSession.swift

@@ -492,6 +492,24 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
     private var lastArFrameMetaEmit: TimeInterval = 0
     private let arFrameMetaLock = NSLock()
 
+    // ──────────────────────────────────────────────────────────────
+    // v0.19.0 — native AR plugin framework (RNISARPluginRegistry)
+    // ──────────────────────────────────────────────────────────────
+    //
+    // While the plugin registry is NON-EMPTY, the per-frame path builds a
+    // `RNISARFrameContext` once and calls each registered plugin's
+    // `process(_:)` on the AR (delegate) thread (see `invokeArPlugins`).
+    // Non-nil SYNC results are cached here so the throttled `onArFrame`
+    // meta build can fold them in under `plugins: { [name]: result }`
+    // without re-running plugins.  Zero-plugin apps skip the whole path
+    // (the registry's `isEmpty` gate), so they pay nothing.
+    //
+    // Written on the AR thread (per-frame) and read on the same thread
+    // (the meta build runs inline in `session(_:didUpdate:)`), but guarded
+    // anyway for defensiveness against any future off-thread reader.
+    private var latestPluginSyncResults: [String: Any] = [:]
+    private let pluginSyncResultsLock = NSLock()
+
     private override init() {
         super.init()
         arSession.delegate = self
@@ -594,6 +612,12 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         detectedPlaneTransformInternal = nil
         bestRejectedAlignment = -1.0
         planeLatchLock.unlock()
+        // v0.19.0 — drop any cached SYNC plugin results so the next
+        // capture's `onArFrame` meta doesn't surface stale plugin output
+        // before the first frame of the new session runs the plugins.
+        pluginSyncResultsLock.lock()
+        latestPluginSyncResults = [:]
+        pluginSyncResultsLock.unlock()
     }
 
     /// Build the `ARWorldTrackingConfiguration` shared by `start()` and
@@ -846,6 +870,16 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         // double-consuming would ingest each frame twice.
         RNSARWorkletRuntime.shared().dispatchFrame(frame, pose: pose)
 
+        // v0.19.0 — native AR plugin framework.  When the registry is
+        // non-empty, build the per-frame `RNISARFrameContext` once and run
+        // every registered plugin's `process(_:)` SYNCHRONOUSLY on this AR
+        // thread (so the live pixel/depth buffers are valid for the call).
+        // Caches non-nil SYNC results for the meta build below.  Cheap
+        // no-op when no plugins are registered (the common case).  Runs
+        // BEFORE `maybeEmitArFrameMeta` so the throttled `onArFrame` meta
+        // can fold in this frame's freshest plugin results.
+        invokeArPlugins(frame, pose: pose)
+
         // v0.18.0 — `onArFrame` LIGHT-metadata channel.  Gated +
         // throttled; builds the ARFrameMeta dictionary and posts it for
         // the bridge to re-emit.  Cheap no-op when disabled (the common
@@ -1433,13 +1467,105 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
         // Obj-C++ extraction helpers + the shared C++ extraction-config
         // gating (depth/anchors/mesh ⇐ enableDepth/enableAnchors/enableMesh).
         let meta = CameraFrameHostObject.lightArFrameMeta(from: frame, pose: pose)
+
+        // v0.19.0 — fold in any SYNC plugin results captured by
+        // `invokeArPlugins` for the freshest frames.  Only attach the
+        // `plugins` key when there's at least one result, so the common
+        // (no-plugin) meta shape is unchanged.  Snapshot under the lock,
+        // then bridge into a fresh dictionary copy.
+        pluginSyncResultsLock.lock()
+        let pluginResults = latestPluginSyncResults
+        pluginSyncResultsLock.unlock()
+        let userInfo: [AnyHashable: Any]
+        if pluginResults.isEmpty {
+            userInfo = meta
+        } else {
+            var withPlugins = meta
+            withPlugins["plugins"] = pluginResults
+            userInfo = withPlugins
+        }
+
         NotificationCenter.default.post(
             name: .retailensARFrameMeta,
             object: nil,
-            userInfo: meta
+            userInfo: userInfo
         )
     }
 
+    /// v0.19.0 — run all registered native AR plugins for this frame.
+    /// Gated on the registry being NON-EMPTY (the cheap `isEmpty` check) so
+    /// zero-plugin apps skip the context build entirely.  When plugins are
+    /// present, builds ONE `RNISARFrameContext` (zero-copy view of the
+    /// frame's live buffers + the already-built anchor dicts) and calls
+    /// each plugin's `process(_:)` SYNCHRONOUSLY on this AR (delegate)
+    /// thread — so the live `pixelBuffer` / `depthBuffer` are valid for the
+    /// call (the plugin must copy before offloading; see the protocol
+    /// docstring).  Non-nil SYNC results are cached in
+    /// `latestPluginSyncResults` for the throttled `onArFrame` meta to fold
+    /// in; ASYNC results arrive later via `RNISARPluginRegistry.emit`.
+    private func invokeArPlugins(_ frame: ARFrame, pose: RNSARFramePose) {
+        let registry = RNISARPluginRegistry.shared
+        guard !registry.isEmpty else { return }
+        let plugins = registry.plugins()
+        guard !plugins.isEmpty else { return }
+
+        // depthBuffer: expose the live sceneDepth map ONLY when the
+        // `<Camera enableDepth>` prop is on (gating read in Obj-C++ so the
+        // C++ extraction-config header stays out of Swift).  Prefer
+        // `sceneDepth`, fall back to `smoothedSceneDepth` — same precedence
+        // as the full extraction path.
+        var depthBuffer: CVPixelBuffer? = nil
+        if CameraFrameHostObject.arExtractionDepthEnabled() {
+            if let dd = frame.sceneDepth ?? frame.smoothedSceneDepth {
+                depthBuffer = dd.depthMap
+            }
+        }
+
+        // anchors: reuse the EXACT light dicts the `onArFrame` meta builds
+        // (gated on `enableAnchors`; empty otherwise) — DRY single source.
+        let anchorDicts = CameraFrameHostObject.arAnchorDicts(from: frame)
+        let anchors = anchorDicts as? [[String: Any]] ?? []
+
+        let context = RNISARFrameContext(
+            pixelBuffer: frame.capturedImage,
+            timestampNs: frame.timestamp * 1e9,
+            fx: pose.fx, fy: pose.fy, cx: pose.cx, cy: pose.cy,
+            imageWidth: pose.imageWidth, imageHeight: pose.imageHeight,
+            poseRotation: [pose.qx, pose.qy, pose.qz, pose.qw],
+            poseTranslation: [pose.tx, pose.ty, pose.tz],
+            trackingState: Self.trackingStateString(pose.trackingState),
+            depthBuffer: depthBuffer,
+            anchors: anchors
+        )
+
+        var syncResults: [String: Any] = [:]
+        for plugin in plugins {
+            // Defensive: a plugin throwing/crashing in `process` would take
+            // down the AR thread, but Swift has no try/catch for non-Error
+            // crashes — the contract is that plugins are well-behaved.  We
+            // simply collect non-nil results keyed by the plugin's name.
+            if let result = plugin.process(context) {
+                syncResults[plugin.name()] = result
+            }
+        }
+
+        pluginSyncResultsLock.lock()
+        latestPluginSyncResults = syncResults
+        pluginSyncResultsLock.unlock()
+    }
+
+    /// Map the SDK's `RNSARTrackingState` to the same string the
+    /// `onArFrame` meta + `CameraFrame.trackingState` use, so plugins see a
+    /// consistent vocabulary.
+    private static func trackingStateString(_ s: RNSARTrackingState) -> String {
+        switch s {
+        case .tracking:     return "normal"
+        case .limited:      return "limited"
+        case .initialising: return "limited"
+        case .notAvailable: return "notAvailable"
+        }
+    }
+
     private func makePose(from frame: ARFrame) -> RNSARFramePose {
         // ARKit's transform is a 4x4 matrix; extract translation
         // (last column) and rotation (top-left 3x3 → quaternion).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-image-stitcher",
-  "version": "0.18.0",
+  "version": "0.20.0",
   "description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",