react-native-image-stitcher 0.16.2 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/RNImageStitcher.podspec +26 -1
- package/android/build.gradle +20 -0
- package/android/src/main/cpp/CMakeLists.txt +46 -3
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +227 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +6 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +55 -6
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +103 -0
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +256 -0
- package/cpp/stitcher_frame_jsi.cpp +214 -0
- package/cpp/stitcher_frame_jsi.hpp +108 -0
- package/cpp/stitcher_proxy_jsi.cpp +109 -0
- package/cpp/stitcher_proxy_jsi.hpp +46 -0
- package/cpp/stitcher_worklet_dispatch.cpp +103 -0
- package/cpp/stitcher_worklet_dispatch.hpp +71 -0
- package/cpp/stitcher_worklet_registry.cpp +91 -0
- package/cpp/stitcher_worklet_registry.hpp +146 -0
- package/dist/camera/ARCameraView.d.ts +20 -0
- package/dist/camera/ARCameraView.js +23 -1
- package/dist/camera/Camera.d.ts +12 -0
- package/dist/camera/Camera.js +2 -2
- package/dist/camera/CaptureMemoryPill.d.ts +4 -3
- package/dist/camera/CaptureMemoryPill.js +4 -3
- package/dist/stitching/ensureStitcherProxyInstalled.d.ts +8 -0
- package/dist/stitching/ensureStitcherProxyInstalled.js +81 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +44 -6
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +128 -0
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +313 -0
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +60 -0
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +214 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +42 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +160 -0
- package/package.json +1 -1
- package/src/camera/ARCameraView.tsx +51 -2
- package/src/camera/Camera.tsx +15 -0
- package/src/camera/CaptureMemoryPill.tsx +4 -3
- package/src/stitching/ensureStitcherProxyInstalled.ts +141 -0
package/dist/camera/Camera.js
CHANGED
|
@@ -310,7 +310,7 @@ function extractPanoramaOverrides(props) {
|
|
|
310
310
|
* The public `<Camera>` component.
|
|
311
311
|
*/
|
|
312
312
|
function Camera(props) {
|
|
313
|
-
const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, engine = 'batch-keyframe',
|
|
313
|
+
const { defaultCaptureSource = 'non-ar', defaultLens = '1x', captureSources = 'both', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, onCaptureAbandoned, flash: controlledFlash, onFlashChange, showFlashButton = true, headerTitle, onHeaderBack, headerBackLabel, headerGuidance, headerColors, thumbnails, thumbnailsMin, thumbnailsMax, onThumbnailPress, capturePreview, capturePreviewActions, onCapturePreviewClose, frameProcessor: hostFrameProcessor, arFrameProcessor, engine = 'batch-keyframe',
|
|
314
314
|
// ── Panorama GUIDANCE (feature/pano-ux-guidance) ──────────────
|
|
315
315
|
panMode = 'vertical', panGuidance = true, maxPanDurationMs = 0, panTooFastThreshold, lateralBudgetCm = 4, rectCrop = false, showPreview = false, guidanceCopy, } = props;
|
|
316
316
|
// Derived guidance state. The landscape-only gate decision itself is
|
|
@@ -1425,7 +1425,7 @@ function Camera(props) {
|
|
|
1425
1425
|
// (V12.14.8 OOM fix). The CaptureStatusOverlay renders the
|
|
1426
1426
|
// "Stitching…" state on top, so no placeholder label is needed
|
|
1427
1427
|
// in that case — only for the camera-switch transition.
|
|
1428
|
-
react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
|
|
1428
|
+
react_1.default.createElement(react_native_1.View, { style: [react_native_1.StyleSheet.absoluteFill, styles.transitionPlaceholder] }, statusPhase === 'stitching' ? null : (react_1.default.createElement(react_native_1.Text, { style: styles.transitionLabel }, "Switching camera\u2026")))) : isAR ? (react_1.default.createElement(ARCameraView_1.ARCameraView, { ref: arViewRef, style: react_native_1.StyleSheet.absoluteFill, arFrameProcessor: arFrameProcessor })) : (react_1.default.createElement(CameraView_1.CameraView, { ref: visionCameraRef, device: capture.device, isActive: true,
|
|
1429
1429
|
// `video={true}` is REQUIRED for takeSnapshot to work on iOS.
|
|
1430
1430
|
// vision-camera v4's iOS implementation of takeSnapshot waits
|
|
1431
1431
|
// for a frame on the video pipeline; with video disabled, the
|
|
@@ -16,9 +16,10 @@
|
|
|
16
16
|
* (false comfort exactly where OOM happens). Falls back to 1500/2200 if the
|
|
17
17
|
* RAM read is unavailable.
|
|
18
18
|
*
|
|
19
|
-
* Backed by the `getMemoryFootprintMB()` native module (iOS:
|
|
20
|
-
* `phys_footprint`; Android: `/proc/self/statm` RSS
|
|
21
|
-
*
|
|
19
|
+
* Backed by the `getMemoryFootprintMB()` native module (iOS:
|
|
20
|
+
* `task_info(TASK_VM_INFO)` `phys_footprint`; Android: `/proc/self/statm` RSS
|
|
21
|
+
* — resident pages, unthrottled — the SAME number the C++ `[memstat]` logs
|
|
22
|
+
* report). Returns -1 if the native call fails.
|
|
22
23
|
*
|
|
23
24
|
* Mount this pill inside a `settings.debug`-gated branch — it
|
|
24
25
|
* polls native every 500 ms and is unwanted in production builds.
|
|
@@ -18,9 +18,10 @@
|
|
|
18
18
|
* (false comfort exactly where OOM happens). Falls back to 1500/2200 if the
|
|
19
19
|
* RAM read is unavailable.
|
|
20
20
|
*
|
|
21
|
-
* Backed by the `getMemoryFootprintMB()` native module (iOS:
|
|
22
|
-
* `phys_footprint`; Android: `/proc/self/statm` RSS
|
|
23
|
-
*
|
|
21
|
+
* Backed by the `getMemoryFootprintMB()` native module (iOS:
|
|
22
|
+
* `task_info(TASK_VM_INFO)` `phys_footprint`; Android: `/proc/self/statm` RSS
|
|
23
|
+
* — resident pages, unthrottled — the SAME number the C++ `[memstat]` logs
|
|
24
|
+
* report). Returns -1 if the native call fails.
|
|
24
25
|
*
|
|
25
26
|
* Mount this pill inside a `settings.debug`-gated branch — it
|
|
26
27
|
* polls native every 500 ms and is unwanted in production builds.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare function ensureStitcherProxyInstalled(): boolean;
|
|
2
|
+
/**
|
|
3
|
+
* Test-only — reset module-internal state. Used by jest to allow
|
|
4
|
+
* multiple test cases to re-trigger the install path independently.
|
|
5
|
+
* NOT exported from `src/index.ts`.
|
|
6
|
+
*/
|
|
7
|
+
export declare function _resetStitcherProxyInstallStateForTests(): void;
|
|
8
|
+
//# sourceMappingURL=ensureStitcherProxyInstalled.d.ts.map
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.ensureStitcherProxyInstalled = ensureStitcherProxyInstalled;
|
|
5
|
+
exports._resetStitcherProxyInstallStateForTests = _resetStitcherProxyInstallStateForTests;
|
|
6
|
+
const react_native_1 = require("react-native");
|
|
7
|
+
/**
|
|
8
|
+
* `__DEV__` is RN's global dev-flag. Guard the read with `typeof`
|
|
9
|
+
* so the helper works in any environment that imports it without
|
|
10
|
+
* defining __DEV__ (jest, SSR, custom tooling). Same pattern RN's
|
|
11
|
+
* own debug code uses.
|
|
12
|
+
*/
|
|
13
|
+
function isDev() {
|
|
14
|
+
return typeof __DEV__ !== 'undefined' && __DEV__;
|
|
15
|
+
}
|
|
16
|
+
let installed = false;
|
|
17
|
+
function ensureStitcherProxyInstalled() {
|
|
18
|
+
if (installed)
|
|
19
|
+
return true;
|
|
20
|
+
// Already installed by an earlier hook mount. Cheap fast-path.
|
|
21
|
+
if (typeof globalThis.__stitcherProxy !== 'undefined') {
|
|
22
|
+
installed = true;
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
25
|
+
const mod = react_native_1.NativeModules
|
|
26
|
+
.StitcherJsiInstaller;
|
|
27
|
+
if (mod == null || typeof mod.install !== 'function') {
|
|
28
|
+
// Module not present — Android until Phase 4b.ii lands, or
|
|
29
|
+
// an old iOS build. Surface this once at debug-info level so
|
|
30
|
+
// the host can see "your worklets are JS-registered only" in
|
|
31
|
+
// logcat / Console.app without a noisy per-frame warning.
|
|
32
|
+
if (isDev() && !warnedAboutMissingModule) {
|
|
33
|
+
warnedAboutMissingModule = true;
|
|
34
|
+
console.info('[react-native-image-stitcher] StitcherJsiInstaller native ' +
|
|
35
|
+
'module not found; host worklets registered in JS-side ' +
|
|
36
|
+
'registry only. AR-mode dispatch requires the native install ' +
|
|
37
|
+
'(iOS Phase 4b.i — included in v0.8.0; Android Phase 4b.ii ' +
|
|
38
|
+
'— follow-up release).');
|
|
39
|
+
}
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
const ok = mod.install();
|
|
44
|
+
if (!ok) {
|
|
45
|
+
// Native module ran but couldn't install (JSI runtime
|
|
46
|
+
// unreachable). Same fallback as the missing-module case.
|
|
47
|
+
if (isDev() && !warnedAboutFailedInstall) {
|
|
48
|
+
warnedAboutFailedInstall = true;
|
|
49
|
+
console.info('[react-native-image-stitcher] StitcherJsiInstaller.install() ' +
|
|
50
|
+
'returned false (JSI runtime unreachable — remote debug ' +
|
|
51
|
+
'mode?). Falling back to JS-side host worklet registry.');
|
|
52
|
+
}
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
installed = true;
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
catch (err) {
|
|
59
|
+
if (isDev() && !warnedAboutFailedInstall) {
|
|
60
|
+
warnedAboutFailedInstall = true;
|
|
61
|
+
console.info('[react-native-image-stitcher] StitcherJsiInstaller.install() ' +
|
|
62
|
+
'threw: ' +
|
|
63
|
+
String(err) +
|
|
64
|
+
'. Falling back to JS-side host worklet registry.');
|
|
65
|
+
}
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
let warnedAboutMissingModule = false;
|
|
70
|
+
let warnedAboutFailedInstall = false;
|
|
71
|
+
/**
|
|
72
|
+
* Test-only — reset module-internal state. Used by jest to allow
|
|
73
|
+
* multiple test cases to re-trigger the install path independently.
|
|
74
|
+
* NOT exported from `src/index.ts`.
|
|
75
|
+
*/
|
|
76
|
+
function _resetStitcherProxyInstallStateForTests() {
|
|
77
|
+
installed = false;
|
|
78
|
+
warnedAboutMissingModule = false;
|
|
79
|
+
warnedAboutFailedInstall = false;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=ensureStitcherProxyInstalled.js.map
|
|
@@ -501,10 +501,34 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
|
|
|
501
501
|
isRunning = true
|
|
502
502
|
currentTrackingState = .initialising
|
|
503
503
|
|
|
504
|
-
//
|
|
505
|
-
//
|
|
506
|
-
//
|
|
507
|
-
//
|
|
504
|
+
// v0.8.0 Phase 3c/4b — wire the AR worklet runtime. The
|
|
505
|
+
// per-frame ingest is routed through `RNSARWorkletRuntime`
|
|
506
|
+
// (see `session(_:didUpdate:)` below) instead of calling the
|
|
507
|
+
// incremental consumer directly. Two steps here:
|
|
508
|
+
//
|
|
509
|
+
// 1. `installIfNeeded()` lazily constructs the worklet
|
|
510
|
+
// runtime's `JsiWorkletContext` + its serial dispatch
|
|
511
|
+
// queue (idempotent; safe across redundant start() calls
|
|
512
|
+
// and multiple <Camera> mounts).
|
|
513
|
+
// 2. `setFirstPartyCallback:` installs the EXISTING first-
|
|
514
|
+
// party stitching behaviour as a closure. The runtime
|
|
515
|
+
// invokes this synchronously on the delegate (caller)
|
|
516
|
+
// thread per frame — byte-identical to the old direct
|
|
517
|
+
// `consumeFrame(...)` call — and then fans the frame out
|
|
518
|
+
// to any host-registered worklets asynchronously on its
|
|
519
|
+
// own queue.
|
|
520
|
+
//
|
|
521
|
+
// The callback captures `self` weakly so the runtime singleton
|
|
522
|
+
// (process-lifetime) never keeps this session alive. It reads
|
|
523
|
+
// `incrementalConsumer` (itself weak) at call time, so a torn-
|
|
524
|
+
// down consumer simply no-ops — same semantics as the prior
|
|
525
|
+
// `incrementalConsumer?.consumeFrame(...)` optional-chain.
|
|
526
|
+
let workletRuntime = RNSARWorkletRuntime.shared()
|
|
527
|
+
workletRuntime.installIfNeeded()
|
|
528
|
+
workletRuntime.setFirstPartyCallback { [weak self] arFrame, pose in
|
|
529
|
+
self?.incrementalConsumer?.consumeFrame(
|
|
530
|
+
pixelBuffer: arFrame.capturedImage, pose: pose)
|
|
531
|
+
}
|
|
508
532
|
}
|
|
509
533
|
|
|
510
534
|
@objc public func stop() {
|
|
@@ -513,6 +537,12 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
|
|
|
513
537
|
isRunning = false
|
|
514
538
|
currentTrackingState = .notAvailable
|
|
515
539
|
clearPoseLog()
|
|
540
|
+
// v0.8.0 Phase 3c — clear the worklet runtime's first-party
|
|
541
|
+
// callback so the (process-lifetime) runtime singleton doesn't
|
|
542
|
+
// hold the closure (and transitively the consumer reference)
|
|
543
|
+
// between captures. `start()` reinstalls it on the next run.
|
|
544
|
+
// Idempotent; safe even if start() never ran the install path.
|
|
545
|
+
RNSARWorkletRuntime.shared().setFirstPartyCallback(nil)
|
|
516
546
|
// V15.0b — clear latched plane so the next capture detects
|
|
517
547
|
// afresh. Plane geometry is per-capture: a different
|
|
518
548
|
// fixture in a different orientation needs a new lock.
|
|
@@ -596,8 +626,16 @@ public final class RNSARSession: NSObject, ARSessionDelegate {
|
|
|
596
626
|
// cv::Mat sync conversion synchronously on the delegate thread
|
|
597
627
|
// before returning, so the captured pixel buffer is safe for
|
|
598
628
|
// ARKit to recycle after this call.
|
|
599
|
-
|
|
600
|
-
|
|
629
|
+
//
|
|
630
|
+
// `dispatchFrame(_:pose:)` runs the first-party callback
|
|
631
|
+
// (installed in `start()`, which wraps the same
|
|
632
|
+
// `incrementalConsumer.consumeFrame(...)` path) SYNCHRONOUSLY on
|
|
633
|
+
// this delegate thread — preserving the pool-reuse contract —
|
|
634
|
+
// and THEN fans the frame out to any host-registered worklets
|
|
635
|
+
// ASYNCHRONOUSLY on the runtime's own queue. Do NOT also call
|
|
636
|
+
// `consumeFrame` here: dispatchFrame already drives it, and
|
|
637
|
+
// double-consuming would ingest each frame twice.
|
|
638
|
+
RNSARWorkletRuntime.shared().dispatchFrame(frame, pose: pose)
|
|
601
639
|
|
|
602
640
|
// If recording is in flight, append this frame to the
|
|
603
641
|
// asset writer DIRECTLY — no queue hop.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// RNSARWorkletRuntime.h — Obj-C facade for the v0.8.0 AR-mode
|
|
4
|
+
// worklet runtime. Wraps `react-native-worklets-core`'s
|
|
5
|
+
// `RNWorklet::JsiWorkletContext` (the same primitive vision-camera
|
|
6
|
+
// uses for its Frame Processor runtime) so the lib can dispatch
|
|
7
|
+
// per-ARFrame worklets on a thread we own — rather than ARKit's
|
|
8
|
+
// delegate queue, where doing significant work would block the
|
|
9
|
+
// AR session's update loop.
|
|
10
|
+
//
|
|
11
|
+
// ## Phase 3b scope (this commit)
|
|
12
|
+
//
|
|
13
|
+
// Owns:
|
|
14
|
+
// - The dispatch queue the worklet runtime pins to.
|
|
15
|
+
// - The underlying `JsiWorkletContext` (constructed lazily on
|
|
16
|
+
// `installIfNeeded`, lives for the singleton's lifetime).
|
|
17
|
+
//
|
|
18
|
+
// Exposes:
|
|
19
|
+
// - `+ shared` singleton accessor.
|
|
20
|
+
// - `- installIfNeeded` (idempotent runtime construction).
|
|
21
|
+
// - `- isInstalled` for diagnostics + tests.
|
|
22
|
+
// - `- dispatchFrame:pose:` — currently a no-op stub; Phase 3c
|
|
23
|
+
// fills in the actual host-object construction + worklet
|
|
24
|
+
// invocation + first-party stitching dispatch.
|
|
25
|
+
//
|
|
26
|
+
// Host-worklet registry is intentionally NOT in Phase 3b — Phase 4
|
|
27
|
+
// lands the JSI plugin + TS-side hook that defines the storage
|
|
28
|
+
// shape (NSMutableArray of boxed shared_ptrs vs a C++ vector ivar
|
|
29
|
+
// vs something else). Pre-committing the storage type here would
|
|
30
|
+
// risk rework. See
|
|
31
|
+
// `docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md`
|
|
32
|
+
// Phase 4 section for the planned API.
|
|
33
|
+
//
|
|
34
|
+
// ## Why Obj-C facade with `.mm` implementation
|
|
35
|
+
//
|
|
36
|
+
// The implementation needs to hold `std::shared_ptr<JsiWorkletContext>`
|
|
37
|
+
// + run JSI value construction, which can't live in pure Swift. Same
|
|
38
|
+
// pattern as `KeyframeGateBridge.{h,mm}` + `StitcherFrameHostObject.{h,mm}`:
|
|
39
|
+
// keep the header umbrella-safe (no JSI imports), put the C++ glue in
|
|
40
|
+
// the .mm.
|
|
41
|
+
//
|
|
42
|
+
// ## Header umbrella safety
|
|
43
|
+
//
|
|
44
|
+
// This .h imports only Foundation + ARKit (both system frameworks).
|
|
45
|
+
// Worklets-core types are confined to the .mm.
|
|
46
|
+
|
|
47
|
+
#pragma once
|
|
48
|
+
|
|
49
|
+
#import <Foundation/Foundation.h>
|
|
50
|
+
#import <ARKit/ARKit.h>
|
|
51
|
+
|
|
52
|
+
@class RNSARFramePose;
|
|
53
|
+
|
|
54
|
+
NS_ASSUME_NONNULL_BEGIN
|
|
55
|
+
|
|
56
|
+
NS_SWIFT_NAME(RNSARWorkletRuntime)
|
|
57
|
+
@interface RNSARWorkletRuntime : NSObject
|
|
58
|
+
|
|
59
|
+
/// Singleton accessor. One AR worklet runtime per process; multiple
|
|
60
|
+
/// `<Camera>` mounts share it. Construction is cheap (just an Obj-C
|
|
61
|
+
/// alloc + an `NSMutableArray`); the heavy JSI work happens in
|
|
62
|
+
/// `-installIfNeeded`.
|
|
63
|
+
+ (instancetype)shared;
|
|
64
|
+
|
|
65
|
+
/// Construct the underlying `JsiWorkletContext` if not yet
|
|
66
|
+
/// installed. Idempotent — repeated calls are no-ops. Called from
|
|
67
|
+
/// `RNSARSession` at AR-mode start time (Phase 3c will wire this
|
|
68
|
+
/// up; Phase 3b ships the method but no one calls it yet).
|
|
69
|
+
///
|
|
70
|
+
/// Threading: safe to call from any thread; internally serialised.
|
|
71
|
+
/// The runtime's own dispatch queue starts running once installed.
|
|
72
|
+
- (void)installIfNeeded;
|
|
73
|
+
|
|
74
|
+
/// Diagnostics + tests. Returns `YES` after a successful
|
|
75
|
+
/// `-installIfNeeded`.
|
|
76
|
+
- (BOOL)isInstalled;
|
|
77
|
+
|
|
78
|
+
/// Phase 3c — type of the first-party stitching callback. Invoked
|
|
79
|
+
/// synchronously on the caller thread (`ARSession.delegateQueue` —
|
|
80
|
+
/// typically main queue today) per AR frame. Block must consume
|
|
81
|
+
/// the pixel buffer before returning (ARKit pool reuse contract).
|
|
82
|
+
typedef void (^RNSARFirstPartyCallback)(ARFrame *arFrame,
|
|
83
|
+
RNSARFramePose *pose);
|
|
84
|
+
|
|
85
|
+
/// Phase 3c — install the closure that takes ownership of the
|
|
86
|
+
/// per-frame first-party stitching dispatch. Called from
|
|
87
|
+
/// `RNSARSession.start()` after the incremental consumer is set;
|
|
88
|
+
/// the block then routes `dispatchFrame:pose:` calls through to
|
|
89
|
+
/// the existing `incrementalConsumer.consumeFrame(...)` path.
|
|
90
|
+
///
|
|
91
|
+
/// Pre-Phase-3c the delegate called the consumer directly. After
|
|
92
|
+
/// Phase 3c the delegate calls `dispatchFrame:pose:` (this class)
|
|
93
|
+
/// which invokes the callback. Net behavior is byte-identical;
|
|
94
|
+
/// the indirection sets up the seam where Phase 4 will fan out to
|
|
95
|
+
/// host worklets without changing the first-party path.
|
|
96
|
+
///
|
|
97
|
+
/// Pass `nil` to clear (e.g. on `RNSARSession.stop()`). Idempotent.
|
|
98
|
+
- (void)setFirstPartyCallback:(nullable RNSARFirstPartyCallback)callback;
|
|
99
|
+
|
|
100
|
+
/// Dispatch one AR frame through the registered worklets. Called
|
|
101
|
+
/// per `ARFrame` by `RNSARSession.delegate` once Phase 3c lands the
|
|
102
|
+
/// migration (Phase 3b ships this method as a no-op stub so the
|
|
103
|
+
/// runtime can be built + linked + the API surface fixed).
|
|
104
|
+
///
|
|
105
|
+
/// The Phase 3c implementation will:
|
|
106
|
+
/// 1. Build a `StitcherFrameHostObject` from `arFrame` + `pose`.
|
|
107
|
+
/// 2. Run the first-party stitching synchronously on the caller
|
|
108
|
+
/// thread (preserves today's `ingestFromARCameraView` cost
|
|
109
|
+
/// envelope at the producer site).
|
|
110
|
+
/// 3. If any host worklets are registered, dispatch the host
|
|
111
|
+
/// object onto the worklet runtime's thread + invoke each
|
|
112
|
+
/// worklet via `RNWorklet::WorkletInvoker::call`.
|
|
113
|
+
/// 4. Invalidate the host object after all worklets return.
|
|
114
|
+
///
|
|
115
|
+
/// Phase 3c gate: install/idempotence tests + this method's
|
|
116
|
+
/// integration test required before merge. See
|
|
117
|
+
/// `docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md`
|
|
118
|
+
/// Phase 3c gate criteria.
|
|
119
|
+
///
|
|
120
|
+
/// Threading: typically called from `ARSession.delegateQueue` (main
|
|
121
|
+
/// queue by default; Phase 3c will pin it explicitly to a
|
|
122
|
+
/// dedicated queue).
|
|
123
|
+
- (void)dispatchFrame:(ARFrame *)arFrame pose:(RNSARFramePose *)pose
|
|
124
|
+
NS_SWIFT_NAME(dispatchFrame(_:pose:));
|
|
125
|
+
|
|
126
|
+
@end
|
|
127
|
+
|
|
128
|
+
NS_ASSUME_NONNULL_END
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// RNSARWorkletRuntime.mm — Obj-C++ implementation. See the header
|
|
4
|
+
// for the API contract. This file owns:
|
|
5
|
+
//
|
|
6
|
+
// - The dispatch queue the worklet runtime is pinned to
|
|
7
|
+
// - The `std::shared_ptr<RNWorklet::JsiWorkletContext>` itself
|
|
8
|
+
// - The registry of host worklets (Phase 4 wiring will populate
|
|
9
|
+
// this via a JSI plugin entry point)
|
|
10
|
+
//
|
|
11
|
+
// Phase 3b scope: construct the context + expose the API. No
|
|
12
|
+
// dispatch logic yet — `dispatchFrame:pose:` is a stub. Phase 3c
|
|
13
|
+
// fills in (a) the host-object construction + worklet invocation,
|
|
14
|
+
// (b) the first-party stitching callback, (c) the migration in
|
|
15
|
+
// `RNSARSession.delegate`.
|
|
16
|
+
//
|
|
17
|
+
// ## Singleton lifetime note (for Leaks-tool readers)
|
|
18
|
+
//
|
|
19
|
+
// `+ shared` uses `dispatch_once`, so the singleton lives for the
|
|
20
|
+
// process lifetime — same pattern as most Obj-C singletons. This
|
|
21
|
+
// means the dispatch queue (created in `init`) + the JsiWorkletContext
|
|
22
|
+
// (constructed lazily in `installIfNeeded`) + the `workletCallInvoker`
|
|
23
|
+
// lambda that captures the queue are ALL retained until process
|
|
24
|
+
// termination. Xcode Instruments → Leaks will flag this as "leaked
|
|
25
|
+
// allocation rooted at the singleton" — that's noise, not a real leak
|
|
26
|
+
// (process termination reclaims it). Phase 3c will keep this shape.
|
|
27
|
+
|
|
28
|
+
#import "RNSARWorkletRuntime.h"
|
|
29
|
+
#import "StitcherFrameHostObject.h"
|
|
30
|
+
|
|
31
|
+
#import <Foundation/Foundation.h>
|
|
32
|
+
#import <os/log.h>
|
|
33
|
+
|
|
34
|
+
#include <jsi/jsi.h>
|
|
35
|
+
// worklets-core headers — use quotes-include since the pod
|
|
36
|
+
// publishes them via HEADER_SEARCH_PATHS, not as a framework
|
|
37
|
+
// module map. Same pattern KeyframeGateFrameProcessor.mm uses
|
|
38
|
+
// for vision-camera headers (which are reachable via <angle>
|
|
39
|
+
// only because vc's podspec sets `define_module` differently).
|
|
40
|
+
#include "WKTJsiWorkletContext.h"
|
|
41
|
+
#include "WKTJsiWorklet.h"
|
|
42
|
+
|
|
43
|
+
#include "stitcher_worklet_registry.hpp"
|
|
44
|
+
|
|
45
|
+
#include <exception>
|
|
46
|
+
#include <memory>
|
|
47
|
+
#include <utility>
|
|
48
|
+
#include <vector>
|
|
49
|
+
|
|
50
|
+
// Forward-declare `RNSARFramePose` — same pattern as
|
|
51
|
+
// StitcherFrameHostObject.mm. We don't read its fields here in
|
|
52
|
+
// Phase 3b (the stub doesn't unpack the pose), but Phase 3c will.
|
|
53
|
+
@class RNSARFramePose;
|
|
54
|
+
|
|
55
|
+
@implementation RNSARWorkletRuntime {
|
|
56
|
+
/// Dispatch queue the worklet runtime's `workletCallInvoker`
|
|
57
|
+
/// posts onto. Serial; `DISPATCH_QUEUE_SERIAL` matches the
|
|
58
|
+
/// existing `IncrementalStitcher::workQueue` cost envelope
|
|
59
|
+
/// (one-at-a-time frame ingest).
|
|
60
|
+
///
|
|
61
|
+
/// Phase 3c will configure `ARSession.delegateQueue` to point
|
|
62
|
+
/// at the same queue so the delegate fires on the worklet
|
|
63
|
+
/// thread — eliminates a thread hop per frame + makes the
|
|
64
|
+
/// "first-party first, host worklets after" ordering trivial
|
|
65
|
+
/// to enforce (all on one queue).
|
|
66
|
+
dispatch_queue_t _dispatchQueue;
|
|
67
|
+
|
|
68
|
+
/// The wrapped worklet-runtime context. Constructed lazily on
|
|
69
|
+
/// `-installIfNeeded`; held for the singleton's lifetime
|
|
70
|
+
/// (process-wide).
|
|
71
|
+
std::shared_ptr<RNWorklet::JsiWorkletContext> _ctx;
|
|
72
|
+
|
|
73
|
+
/// Single-flight install guard. `BOOL` is sufficient because
|
|
74
|
+
/// `-installIfNeeded` synchronises on `_installLock` below.
|
|
75
|
+
BOOL _installed;
|
|
76
|
+
|
|
77
|
+
/// Lock for `_installed` + `_ctx`. Construction may race with
|
|
78
|
+
/// concurrent first-mount calls from multiple `<Camera>`
|
|
79
|
+
/// instances; serialise to ensure exactly-once init.
|
|
80
|
+
NSLock *_installLock;
|
|
81
|
+
|
|
82
|
+
// Phase 4 will add the host-worklet registry here. Storage
|
|
83
|
+
// shape (NSMutableArray of boxed shared_ptrs vs C++ vector
|
|
84
|
+
// ivar) is intentionally NOT pre-committed in Phase 3b — let
|
|
85
|
+
// the JSI plugin's actual register/unregister implementation
|
|
86
|
+
// pick the natural shape.
|
|
87
|
+
|
|
88
|
+
/// Phase 3c — first-party callback installed by RNSARSession.
|
|
89
|
+
/// Invoked synchronously on the caller thread per AR frame.
|
|
90
|
+
/// Cleared on RNSARSession.stop() to avoid retain cycles.
|
|
91
|
+
///
|
|
92
|
+
/// Atomic property protects against the delegate firing
|
|
93
|
+
/// concurrently with a setFirstPartyCallback: call on a
|
|
94
|
+
/// different thread (rare but possible: setter on main thread
|
|
95
|
+
/// from RNSARSession.start while a delayed delegate frame
|
|
96
|
+
/// arrives).
|
|
97
|
+
RNSARFirstPartyCallback _firstPartyCallback;
|
|
98
|
+
|
|
99
|
+
/// Lock for `_firstPartyCallback` reads + writes. The
|
|
100
|
+
/// `_installLock` above is dispatch-queue-scoped (install);
|
|
101
|
+
/// callback rotation is a separate concern.
|
|
102
|
+
NSLock *_callbackLock;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
+ (instancetype)shared {
|
|
106
|
+
static RNSARWorkletRuntime *sInstance;
|
|
107
|
+
static dispatch_once_t once;
|
|
108
|
+
dispatch_once(&once, ^{ sInstance = [[self alloc] init]; });
|
|
109
|
+
return sInstance;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
- (instancetype)init {
|
|
113
|
+
if ((self = [super init])) {
|
|
114
|
+
_dispatchQueue = dispatch_queue_create(
|
|
115
|
+
"io.imagestitcher.ar-worklet-runtime", DISPATCH_QUEUE_SERIAL);
|
|
116
|
+
_installed = NO;
|
|
117
|
+
_installLock = [[NSLock alloc] init];
|
|
118
|
+
_callbackLock = [[NSLock alloc] init];
|
|
119
|
+
_firstPartyCallback = nil;
|
|
120
|
+
}
|
|
121
|
+
return self;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
- (void)setFirstPartyCallback:(RNSARFirstPartyCallback)callback {
|
|
125
|
+
[_callbackLock lock];
|
|
126
|
+
// Copy the block to move it from stack to heap (ARC handles
|
|
127
|
+
// the copy semantics for blocks assigned to strong ivars).
|
|
128
|
+
_firstPartyCallback = [callback copy];
|
|
129
|
+
[_callbackLock unlock];
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
- (void)installIfNeeded {
|
|
133
|
+
[_installLock lock];
|
|
134
|
+
if (_installed) {
|
|
135
|
+
[_installLock unlock];
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Build the `workletCallInvoker`. `RNWorklet::JsiWorkletContext`
|
|
140
|
+
// accepts a `std::function<void(std::function<void()>&&)>` that
|
|
141
|
+
// posts a task onto whatever thread the runtime should execute
|
|
142
|
+
// on. We post onto `_dispatchQueue` (a serial GCD queue).
|
|
143
|
+
//
|
|
144
|
+
// The captured `fp` is moved into a `std::shared_ptr` so the
|
|
145
|
+
// dispatch_async block (which can only capture copyable types)
|
|
146
|
+
// can hold + invoke it. Without the shared_ptr indirection
|
|
147
|
+
// we'd hit `std::function` copy-construction on the
|
|
148
|
+
// non-copyable forward closure.
|
|
149
|
+
dispatch_queue_t queue = _dispatchQueue;
|
|
150
|
+
auto invoker = [queue](std::function<void()>&& fp) {
|
|
151
|
+
auto fpHolder = std::make_shared<std::function<void()>>(std::move(fp));
|
|
152
|
+
dispatch_async(queue, ^{ (*fpHolder)(); });
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
_ctx = std::make_shared<RNWorklet::JsiWorkletContext>(
|
|
156
|
+
"stitcher.ar", std::move(invoker));
|
|
157
|
+
_installed = YES;
|
|
158
|
+
[_installLock unlock];
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
- (BOOL)isInstalled {
|
|
162
|
+
[_installLock lock];
|
|
163
|
+
BOOL result = _installed;
|
|
164
|
+
[_installLock unlock];
|
|
165
|
+
return result;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
- (void)dispatchFrame:(ARFrame *)arFrame pose:(RNSARFramePose *)pose {
|
|
169
|
+
// ── Phase 3c — first-party (synchronous on caller thread) ────
|
|
170
|
+
//
|
|
171
|
+
// The callback (installed by RNSARSession.start) wraps the
|
|
172
|
+
// existing `incrementalConsumer.consumeFrame(...)` call path,
|
|
173
|
+
// so net behavior is byte-identical to the v0.7.x direct call.
|
|
174
|
+
//
|
|
175
|
+
// **Why first-party runs on the CALLER thread (not the worklet
|
|
176
|
+
// thread):** ARKit's pool reuse contract requires the pixel
|
|
177
|
+
// buffer to be consumed before this method returns. The Swift
|
|
178
|
+
// consumer does that synchronously inside `consumeFrame(...)`
|
|
179
|
+
// (converts NV12 → cv::Mat synchronously, then defers heavier
|
|
180
|
+
// work to its own queue). If we posted the callback onto
|
|
181
|
+
// `_dispatchQueue`, the delegate would return before
|
|
182
|
+
// `consumeFrame` ran, ARKit could reclaim the buffer, and we'd
|
|
183
|
+
// get torn frames.
|
|
184
|
+
//
|
|
185
|
+
// Pull the callback under the lock so a concurrent
|
|
186
|
+
// `setFirstPartyCallback:` doesn't race with our invocation.
|
|
187
|
+
[_callbackLock lock];
|
|
188
|
+
RNSARFirstPartyCallback cb = _firstPartyCallback;
|
|
189
|
+
[_callbackLock unlock];
|
|
190
|
+
if (cb != nil) {
|
|
191
|
+
cb(arFrame, pose);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ── Phase 4b — host-worklet fan-out (async on worklet thread) ──
|
|
195
|
+
//
|
|
196
|
+
// Snapshot the native registry. Fast-path early-exit when no
|
|
197
|
+
// host worklets are registered — saves the host-object alloc
|
|
198
|
+
// + dispatch_async hop on every frame (the common case in
|
|
199
|
+
// first-party-only deployments).
|
|
200
|
+
auto invokers = retailens::StitcherWorkletRegistry::shared().snapshot();
|
|
201
|
+
if (invokers.empty()) {
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Construction must happen on the caller thread. The
|
|
206
|
+
// `IOSPixelBufferReader` ctor takes a `CFBridgingRetain(arFrame)`
|
|
207
|
+
// so the underlying CVPixelBuffer stays alive until the host
|
|
208
|
+
// object's `invalidate` runs. ARKit's pool will throttle the
|
|
209
|
+
// *next* frame's delegate call while we hold this retain
|
|
210
|
+
// (acceptable for Phase 4b minimum-viable; a per-frame buffer
|
|
211
|
+
// copy is a known optimization for later if throughput
|
|
212
|
+
// suffers).
|
|
213
|
+
StitcherFrameHostObject *hostObj =
|
|
214
|
+
[StitcherFrameHostObject fromARFrame:arFrame pose:pose];
|
|
215
|
+
|
|
216
|
+
// Hand the host object's jsi::HostObject shared_ptr (boxed as
|
|
217
|
+
// void*) into the lambda. The lambda will:
|
|
218
|
+
// 1. Cast back to `std::shared_ptr<jsi::HostObject>*`
|
|
219
|
+
// 2. Construct the JS-side `jsi::Object` from the host object
|
|
220
|
+
// 3. Invoke each registered WorkletInvoker with the JS-side
|
|
221
|
+
// object as its single argument
|
|
222
|
+
// 4. Delete the boxed shared_ptr
|
|
223
|
+
// 5. Invalidate the host object on caller-side retained ref
|
|
224
|
+
//
|
|
225
|
+
// The dispatch is via worklets-core's `JsiWorkletContext::
|
|
226
|
+
// invokeOnWorkletThread` — internally posts onto our serial
|
|
227
|
+
// `_dispatchQueue` via the `workletCallInvoker` we set up in
|
|
228
|
+
// `installIfNeeded`.
|
|
229
|
+
//
|
|
230
|
+
// `hostObj` (the Obj-C facade) is captured by the block; ARC
|
|
231
|
+
// retains it for the block's lifetime, so the host object
|
|
232
|
+
// outlives the dispatch. We invalidate AFTER all worklets
|
|
233
|
+
// return.
|
|
234
|
+
void *hostObjPtr = [hostObj jsiHostObjectPtr];
|
|
235
|
+
if (hostObjPtr == NULL) {
|
|
236
|
+
// Host object construction failed (e.g., ARFrame was nil).
|
|
237
|
+
// Skip fan-out.
|
|
238
|
+
os_log_error(OS_LOG_DEFAULT,
|
|
239
|
+
"[RNSARWorkletRuntime] host object jsiHostObjectPtr was NULL; "
|
|
240
|
+
"skipping host-worklet fan-out for this frame.");
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
if (_ctx == nullptr) {
|
|
245
|
+
// installIfNeeded wasn't called. This shouldn't happen
|
|
246
|
+
// because RNSARSession.start calls installIfNeeded before
|
|
247
|
+
// any frames arrive, but guard defensively.
|
|
248
|
+
os_log_error(OS_LOG_DEFAULT,
|
|
249
|
+
"[RNSARWorkletRuntime] _ctx is nullptr in dispatchFrame; "
|
|
250
|
+
"did installIfNeeded run? Skipping host-worklet fan-out.");
|
|
251
|
+
// Leaked: hostObjPtr (boxed shared_ptr). Reclaim it here so
|
|
252
|
+
// we don't leak even on the defensive path.
|
|
253
|
+
delete static_cast<std::shared_ptr<facebook::jsi::HostObject>*>(hostObjPtr);
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
_ctx->invokeOnWorkletThread(
|
|
258
|
+
[invokers, hostObjPtr, hostObj](
|
|
259
|
+
RNWorklet::JsiWorkletContext* /*ctx*/,
|
|
260
|
+
facebook::jsi::Runtime& rt) {
|
|
261
|
+
// Reclaim the boxed shared_ptr. After this scope the
|
|
262
|
+
// unique_ptr automatically deletes the heap allocation
|
|
263
|
+
// even if the JSI call below throws.
|
|
264
|
+
std::unique_ptr<std::shared_ptr<facebook::jsi::HostObject>> spBox(
|
|
265
|
+
static_cast<std::shared_ptr<facebook::jsi::HostObject>*>(
|
|
266
|
+
hostObjPtr));
|
|
267
|
+
|
|
268
|
+
facebook::jsi::Object frameJsi =
|
|
269
|
+
facebook::jsi::Object::createFromHostObject(rt, *spBox);
|
|
270
|
+
// Pass the host object as a single argument. The
|
|
271
|
+
// worklet's signature is `(frame: StitcherFrame) =>
|
|
272
|
+
// void` — matches.
|
|
273
|
+
//
|
|
274
|
+
// Construct the argument value as a copy of the
|
|
275
|
+
// Object (jsi::Value(rt, obj) makes a fresh Value
|
|
276
|
+
// wrapping the same host object — refcounted by JSI).
|
|
277
|
+
facebook::jsi::Value frameVal(rt, frameJsi);
|
|
278
|
+
|
|
279
|
+
for (const auto& entry : invokers) {
|
|
280
|
+
if (!entry.invoker) continue;
|
|
281
|
+
try {
|
|
282
|
+
entry.invoker->call(rt, facebook::jsi::Value::undefined(),
|
|
283
|
+
&frameVal, 1);
|
|
284
|
+
} catch (const facebook::jsi::JSError& jsErr) {
|
|
285
|
+
// Per-worklet failure isolation: one host
|
|
286
|
+
// worklet throwing must NOT stop the lib's own
|
|
287
|
+
// path or other host worklets. Log + continue.
|
|
288
|
+
os_log_error(OS_LOG_DEFAULT,
|
|
289
|
+
"[RNSARWorkletRuntime] host worklet '%{public}s' "
|
|
290
|
+
"threw JS error: %{public}s",
|
|
291
|
+
entry.id.c_str(), jsErr.what());
|
|
292
|
+
} catch (const std::exception& e) {
|
|
293
|
+
os_log_error(OS_LOG_DEFAULT,
|
|
294
|
+
"[RNSARWorkletRuntime] host worklet '%{public}s' "
|
|
295
|
+
"threw native exception: %{public}s",
|
|
296
|
+
entry.id.c_str(), e.what());
|
|
297
|
+
} catch (...) {
|
|
298
|
+
os_log_error(OS_LOG_DEFAULT,
|
|
299
|
+
"[RNSARWorkletRuntime] host worklet '%{public}s' "
|
|
300
|
+
"threw unknown exception", entry.id.c_str());
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// Drop the JSI references BEFORE invalidating the host
|
|
305
|
+
// object — `frameJsi` / `frameVal` go out of scope at
|
|
306
|
+
// end of lambda anyway, but be explicit. Then
|
|
307
|
+
// invalidate the Obj-C facade which releases the
|
|
308
|
+
// CFBridgingRetain'd ARFrame so ARKit's pool can recycle.
|
|
309
|
+
[hostObj invalidate];
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
@end
|