react-native-image-stitcher 0.14.2 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/CHANGELOG.md +131 -0
  2. package/README.md +35 -0
  3. package/RNImageStitcher.podspec +8 -7
  4. package/android/build.gradle +0 -16
  5. package/android/src/main/cpp/CMakeLists.txt +2 -63
  6. package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
  7. package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
  8. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
  9. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
  10. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
  11. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
  12. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
  13. package/cpp/keyframe_gate.cpp +82 -23
  14. package/cpp/keyframe_gate.hpp +31 -2
  15. package/cpp/stitcher.cpp +208 -28
  16. package/cpp/tests/CMakeLists.txt +18 -12
  17. package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
  18. package/cpp/tests/warp_guard_test.cpp +48 -0
  19. package/cpp/warp_guard.hpp +41 -0
  20. package/dist/camera/Camera.d.ts +31 -16
  21. package/dist/camera/Camera.js +10 -2
  22. package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
  23. package/dist/camera/CaptureStitchStatsToast.js +27 -7
  24. package/dist/camera/PanoramaSettings.d.ts +10 -223
  25. package/dist/camera/PanoramaSettings.js +6 -28
  26. package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
  27. package/dist/camera/PanoramaSettingsBridge.js +3 -102
  28. package/dist/camera/PanoramaSettingsModal.js +7 -1
  29. package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
  30. package/dist/camera/buildPanoramaInitialSettings.js +4 -0
  31. package/dist/camera/cameraErrorMessages.d.ts +32 -0
  32. package/dist/camera/cameraErrorMessages.js +53 -0
  33. package/dist/camera/selectCaptureDevice.d.ts +5 -1
  34. package/dist/camera/selectCaptureDevice.js +22 -2
  35. package/dist/camera/useCapture.js +38 -0
  36. package/dist/index.d.ts +5 -8
  37. package/dist/index.js +11 -34
  38. package/dist/stitching/incremental.d.ts +1 -117
  39. package/dist/stitching/stitchVideo.d.ts +0 -35
  40. package/dist/types.d.ts +0 -87
  41. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
  42. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
  43. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
  44. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
  45. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
  46. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
  47. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
  48. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
  49. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
  50. package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
  51. package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
  52. package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
  53. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
  54. package/package.json +3 -2
  55. package/src/camera/Camera.tsx +43 -22
  56. package/src/camera/CaptureStitchStatsToast.tsx +58 -14
  57. package/src/camera/PanoramaSettings.ts +16 -289
  58. package/src/camera/PanoramaSettingsBridge.ts +3 -114
  59. package/src/camera/PanoramaSettingsModal.tsx +14 -1
  60. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
  61. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
  62. package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
  63. package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
  64. package/src/camera/buildPanoramaInitialSettings.ts +17 -0
  65. package/src/camera/cameraErrorMessages.ts +84 -0
  66. package/src/camera/selectCaptureDevice.ts +28 -3
  67. package/src/camera/useCapture.ts +44 -1
  68. package/src/index.ts +11 -40
  69. package/src/stitching/incremental.ts +3 -140
  70. package/src/stitching/stitchVideo.ts +0 -26
  71. package/src/types.ts +0 -95
  72. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
  73. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
  74. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
  75. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
  76. package/cpp/stitcher_frame_jsi.cpp +0 -214
  77. package/cpp/stitcher_frame_jsi.hpp +0 -108
  78. package/cpp/stitcher_proxy_jsi.cpp +0 -109
  79. package/cpp/stitcher_proxy_jsi.hpp +0 -46
  80. package/cpp/stitcher_worklet_dispatch.cpp +0 -103
  81. package/cpp/stitcher_worklet_dispatch.hpp +0 -71
  82. package/cpp/stitcher_worklet_registry.cpp +0 -91
  83. package/cpp/stitcher_worklet_registry.hpp +0 -146
  84. package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
  85. package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
  86. package/dist/stitching/IncrementalStitcherView.js +0 -157
  87. package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
  88. package/dist/stitching/StitcherWorkletRegistry.js +0 -78
  89. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
  90. package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
  91. package/dist/stitching/useFrameProcessor.d.ts +0 -119
  92. package/dist/stitching/useFrameProcessor.js +0 -196
  93. package/dist/stitching/useFrameStream.d.ts +0 -34
  94. package/dist/stitching/useFrameStream.js +0 -234
  95. package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
  96. package/dist/stitching/useThrottledFrameProcessor.js +0 -132
  97. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
  98. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
  99. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
  100. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
  101. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
  102. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
  103. package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
  104. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  105. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
  106. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
  107. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
  108. package/src/stitching/IncrementalStitcherView.tsx +0 -198
  109. package/src/stitching/StitcherWorkletRegistry.ts +0 -156
  110. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
  111. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
  112. package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
  113. package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
  114. package/src/stitching/useFrameProcessor.ts +0 -226
  115. package/src/stitching/useFrameStream.ts +0 -271
  116. package/src/stitching/useThrottledFrameProcessor.ts +0 -145
@@ -1,128 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // RNSARWorkletRuntime.h — Obj-C facade for the v0.8.0 AR-mode
4
- // worklet runtime. Wraps `react-native-worklets-core`'s
5
- // `RNWorklet::JsiWorkletContext` (the same primitive vision-camera
6
- // uses for its Frame Processor runtime) so the lib can dispatch
7
- // per-ARFrame worklets on a thread we own — rather than ARKit's
8
- // delegate queue, where doing significant work would block the
9
- // AR session's update loop.
10
- //
11
- // ## Phase 3b scope (this commit)
12
- //
13
- // Owns:
14
- // - The dispatch queue the worklet runtime pins to.
15
- // - The underlying `JsiWorkletContext` (constructed lazily on
16
- // `installIfNeeded`, lives for the singleton's lifetime).
17
- //
18
- // Exposes:
19
- // - `+ shared` singleton accessor.
20
- // - `- installIfNeeded` (idempotent runtime construction).
21
- // - `- isInstalled` for diagnostics + tests.
22
- // - `- dispatchFrame:pose:` — currently a no-op stub; Phase 3c
23
- // fills in the actual host-object construction + worklet
24
- // invocation + first-party stitching dispatch.
25
- //
26
- // Host-worklet registry is intentionally NOT in Phase 3b — Phase 4
27
- // lands the JSI plugin + TS-side hook that defines the storage
28
- // shape (NSMutableArray of boxed shared_ptrs vs a C++ vector ivar
29
- // vs something else). Pre-committing the storage type here would
30
- // risk rework. See
31
- // `docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md`
32
- // Phase 4 section for the planned API.
33
- //
34
- // ## Why Obj-C facade with `.mm` implementation
35
- //
36
- // The implementation needs to hold `std::shared_ptr<JsiWorkletContext>`
37
- // + run JSI value construction, which can't live in pure Swift. Same
38
- // pattern as `KeyframeGateBridge.{h,mm}` + `StitcherFrameHostObject.{h,mm}`:
39
- // keep the header umbrella-safe (no JSI imports), put the C++ glue in
40
- // the .mm.
41
- //
42
- // ## Header umbrella safety
43
- //
44
- // This .h imports only Foundation + ARKit (both system frameworks).
45
- // Worklets-core types are confined to the .mm.
46
-
47
- #pragma once
48
-
49
- #import <Foundation/Foundation.h>
50
- #import <ARKit/ARKit.h>
51
-
52
- @class RNSARFramePose;
53
-
54
- NS_ASSUME_NONNULL_BEGIN
55
-
56
- NS_SWIFT_NAME(RNSARWorkletRuntime)
57
- @interface RNSARWorkletRuntime : NSObject
58
-
59
- /// Singleton accessor. One AR worklet runtime per process; multiple
60
- /// `<Camera>` mounts share it. Construction is cheap (just an Obj-C
61
- /// alloc + an `NSMutableArray`); the heavy JSI work happens in
62
- /// `-installIfNeeded`.
63
- + (instancetype)shared;
64
-
65
- /// Construct the underlying `JsiWorkletContext` if not yet
66
- /// installed. Idempotent — repeated calls are no-ops. Called from
67
- /// `RNSARSession` at AR-mode start time (Phase 3c will wire this
68
- /// up; Phase 3b ships the method but no one calls it yet).
69
- ///
70
- /// Threading: safe to call from any thread; internally serialised.
71
- /// The runtime's own dispatch queue starts running once installed.
72
- - (void)installIfNeeded;
73
-
74
- /// Diagnostics + tests. Returns `YES` after a successful
75
- /// `-installIfNeeded`.
76
- - (BOOL)isInstalled;
77
-
78
- /// Phase 3c — type of the first-party stitching callback. Invoked
79
- /// synchronously on the caller thread (`ARSession.delegateQueue` —
80
- /// typically main queue today) per AR frame. Block must consume
81
- /// the pixel buffer before returning (ARKit pool reuse contract).
82
- typedef void (^RNSARFirstPartyCallback)(ARFrame *arFrame,
83
- RNSARFramePose *pose);
84
-
85
- /// Phase 3c — install the closure that takes ownership of the
86
- /// per-frame first-party stitching dispatch. Called from
87
- /// `RNSARSession.start()` after the incremental consumer is set;
88
- /// the block then routes `dispatchFrame:pose:` calls through to
89
- /// the existing `incrementalConsumer.consumeFrame(...)` path.
90
- ///
91
- /// Pre-Phase-3c the delegate called the consumer directly. After
92
- /// Phase 3c the delegate calls `dispatchFrame:pose:` (this class)
93
- /// which invokes the callback. Net behavior is byte-identical;
94
- /// the indirection sets up the seam where Phase 4 will fan out to
95
- /// host worklets without changing the first-party path.
96
- ///
97
- /// Pass `nil` to clear (e.g. on `RNSARSession.stop()`). Idempotent.
98
- - (void)setFirstPartyCallback:(nullable RNSARFirstPartyCallback)callback;
99
-
100
- /// Dispatch one AR frame through the registered worklets. Called
101
- /// per `ARFrame` by `RNSARSession.delegate` once Phase 3c lands the
102
- /// migration (Phase 3b ships this method as a no-op stub so the
103
- /// runtime can be built + linked + the API surface fixed).
104
- ///
105
- /// The Phase 3c implementation will:
106
- /// 1. Build a `StitcherFrameHostObject` from `arFrame` + `pose`.
107
- /// 2. Run the first-party stitching synchronously on the caller
108
- /// thread (preserves today's `ingestFromARCameraView` cost
109
- /// envelope at the producer site).
110
- /// 3. If any host worklets are registered, dispatch the host
111
- /// object onto the worklet runtime's thread + invoke each
112
- /// worklet via `RNWorklet::WorkletInvoker::call`.
113
- /// 4. Invalidate the host object after all worklets return.
114
- ///
115
- /// Phase 3c gate: install/idempotence tests + this method's
116
- /// integration test required before merge. See
117
- /// `docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md`
118
- /// Phase 3c gate criteria.
119
- ///
120
- /// Threading: typically called from `ARSession.delegateQueue` (main
121
- /// queue by default; Phase 3c will pin it explicitly to a
122
- /// dedicated queue).
123
- - (void)dispatchFrame:(ARFrame *)arFrame pose:(RNSARFramePose *)pose
124
- NS_SWIFT_NAME(dispatchFrame(_:pose:));
125
-
126
- @end
127
-
128
- NS_ASSUME_NONNULL_END
@@ -1,313 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // RNSARWorkletRuntime.mm — Obj-C++ implementation. See the header
4
- // for the API contract. This file owns:
5
- //
6
- // - The dispatch queue the worklet runtime is pinned to
7
- // - The `std::shared_ptr<RNWorklet::JsiWorkletContext>` itself
8
- // - The registry of host worklets (Phase 4 wiring will populate
9
- // this via a JSI plugin entry point)
10
- //
11
- // Phase 3b scope: construct the context + expose the API. No
12
- // dispatch logic yet — `dispatchFrame:pose:` is a stub. Phase 3c
13
- // fills in (a) the host-object construction + worklet invocation,
14
- // (b) the first-party stitching callback, (c) the migration in
15
- // `RNSARSession.delegate`.
16
- //
17
- // ## Singleton lifetime note (for Leaks-tool readers)
18
- //
19
- // `+ shared` uses `dispatch_once`, so the singleton lives for the
20
- // process lifetime — same pattern as most Obj-C singletons. This
21
- // means the dispatch queue (created in `init`) + the JsiWorkletContext
22
- // (constructed lazily in `installIfNeeded`) + the `workletCallInvoker`
23
- // lambda that captures the queue are ALL retained until process
24
- // termination. Xcode Instruments → Leaks will flag this as "leaked
25
- // allocation rooted at the singleton" — that's noise, not a real leak
26
- // (process termination reclaims it). Phase 3c will keep this shape.
27
-
28
- #import "RNSARWorkletRuntime.h"
29
- #import "StitcherFrameHostObject.h"
30
-
31
- #import <Foundation/Foundation.h>
32
- #import <os/log.h>
33
-
34
- #include <jsi/jsi.h>
35
- // worklets-core headers — use quotes-include since the pod
36
- // publishes them via HEADER_SEARCH_PATHS, not as a framework
37
- // module map. Same pattern KeyframeGateFrameProcessor.mm uses
38
- // for vision-camera headers (which are reachable via <angle>
39
- // only because vc's podspec sets `define_module` differently).
40
- #include "WKTJsiWorkletContext.h"
41
- #include "WKTJsiWorklet.h"
42
-
43
- #include "stitcher_worklet_registry.hpp"
44
-
45
- #include <exception>
46
- #include <memory>
47
- #include <utility>
48
- #include <vector>
49
-
50
- // Forward-declare `RNSARFramePose` — same pattern as
51
- // StitcherFrameHostObject.mm. We don't read its fields here in
52
- // Phase 3b (the stub doesn't unpack the pose), but Phase 3c will.
53
- @class RNSARFramePose;
54
-
55
- @implementation RNSARWorkletRuntime {
56
- /// Dispatch queue the worklet runtime's `workletCallInvoker`
57
- /// posts onto. Serial; `DISPATCH_QUEUE_SERIAL` matches the
58
- /// existing `IncrementalStitcher::workQueue` cost envelope
59
- /// (one-at-a-time frame ingest).
60
- ///
61
- /// Phase 3c will configure `ARSession.delegateQueue` to point
62
- /// at the same queue so the delegate fires on the worklet
63
- /// thread — eliminates a thread hop per frame + makes the
64
- /// "first-party first, host worklets after" ordering trivial
65
- /// to enforce (all on one queue).
66
- dispatch_queue_t _dispatchQueue;
67
-
68
- /// The wrapped worklet-runtime context. Constructed lazily on
69
- /// `-installIfNeeded`; held for the singleton's lifetime
70
- /// (process-wide).
71
- std::shared_ptr<RNWorklet::JsiWorkletContext> _ctx;
72
-
73
- /// Single-flight install guard. `BOOL` is sufficient because
74
- /// `-installIfNeeded` synchronises on `_installLock` below.
75
- BOOL _installed;
76
-
77
- /// Lock for `_installed` + `_ctx`. Construction may race with
78
- /// concurrent first-mount calls from multiple `<Camera>`
79
- /// instances; serialise to ensure exactly-once init.
80
- NSLock *_installLock;
81
-
82
- // Phase 4 will add the host-worklet registry here. Storage
83
- // shape (NSMutableArray of boxed shared_ptrs vs C++ vector
84
- // ivar) is intentionally NOT pre-committed in Phase 3b — let
85
- // the JSI plugin's actual register/unregister implementation
86
- // pick the natural shape.
87
-
88
- /// Phase 3c — first-party callback installed by RNSARSession.
89
- /// Invoked synchronously on the caller thread per AR frame.
90
- /// Cleared on RNSARSession.stop() to avoid retain cycles.
91
- ///
92
- /// Atomic property protects against the delegate firing
93
- /// concurrently with a setFirstPartyCallback: call on a
94
- /// different thread (rare but possible: setter on main thread
95
- /// from RNSARSession.start while a delayed delegate frame
96
- /// arrives).
97
- RNSARFirstPartyCallback _firstPartyCallback;
98
-
99
- /// Lock for `_firstPartyCallback` reads + writes. The
100
- /// `_installLock` above is dispatch-queue-scoped (install);
101
- /// callback rotation is a separate concern.
102
- NSLock *_callbackLock;
103
- }
104
-
105
- + (instancetype)shared {
106
- static RNSARWorkletRuntime *sInstance;
107
- static dispatch_once_t once;
108
- dispatch_once(&once, ^{ sInstance = [[self alloc] init]; });
109
- return sInstance;
110
- }
111
-
112
- - (instancetype)init {
113
- if ((self = [super init])) {
114
- _dispatchQueue = dispatch_queue_create(
115
- "io.imagestitcher.ar-worklet-runtime", DISPATCH_QUEUE_SERIAL);
116
- _installed = NO;
117
- _installLock = [[NSLock alloc] init];
118
- _callbackLock = [[NSLock alloc] init];
119
- _firstPartyCallback = nil;
120
- }
121
- return self;
122
- }
123
-
124
- - (void)setFirstPartyCallback:(RNSARFirstPartyCallback)callback {
125
- [_callbackLock lock];
126
- // Copy the block to move it from stack to heap (ARC handles
127
- // the copy semantics for blocks assigned to strong ivars).
128
- _firstPartyCallback = [callback copy];
129
- [_callbackLock unlock];
130
- }
131
-
132
- - (void)installIfNeeded {
133
- [_installLock lock];
134
- if (_installed) {
135
- [_installLock unlock];
136
- return;
137
- }
138
-
139
- // Build the `workletCallInvoker`. `RNWorklet::JsiWorkletContext`
140
- // accepts a `std::function<void(std::function<void()>&&)>` that
141
- // posts a task onto whatever thread the runtime should execute
142
- // on. We post onto `_dispatchQueue` (a serial GCD queue).
143
- //
144
- // The captured `fp` is moved into a `std::shared_ptr` so the
145
- // dispatch_async block (which can only capture copyable types)
146
- // can hold + invoke it. Without the shared_ptr indirection
147
- // we'd hit `std::function` copy-construction on the
148
- // non-copyable forward closure.
149
- dispatch_queue_t queue = _dispatchQueue;
150
- auto invoker = [queue](std::function<void()>&& fp) {
151
- auto fpHolder = std::make_shared<std::function<void()>>(std::move(fp));
152
- dispatch_async(queue, ^{ (*fpHolder)(); });
153
- };
154
-
155
- _ctx = std::make_shared<RNWorklet::JsiWorkletContext>(
156
- "stitcher.ar", std::move(invoker));
157
- _installed = YES;
158
- [_installLock unlock];
159
- }
160
-
161
- - (BOOL)isInstalled {
162
- [_installLock lock];
163
- BOOL result = _installed;
164
- [_installLock unlock];
165
- return result;
166
- }
167
-
168
- - (void)dispatchFrame:(ARFrame *)arFrame pose:(RNSARFramePose *)pose {
169
- // ── Phase 3c — first-party (synchronous on caller thread) ────
170
- //
171
- // The callback (installed by RNSARSession.start) wraps the
172
- // existing `incrementalConsumer.consumeFrame(...)` call path,
173
- // so net behavior is byte-identical to the v0.7.x direct call.
174
- //
175
- // **Why first-party runs on the CALLER thread (not the worklet
176
- // thread):** ARKit's pool reuse contract requires the pixel
177
- // buffer to be consumed before this method returns. The Swift
178
- // consumer does that synchronously inside `consumeFrame(...)`
179
- // (converts NV12 → cv::Mat synchronously, then defers heavier
180
- // work to its own queue). If we posted the callback onto
181
- // `_dispatchQueue`, the delegate would return before
182
- // `consumeFrame` ran, ARKit could reclaim the buffer, and we'd
183
- // get torn frames.
184
- //
185
- // Pull the callback under the lock so a concurrent
186
- // `setFirstPartyCallback:` doesn't race with our invocation.
187
- [_callbackLock lock];
188
- RNSARFirstPartyCallback cb = _firstPartyCallback;
189
- [_callbackLock unlock];
190
- if (cb != nil) {
191
- cb(arFrame, pose);
192
- }
193
-
194
- // ── Phase 4b — host-worklet fan-out (async on worklet thread) ──
195
- //
196
- // Snapshot the native registry. Fast-path early-exit when no
197
- // host worklets are registered — saves the host-object alloc
198
- // + dispatch_async hop on every frame (the common case in
199
- // first-party-only deployments).
200
- auto invokers = retailens::StitcherWorkletRegistry::shared().snapshot();
201
- if (invokers.empty()) {
202
- return;
203
- }
204
-
205
- // Construction must happen on the caller thread. The
206
- // `IOSPixelBufferReader` ctor takes a `CFBridgingRetain(arFrame)`
207
- // so the underlying CVPixelBuffer stays alive until the host
208
- // object's `invalidate` runs. ARKit's pool will throttle the
209
- // *next* frame's delegate call while we hold this retain
210
- // (acceptable for Phase 4b minimum-viable; a per-frame buffer
211
- // copy is a known optimization for later if throughput
212
- // suffers).
213
- StitcherFrameHostObject *hostObj =
214
- [StitcherFrameHostObject fromARFrame:arFrame pose:pose];
215
-
216
- // Hand the host object's jsi::HostObject shared_ptr (boxed as
217
- // void*) into the lambda. The lambda will:
218
- // 1. Cast back to `std::shared_ptr<jsi::HostObject>*`
219
- // 2. Construct the JS-side `jsi::Object` from the host object
220
- // 3. Invoke each registered WorkletInvoker with the JS-side
221
- // object as its single argument
222
- // 4. Delete the boxed shared_ptr
223
- // 5. Invalidate the host object on caller-side retained ref
224
- //
225
- // The dispatch is via worklets-core's `JsiWorkletContext::
226
- // invokeOnWorkletThread` — internally posts onto our serial
227
- // `_dispatchQueue` via the `workletCallInvoker` we set up in
228
- // `installIfNeeded`.
229
- //
230
- // `hostObj` (the Obj-C facade) is captured by the block; ARC
231
- // retains it for the block's lifetime, so the host object
232
- // outlives the dispatch. We invalidate AFTER all worklets
233
- // return.
234
- void *hostObjPtr = [hostObj jsiHostObjectPtr];
235
- if (hostObjPtr == NULL) {
236
- // Host object construction failed (e.g., ARFrame was nil).
237
- // Skip fan-out.
238
- os_log_error(OS_LOG_DEFAULT,
239
- "[RNSARWorkletRuntime] host object jsiHostObjectPtr was NULL; "
240
- "skipping host-worklet fan-out for this frame.");
241
- return;
242
- }
243
-
244
- if (_ctx == nullptr) {
245
- // installIfNeeded wasn't called. This shouldn't happen
246
- // because RNSARSession.start calls installIfNeeded before
247
- // any frames arrive, but guard defensively.
248
- os_log_error(OS_LOG_DEFAULT,
249
- "[RNSARWorkletRuntime] _ctx is nullptr in dispatchFrame; "
250
- "did installIfNeeded run? Skipping host-worklet fan-out.");
251
- // Leaked: hostObjPtr (boxed shared_ptr). Reclaim it here so
252
- // we don't leak even on the defensive path.
253
- delete static_cast<std::shared_ptr<facebook::jsi::HostObject>*>(hostObjPtr);
254
- return;
255
- }
256
-
257
- _ctx->invokeOnWorkletThread(
258
- [invokers, hostObjPtr, hostObj](
259
- RNWorklet::JsiWorkletContext* /*ctx*/,
260
- facebook::jsi::Runtime& rt) {
261
- // Reclaim the boxed shared_ptr. After this scope the
262
- // unique_ptr automatically deletes the heap allocation
263
- // even if the JSI call below throws.
264
- std::unique_ptr<std::shared_ptr<facebook::jsi::HostObject>> spBox(
265
- static_cast<std::shared_ptr<facebook::jsi::HostObject>*>(
266
- hostObjPtr));
267
-
268
- facebook::jsi::Object frameJsi =
269
- facebook::jsi::Object::createFromHostObject(rt, *spBox);
270
- // Pass the host object as a single argument. The
271
- // worklet's signature is `(frame: StitcherFrame) =>
272
- // void` — matches.
273
- //
274
- // Construct the argument value as a copy of the
275
- // Object (jsi::Value(rt, obj) makes a fresh Value
276
- // wrapping the same host object — refcounted by JSI).
277
- facebook::jsi::Value frameVal(rt, frameJsi);
278
-
279
- for (const auto& entry : invokers) {
280
- if (!entry.invoker) continue;
281
- try {
282
- entry.invoker->call(rt, facebook::jsi::Value::undefined(),
283
- &frameVal, 1);
284
- } catch (const facebook::jsi::JSError& jsErr) {
285
- // Per-worklet failure isolation: one host
286
- // worklet throwing must NOT stop the lib's own
287
- // path or other host worklets. Log + continue.
288
- os_log_error(OS_LOG_DEFAULT,
289
- "[RNSARWorkletRuntime] host worklet '%{public}s' "
290
- "threw JS error: %{public}s",
291
- entry.id.c_str(), jsErr.what());
292
- } catch (const std::exception& e) {
293
- os_log_error(OS_LOG_DEFAULT,
294
- "[RNSARWorkletRuntime] host worklet '%{public}s' "
295
- "threw native exception: %{public}s",
296
- entry.id.c_str(), e.what());
297
- } catch (...) {
298
- os_log_error(OS_LOG_DEFAULT,
299
- "[RNSARWorkletRuntime] host worklet '%{public}s' "
300
- "threw unknown exception", entry.id.c_str());
301
- }
302
- }
303
-
304
- // Drop the JSI references BEFORE invalidating the host
305
- // object — `frameJsi` / `frameVal` go out of scope at
306
- // end of lambda anyway, but be explicit. Then
307
- // invalidate the Obj-C facade which releases the
308
- // CFBridgingRetain'd ARFrame so ARKit's pool can recycle.
309
- [hostObj invalidate];
310
- });
311
- }
312
-
313
- @end
@@ -1,185 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // SaveFrameAsJpegPlugin.mm — v0.9.0 Layer 1: vc Frame Processor plugin
4
- // that JPEG-encodes the supplied frame's pixel buffer to a host-
5
- // supplied path. Worklet-callable; thin wrapper around the standard
6
- // iOS CIImage → CGImage → UIImage → UIImageJPEGRepresentation path.
7
- //
8
- // JS-side usage (from a worklet — typically inside `useFrameStream`
9
- // (Layer 3) or directly from a custom `useFrameProcessor` body):
10
- //
11
- // const plugin = VisionCameraProxy.initFrameProcessorPlugin(
12
- // 'save_frame_as_jpeg', {},
13
- // );
14
- //
15
- // const fp = useFrameProcessor((frame) => {
16
- // 'worklet';
17
- // if (plugin == null) return;
18
- // const result = plugin.call(frame, {
19
- // path: '/path/to/output.jpg',
20
- // quality: 75, // 0-100; defaults to 75
21
- // });
22
- // // result: { ok: true, path, width, height } OR
23
- // // { ok: false, error: "..." }
24
- // }, [plugin]);
25
- //
26
- // ## Why a separate plugin (not folded into KeyframeGateFrameProcessor)
27
- //
28
- // `cv_flow_gate_process_frame` (the existing plugin) drives the lib's
29
- // FIRST-PARTY stitching pipeline: it consumes the frame, evaluates
30
- // the keyframe gate, dispatches into `IncrementalStitcher`. It owns
31
- // state.
32
- //
33
- // `save_frame_as_jpeg` is STATELESS — a pure encode-and-write function.
34
- // Mixing them would force every JS-side caller of either to pay both
35
- // codepaths' arg-parsing costs (and would confuse the use-case
36
- // boundary). Two plugins, one job each.
37
- //
38
- // ## CONDITIONAL COMPILATION
39
- //
40
- // Same `__has_include` guard as `KeyframeGateFrameProcessor.mm` — if
41
- // vision-camera isn't on the host's classpath, this file is a no-op
42
- // translation unit. See that file's header for the rationale.
43
-
44
- #import <Foundation/Foundation.h>
45
-
46
- #if __has_include(<VisionCamera/FrameProcessorPlugin.h>)
47
-
48
- #import <VisionCamera/Frame.h>
49
- #import <VisionCamera/FrameProcessorPlugin.h>
50
- #import <VisionCamera/FrameProcessorPluginRegistry.h>
51
- #import <VisionCamera/VisionCameraProxyHolder.h>
52
- #import <CoreVideo/CoreVideo.h>
53
- #import <CoreImage/CoreImage.h>
54
- #import <UIKit/UIKit.h>
55
-
56
- @interface SaveFrameAsJpegPlugin : FrameProcessorPlugin
57
- @end
58
-
59
- @implementation SaveFrameAsJpegPlugin
60
-
61
- - (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy
62
- withOptions:(NSDictionary* _Nullable)options {
63
- return [super initWithProxy:proxy withOptions:options];
64
- }
65
-
66
- // Helper: read a string arg with a fallback. Returns nil only when
67
- // the arg is missing AND no fallback was supplied.
68
- static NSString* sfj_argString(NSDictionary* args, NSString* key,
69
- NSString* _Nullable fallback) {
70
- id v = args[key];
71
- if ([v isKindOfClass:[NSString class]]) return (NSString*)v;
72
- return fallback;
73
- }
74
-
75
- // Helper: read a numeric arg (NSNumber or NSString-parseable) with a
76
- // fallback. Matches the pattern in KeyframeGateFrameProcessor.mm.
77
- static double sfj_argDouble(NSDictionary* args, NSString* key,
78
- double fallback) {
79
- id v = args[key];
80
- if ([v isKindOfClass:[NSNumber class]]) return [(NSNumber*)v doubleValue];
81
- if ([v isKindOfClass:[NSString class]]) return [(NSString*)v doubleValue];
82
- return fallback;
83
- }
84
-
85
- // The host-callable plugin entry point. vc dispatches each
86
- // `plugin.call(frame, args)` from a worklet here.
87
- //
88
- // ## Arguments
89
- //
90
- // - `path` (string, REQUIRED): absolute filesystem path to write
91
- // the JPEG to. Parent directory must exist (we don't `mkdir -p`).
92
- // Existing file is overwritten atomically.
93
- // - `quality` (number, optional): 0-100 JPEG quality. Default 75
94
- // (matches `KeyframeGate.onAccept`'s encoder). Clamped silently
95
- // to `[1, 100]`.
96
- //
97
- // ## Returns
98
- //
99
- // - On success: `{ ok: YES, path: <path>, width: <px>, height: <px> }`
100
- // - On failure: `{ ok: NO, error: "<reason>" }`
101
- //
102
- // Errors are surfaced via the result dict, NOT thrown as `JSError` —
103
- // host worklets that want to react to encoder failures (e.g., to
104
- // rotate slot paths, or to back off) can branch on `result.ok`
105
- // without try/catch boilerplate. Throwing would break the
106
- // Layer 3 `useFrameStream` flow which only sees the result.
107
- - (id)callback:(Frame*)frame withArguments:(NSDictionary*)arguments {
108
- NSString* path = sfj_argString(arguments, @"path", nil);
109
- if (path == nil) {
110
- return @{@"ok": @NO, @"error": @"missing required `path` argument"};
111
- }
112
- double q = sfj_argDouble(arguments, @"quality", 75.0);
113
- if (q < 1.0) q = 1.0;
114
- if (q > 100.0) q = 100.0;
115
-
116
- CMSampleBufferRef sampleBuffer = frame.buffer;
117
- if (sampleBuffer == NULL) {
118
- return @{@"ok": @NO, @"error": @"frame.buffer was NULL"};
119
- }
120
- CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
121
- if (pixelBuffer == NULL) {
122
- return @{@"ok": @NO, @"error": @"CMSampleBufferGetImageBuffer returned NULL"};
123
- }
124
-
125
- // CIImage → CGImage → UIImage → JPEG. Standard iOS path; the
126
- // CIContext + colorSpace are cheap to construct per-call (CoreImage
127
- // caches GPU resources internally). If profiling shows this in
128
- // the hot path, lift the context to a static; for v0.9.0 baseline,
129
- // per-call construction is fine.
130
- CIImage* ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
131
- if (ciImage == nil) {
132
- return @{@"ok": @NO, @"error": @"CIImage imageWithCVPixelBuffer returned nil"};
133
- }
134
- CIContext* ctx = [CIContext context];
135
- CGImageRef cgImage = [ctx createCGImage:ciImage fromRect:ciImage.extent];
136
- if (cgImage == NULL) {
137
- return @{@"ok": @NO, @"error": @"CIContext createCGImage failed"};
138
- }
139
- UIImage* uiImage = [UIImage imageWithCGImage:cgImage];
140
- size_t width = CGImageGetWidth(cgImage);
141
- size_t height = CGImageGetHeight(cgImage);
142
- CGImageRelease(cgImage);
143
-
144
- NSData* jpegData = UIImageJPEGRepresentation(uiImage, (CGFloat)(q / 100.0));
145
- if (jpegData == nil) {
146
- return @{@"ok": @NO, @"error": @"UIImageJPEGRepresentation returned nil"};
147
- }
148
-
149
- // Atomic write — under the hood NSData writes to a temp file then
150
- // renames. Avoids torn writes if a reader tries to open the path
151
- // mid-write (would otherwise see a partial JPEG and choke).
152
- NSError* err = nil;
153
- BOOL ok = [jpegData writeToFile:path
154
- options:NSDataWritingAtomic
155
- error:&err];
156
- if (!ok) {
157
- NSString* msg = err.localizedDescription ?: @"NSData writeToFile returned NO";
158
- return @{@"ok": @NO, @"error": msg};
159
- }
160
-
161
- return @{
162
- @"ok": @YES,
163
- @"path": path,
164
- @"width": @(width),
165
- @"height": @(height),
166
- };
167
- }
168
-
169
- // Auto-register the plugin at class-load time. Name must match what
170
- // JS passes to `VisionCameraProxy.initFrameProcessorPlugin('save_frame_as_jpeg')`.
171
- // Same pattern as KeyframeGateFrameProcessor's +load.
172
- + (void)load {
173
- [FrameProcessorPluginRegistry
174
- addFrameProcessorPlugin:@"save_frame_as_jpeg"
175
- withInitializer:^FrameProcessorPlugin* _Nonnull(
176
- VisionCameraProxyHolder* proxy,
177
- NSDictionary* _Nullable options) {
178
- return [[SaveFrameAsJpegPlugin alloc]
179
- initWithProxy:proxy withOptions:options];
180
- }];
181
- }
182
-
183
- @end
184
-
185
- #endif // __has_include(<VisionCamera/FrameProcessorPlugin.h>)