react-native-image-stitcher 0.14.2 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/CHANGELOG.md +131 -0
  2. package/README.md +35 -0
  3. package/RNImageStitcher.podspec +8 -7
  4. package/android/build.gradle +0 -16
  5. package/android/src/main/cpp/CMakeLists.txt +2 -63
  6. package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
  7. package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
  8. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
  9. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
  10. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
  11. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
  12. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
  13. package/cpp/keyframe_gate.cpp +82 -23
  14. package/cpp/keyframe_gate.hpp +31 -2
  15. package/cpp/stitcher.cpp +208 -28
  16. package/cpp/tests/CMakeLists.txt +18 -12
  17. package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
  18. package/cpp/tests/warp_guard_test.cpp +48 -0
  19. package/cpp/warp_guard.hpp +41 -0
  20. package/dist/camera/Camera.d.ts +31 -16
  21. package/dist/camera/Camera.js +10 -2
  22. package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
  23. package/dist/camera/CaptureStitchStatsToast.js +27 -7
  24. package/dist/camera/PanoramaSettings.d.ts +10 -223
  25. package/dist/camera/PanoramaSettings.js +6 -28
  26. package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
  27. package/dist/camera/PanoramaSettingsBridge.js +3 -102
  28. package/dist/camera/PanoramaSettingsModal.js +7 -1
  29. package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
  30. package/dist/camera/buildPanoramaInitialSettings.js +4 -0
  31. package/dist/camera/cameraErrorMessages.d.ts +32 -0
  32. package/dist/camera/cameraErrorMessages.js +53 -0
  33. package/dist/camera/selectCaptureDevice.d.ts +5 -1
  34. package/dist/camera/selectCaptureDevice.js +22 -2
  35. package/dist/camera/useCapture.js +38 -0
  36. package/dist/index.d.ts +5 -8
  37. package/dist/index.js +11 -34
  38. package/dist/stitching/incremental.d.ts +1 -117
  39. package/dist/stitching/stitchVideo.d.ts +0 -35
  40. package/dist/types.d.ts +0 -87
  41. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
  42. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
  43. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
  44. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
  45. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
  46. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
  47. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
  48. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
  49. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
  50. package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
  51. package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
  52. package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
  53. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
  54. package/package.json +3 -2
  55. package/src/camera/Camera.tsx +43 -22
  56. package/src/camera/CaptureStitchStatsToast.tsx +58 -14
  57. package/src/camera/PanoramaSettings.ts +16 -289
  58. package/src/camera/PanoramaSettingsBridge.ts +3 -114
  59. package/src/camera/PanoramaSettingsModal.tsx +14 -1
  60. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
  61. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
  62. package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
  63. package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
  64. package/src/camera/buildPanoramaInitialSettings.ts +17 -0
  65. package/src/camera/cameraErrorMessages.ts +84 -0
  66. package/src/camera/selectCaptureDevice.ts +28 -3
  67. package/src/camera/useCapture.ts +44 -1
  68. package/src/index.ts +11 -40
  69. package/src/stitching/incremental.ts +3 -140
  70. package/src/stitching/stitchVideo.ts +0 -26
  71. package/src/types.ts +0 -95
  72. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
  73. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
  74. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
  75. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
  76. package/cpp/stitcher_frame_jsi.cpp +0 -214
  77. package/cpp/stitcher_frame_jsi.hpp +0 -108
  78. package/cpp/stitcher_proxy_jsi.cpp +0 -109
  79. package/cpp/stitcher_proxy_jsi.hpp +0 -46
  80. package/cpp/stitcher_worklet_dispatch.cpp +0 -103
  81. package/cpp/stitcher_worklet_dispatch.hpp +0 -71
  82. package/cpp/stitcher_worklet_registry.cpp +0 -91
  83. package/cpp/stitcher_worklet_registry.hpp +0 -146
  84. package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
  85. package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
  86. package/dist/stitching/IncrementalStitcherView.js +0 -157
  87. package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
  88. package/dist/stitching/StitcherWorkletRegistry.js +0 -78
  89. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
  90. package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
  91. package/dist/stitching/useFrameProcessor.d.ts +0 -119
  92. package/dist/stitching/useFrameProcessor.js +0 -196
  93. package/dist/stitching/useFrameStream.d.ts +0 -34
  94. package/dist/stitching/useFrameStream.js +0 -234
  95. package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
  96. package/dist/stitching/useThrottledFrameProcessor.js +0 -132
  97. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
  98. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
  99. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
  100. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
  101. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
  102. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
  103. package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
  104. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  105. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
  106. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
  107. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
  108. package/src/stitching/IncrementalStitcherView.tsx +0 -198
  109. package/src/stitching/StitcherWorkletRegistry.ts +0 -156
  110. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
  111. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
  112. package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
  113. package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
  114. package/src/stitching/useFrameProcessor.ts +0 -226
  115. package/src/stitching/useFrameStream.ts +0 -271
  116. package/src/stitching/useThrottledFrameProcessor.ts +0 -145
@@ -1,103 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- package io.imagestitcher.rn
3
-
4
- import android.util.Log
5
- import com.facebook.react.bridge.ReactApplicationContext
6
- import com.facebook.react.bridge.ReactContextBaseJavaModule
7
- import com.facebook.react.bridge.ReactMethod
8
-
9
- /**
10
- * v0.8.0 Phase 4b.ii — Android-side JSI installer for the host
11
- * worklet proxy. Mirror of iOS' `StitcherJsiInstaller`.
12
- *
13
- * The module exposes one synchronous method, `install()`, which JS
14
- * calls once at lib bootstrap (via the
15
- * `ensureStitcherProxyInstalled` helper in
16
- * `src/stitching/ensureStitcherProxyInstalled.ts`). We reach into
17
- * the main JS runtime via `ReactApplicationContext.getJavaScriptContextHolder().get()`
18
- * — the canonical bridgeless-compatible accessor in modern RN
19
- * (worklets-core's `WorkletsModule` uses the same pattern, verified
20
- * working on RN 0.84.1 + new arch + Hermes).
21
- *
22
- * The native `nativeInstall(jsiRuntimeRef)` JNI then casts the long
23
- * back to a `jsi::Runtime*` and calls into the shared C++
24
- * `retailens::installStitcherProxy(runtime)` (in
25
- * `cpp/stitcher_proxy_jsi.{hpp,cpp}`). Identical destination on
26
- * both platforms — `globalThis.__stitcherProxy` exposes the same
27
- * `install` / `uninstall` / `count` host functions.
28
- *
29
- * ## Returning `Boolean` (not `Promise`) from a sync method
30
- *
31
- * `isBlockingSynchronousMethod = true` + `Boolean` return is the
32
- * documented pattern for "I'm doing one-shot native setup that
33
- * needs to complete before the next JS line runs." Same shape as
34
- * `WorkletsModule.install()`.
35
- *
36
- * ## What we DON'T do here (Phase 4b.ii follow-up)
37
- *
38
- * Phase 4b.ii's MVP installs the proxy ONLY. Host worklets that
39
- * register through `__stitcherProxy.install` land in the native
40
- * `retailens::StitcherWorkletRegistry`. Per-frame fan-out from
41
- * Android's `StitcherWorkletRuntime` is a separate piece of work
42
- * (Phase 4b.ii follow-up) — needs the Kotlin↔JNI bridge that
43
- * constructs a `StitcherFrameJsiHostObject` from an `ArImage` +
44
- * pose and posts it through a worklet runtime. Until that lands,
45
- * Android-registered worklets behave exactly like iOS-registered
46
- * worklets BEFORE Phase 4b.i: they exist in the registry but
47
- * aren't invoked.
48
- *
49
- * The proxy install itself is still useful as a foundation —
50
- * verifies the JNI handshake works, exercises the bridgeless
51
- * runtime accessor, and gives us a `count()` smoke test for the
52
- * device verification step.
53
- */
54
- class StitcherJsiInstallerModule(
55
- private val reactContext: ReactApplicationContext,
56
- ) : ReactContextBaseJavaModule(reactContext) {
57
- override fun getName(): String = NAME
58
-
59
- @ReactMethod(isBlockingSynchronousMethod = true)
60
- fun install(): Boolean {
61
- return try {
62
- // `getJavaScriptContextHolder().get()` returns a raw
63
- // `jsi::Runtime*` boxed as `Long`. Same accessor
64
- // worklets-core's `WorkletsModule.install()` uses;
65
- // documented to work in both legacy + bridgeless modes
66
- // on RN 0.71+.
67
- val holder = reactContext.javaScriptContextHolder
68
- if (holder == null) {
69
- Log.e(TAG, "getJavaScriptContextHolder() returned null; runtime unreachable")
70
- return false
71
- }
72
- val runtimeRef = holder.get()
73
- if (runtimeRef == 0L) {
74
- Log.e(TAG, "JavaScriptContextHolder.get() returned 0; runtime not initialized yet")
75
- return false
76
- }
77
- val ok = nativeInstall(runtimeRef)
78
- if (!ok) {
79
- Log.e(TAG, "nativeInstall(runtimeRef=$runtimeRef) returned false")
80
- }
81
- ok
82
- } catch (t: Throwable) {
83
- Log.e(TAG, "install() threw — falling back to JS-side registry", t)
84
- false
85
- }
86
- }
87
-
88
- private external fun nativeInstall(jsiRuntimeRef: Long): Boolean
89
-
90
- companion object {
91
- const val NAME = "StitcherJsiInstaller"
92
- private const val TAG = "StitcherJsiInstaller"
93
-
94
- init {
95
- // The Phase 3a JNI shim (`libimage_stitcher.so`) absorbed
96
- // the JSI-install JNI binding from Phase 4b.ii. Loading
97
- // it once is enough — Android's loader deduplicates,
98
- // so even if `IncrementalStitcher.kt`'s init block
99
- // already loaded the lib, calling again is a cheap no-op.
100
- System.loadLibrary("image_stitcher")
101
- }
102
- }
103
- }
@@ -1,256 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- package io.imagestitcher.rn
3
-
4
- import android.os.HandlerThread
5
- import android.util.Log
6
- import java.util.concurrent.atomic.AtomicBoolean
7
-
8
- /**
9
- * v0.8.0 Phase 3b — Android twin of iOS' `RNSARWorkletRuntime`.
10
- * Owns the per-AR-frame worklet runtime + the thread it dispatches
11
- * on. Symmetric API shape to the iOS class so the cross-platform
12
- * dispatch story (Phase 3c) lands in lockstep.
13
- *
14
- * ## Phase 3b scope (this commit)
15
- *
16
- * - Singleton accessor + lifecycle (installIfNeeded / isInstalled).
17
- * - Dedicated `HandlerThread` for worklet dispatch — keeps work off
18
- * the GLSurfaceView GL render thread (audit caveat #4 from the
19
- * Phase-0 audit).
20
- * - `dispatchFrame()` stub — Phase 3c will fill in:
21
- * 1. Build `StitcherFrameHostObject` from ARCore Frame + pose
22
- * (via the shared C++ JSI host object now linked into
23
- * `libimage_stitcher.so` post-Phase-3a).
24
- * 2. Run first-party stitching synchronously on the caller
25
- * thread (`onDrawFrame`'s GL thread, today).
26
- * 3. If host worklets are registered, dispatch the host object
27
- * onto this runtime's `HandlerThread` + invoke each worklet
28
- * via JNI → `RNWorklet::WorkletInvoker::call`.
29
- * 4. Invalidate the host object after all worklets return.
30
- *
31
- * ## Worklet-runtime construction model
32
- *
33
- * Unlike iOS (where the lib's `.mm` directly `std::make_shared`s
34
- * a `RNWorklet::JsiWorkletContext`), Android can't construct the
35
- * context purely from native C++ without JNI plumbing. Phase 3c
36
- * will choose between two paths:
37
- *
38
- * - **Option A:** JS-side code calls
39
- * `Worklets.createContext("stitcher.ar")` at AR-mode start;
40
- * hands the resulting context pointer to this Kotlin class via
41
- * a small JSI plugin. Minimal new JNI. **Phase 3b's
42
- * HandlerThread becomes dead code** under this option — the
43
- * JS-side `Worklets.createContext` picks its own thread. We'd
44
- * need to remove the HandlerThread + change `installIfNeeded`
45
- * into a no-op until a `setContextHandle(Long)` setter lands.
46
- * - **Option B:** Direct JNI binding to worklets-core's C++
47
- * constructor. More native code but no JS dependency at runtime.
48
- * **Phase 3b's HandlerThread is exactly the right scaffold**
49
- * under this option — its looper becomes the JsiWorkletContext's
50
- * `workletCallInvoker` target.
51
- *
52
- * **Phase 3b assumption: Option B is the more likely path.** The
53
- * scaffolding below (HandlerThread + serial dispatch) fits Option
54
- * B; if Phase 3c picks Option A instead, the HandlerThread becomes
55
- * unused and Phase 3c will refactor accordingly.
56
- *
57
- * @see [RNSARWorkletRuntime] iOS equivalent
58
- * @see docs/plans/handoff/2026-05-26-v0.8.0-phase-0-audit.md
59
- * worklets-core API rationale (Audit 2: `JsiWorkletContext`).
60
- * @see docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md
61
- * Phase 3c implementation plan.
62
- */
63
- object StitcherWorkletRuntime {
64
- private const val TAG = "StitcherWorkletRuntime"
65
-
66
- /// Single-flight install guard. `compareAndSet` makes the
67
- /// runtime construction race-safe across concurrent first-mount
68
- /// calls from multiple `<Camera>` instances.
69
- private val installed = AtomicBoolean(false)
70
-
71
- /// Dedicated dispatch thread. Constructed eagerly so we can
72
- /// validate the thread starts cleanly during `installIfNeeded`.
73
- /// Off the GLSurfaceView GL render thread (audit caveat #4)
74
- /// + off the main thread. Phase 3c will configure the worklet
75
- /// context's `workletCallInvoker` to post onto this thread's
76
- /// looper.
77
- private val dispatchThread: HandlerThread by lazy {
78
- HandlerThread("io.imagestitcher.ar-worklet-runtime").apply { start() }
79
- }
80
-
81
- /// Construct the underlying worklet context if not yet installed.
82
- /// Idempotent — repeated calls are no-ops.
83
- ///
84
- /// Phase 3b: starts the dispatch thread; no JsiWorkletContext
85
- /// construction yet (deferred to Phase 3c).
86
- /// Phase 3c: also wires the JsiWorkletContext + binds it to the
87
- /// dispatch thread's looper.
88
- @JvmStatic
89
- fun installIfNeeded() {
90
- if (!installed.compareAndSet(false, true)) {
91
- return
92
- }
93
- // Force the lazy `dispatchThread` to initialise. If the
94
- // OS denies thread creation (extreme memory pressure on a
95
- // budget device), `HandlerThread.start()` won't throw but
96
- // the looper won't be available — the Phase 3c dispatch
97
- // logic will need to defend against that. For Phase 3b
98
- // we only care that this method returns without throwing.
99
- //
100
- // Log `Thread.id` (Java-side monotonic, always non-zero) —
101
- // NOT `HandlerThread.threadId` (Linux tid set after first
102
- // Looper-prepared message; reading from this caller thread
103
- // immediately after .start() returns -1 until scheduled).
104
- val javaThreadId = dispatchThread.id
105
- Log.i(TAG, "installed runtime; dispatch java-thread id=$javaThreadId")
106
- }
107
-
108
- /// Diagnostics + tests. Returns `true` after a successful
109
- /// `installIfNeeded()`.
110
- @JvmStatic
111
- fun isInstalled(): Boolean = installed.get()
112
-
113
- /// v0.8.0 Phase 3c — first-party stitching dispatch. Invokes
114
- /// the supplied block synchronously on the caller thread
115
- /// (`onDrawFrame`'s GL render thread today).
116
- ///
117
- /// Phase 3c minimum-viable: this is the closure-based equivalent
118
- /// of iOS' first-party callback. The block is the original
119
- /// `module.ingestFromARCameraView(...)` call site moved
120
- /// verbatim into a lambda — no behaviour change, just an
121
- /// indirection so Phase 4 can interpose host-worklet fanout
122
- /// without touching the engine ingest path.
123
- ///
124
- /// **Why synchronous + on the caller thread:** the engine's
125
- /// `ingestFromARCameraView` takes ownership of the ARCore
126
- /// `Image`-derived NV21 buffer (via the v0.10.0 `TransferredNV21`
127
- /// wrapper). ARCore's `Image.close()` happens after this call
128
- /// returns, so the consumer must finish reading the bytes before
129
- /// we return — exactly what synchronous block invocation
130
- /// provides. Phase 4 will copy the buffer for off-thread
131
- /// access in host worklets; Phase 3c keeps the sync contract.
132
- ///
133
- /// If `installIfNeeded()` hasn't been called yet, the block
134
- /// still runs (no-op on the registry side). Defensive — the
135
- /// caller may call this method before `installIfNeeded` is
136
- /// wired up.
137
- @JvmStatic
138
- fun runFirstParty(block: () -> Unit) {
139
- // Synchronous invocation — Phase 4 will extend this to also
140
- // post the registered host worklets onto `dispatchThread`.
141
- // Not `inline`: Phase 4 will need to read `dispatchThread`
142
- // (private) from inside this function body, and Kotlin's
143
- // inline functions can't access private members from call
144
- // sites outside the declaring class. Per-frame lambda
145
- // alloc is ~ns and the alternative (callers passing a
146
- // method reference) doesn't materially change cost.
147
- block()
148
- }
149
-
150
- /// v0.8.0 Phase 4b.iii — fan out one AR frame to every host
151
- /// worklet registered in the shared C++ `StitcherWorkletRegistry`
152
- /// (populated from JS via `__stitcherProxy.install(workletFn)`).
153
- ///
154
- /// Called from `RNSARCameraView.onDrawFrame` immediately after
155
- /// `runFirstParty { ... }` returns, with the already-extracted
156
- /// AR frame data (pose + NV21 bytes + dimensions + tracking
157
- /// state).
158
- ///
159
- /// **Fast-path:** the native side queries the registry's count
160
- /// FIRST and returns before copying any bytes when no host
161
- /// worklets are registered. In the common first-party-only
162
- /// deployment, this method costs one JNI call + one C++ atomic
163
- /// read per frame — negligible.
164
- ///
165
- /// **When host worklets ARE registered:** the JNI layer copies
166
- /// the NV21 byte array into an owned C++ `std::vector` (so the
167
- /// async dispatch can outlive ARCore's `Image.close()` scope),
168
- /// builds a `StitcherFrameJsiHostObject`, and posts a lambda
169
- /// onto worklets-core's default `JsiWorkletContext`'s worklet
170
- /// thread. The lambda iterates the registry's
171
- /// `WorkletInvoker`s, calls each with the JSI host object as
172
- /// its argument, and invalidates the host object after the
173
- /// last invoker returns. Per-worklet failure isolation: one
174
- /// host worklet throwing does NOT stop the lib's stitching or
175
- /// the other host worklets.
176
- ///
177
- /// **Threading:** this method returns synchronously on the
178
- /// caller's thread. The actual worklet invocations happen
179
- /// asynchronously on the worklets-core thread; the caller does
180
- /// NOT block on them.
181
- ///
182
- /// **Caller-thread contract:** the caller (`RNSARCameraView`'s
183
- /// `onDrawFrame`) MUST have already invoked `runFirstParty`
184
- /// before calling this method. The first-party stitching
185
- /// path holds the synchronous ARCore Image consumption
186
- /// contract; the host-worklet dispatch does not.
187
- ///
188
- /// @param nv21Bytes Pre-packed NV21 byte array. COPIED
189
- /// into a native owned buffer; caller can
190
- /// release the reference after return.
191
- /// @param width Camera image width (pixels).
192
- /// @param height Camera image height (pixels).
193
- /// @param qx,qy,qz,qw Pose rotation quaternion (unit length).
194
- /// @param tx,ty,tz Pose translation (metres, world coords).
195
- /// @param timestampNs Frame timestamp in nanoseconds.
196
- /// @param trackingState One of "" / "notAvailable" / "limited"
197
- /// / "normal". Empty string ⇒ JS-side
198
- /// `arTrackingState` is `undefined`.
199
- @JvmStatic
200
- fun dispatchToHostWorklets(
201
- nv21Bytes: ByteArray,
202
- width: Int,
203
- height: Int,
204
- qx: Double, qy: Double, qz: Double, qw: Double,
205
- tx: Double, ty: Double, tz: Double,
206
- timestampNs: Double,
207
- trackingState: String,
208
- ) {
209
- if (!installed.get()) return
210
- nativeDispatchToHostWorklets(
211
- nv21Bytes, width, height,
212
- qx, qy, qz, qw,
213
- tx, ty, tz,
214
- timestampNs, trackingState,
215
- )
216
- }
217
-
218
- /// v0.8.0 Phase 4b.iii — number of registered host worklets.
219
- /// Cheap (microsecond) call into the native registry. Used by
220
- /// `RNSARCameraView.onDrawFrame` to gate the per-frame
221
- /// NV21-pack + dispatch path: when no worklets are registered
222
- /// AND no capture is active, the entire `forwardToIncremental`
223
- /// branch can be skipped, saving the ~3-5ms NV21 pack cost per
224
- /// idle preview frame.
225
- @JvmStatic
226
- fun hasHostWorklets(): Boolean {
227
- if (!installed.get()) return false
228
- return nativeRegistryCount() > 0
229
- }
230
-
231
- @JvmStatic
232
- private external fun nativeRegistryCount(): Int
233
-
234
- /// JNI binding: `android/src/main/cpp/stitcher_jsi_install_jni.cpp`'s
235
- /// `nativeDispatchToHostWorklets`. Fast-path early-exit lives
236
- /// inside the native function — see its docstring.
237
- @JvmStatic
238
- private external fun nativeDispatchToHostWorklets(
239
- nv21Bytes: ByteArray,
240
- width: Int,
241
- height: Int,
242
- qx: Double, qy: Double, qz: Double, qw: Double,
243
- tx: Double, ty: Double, tz: Double,
244
- timestampNs: Double,
245
- trackingState: String,
246
- )
247
-
248
- init {
249
- // The JSI install module (`StitcherJsiInstallerModule`)
250
- // already loads `libimage_stitcher` at class load. We
251
- // load it again here defensively in case
252
- // `StitcherWorkletRuntime` is referenced before the install
253
- // module — `System.loadLibrary` is idempotent.
254
- System.loadLibrary("image_stitcher")
255
- }
256
- }
@@ -1,214 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // stitcher_frame_jsi.cpp — implementation of the shared C++ JSI
4
- // host object. See stitcher_frame_jsi.hpp for class docs.
5
-
6
- #include "stitcher_frame_jsi.hpp"
7
-
8
- #include <string>
9
- #include <utility>
10
-
11
- namespace retailens {
12
-
13
- using facebook::jsi::Array;
14
- using facebook::jsi::Function;
15
- using facebook::jsi::HostFunctionType;
16
- using facebook::jsi::JSError;
17
- using facebook::jsi::Object;
18
- using facebook::jsi::PropNameID;
19
- using facebook::jsi::Runtime;
20
- using facebook::jsi::String;
21
- using facebook::jsi::Value;
22
-
23
- StitcherFrameJsiHostObject::StitcherFrameJsiHostObject(StitcherFrameData data)
24
- : _data(std::move(data)), _isValid(true) {}
25
-
26
- void StitcherFrameJsiHostObject::invalidate() {
27
- _isValid = false;
28
- // Release the pixel reader immediately so the underlying camera
29
- // buffer can be reclaimed. ARKit's ARFrame uses a pooled
30
- // CVPixelBuffer; holding past the dispatch scope causes
31
- // back-pressure. ARCore's ArImage must be explicitly released
32
- // for the next frame's acquire to succeed.
33
- _data.pixelReader.reset();
34
- }
35
-
36
- std::vector<PropNameID> StitcherFrameJsiHostObject::getPropertyNames(
37
- Runtime& rt) {
38
- std::vector<PropNameID> names;
39
- names.push_back(PropNameID::forUtf8(rt, "isValid"));
40
- if (!_isValid) return names;
41
-
42
- names.push_back(PropNameID::forUtf8(rt, "width"));
43
- names.push_back(PropNameID::forUtf8(rt, "height"));
44
- names.push_back(PropNameID::forUtf8(rt, "pixelFormat"));
45
- names.push_back(PropNameID::forUtf8(rt, "orientation"));
46
- names.push_back(PropNameID::forUtf8(rt, "timestamp"));
47
- names.push_back(PropNameID::forUtf8(rt, "pose"));
48
- names.push_back(PropNameID::forUtf8(rt, "source"));
49
- names.push_back(PropNameID::forUtf8(rt, "toArrayBuffer"));
50
- if (!_data.arTrackingState.empty()) {
51
- names.push_back(PropNameID::forUtf8(rt, "arTrackingState"));
52
- }
53
- return names;
54
- }
55
-
56
- Value StitcherFrameJsiHostObject::get(Runtime& rt,
57
- const PropNameID& propName) {
58
- const std::string name = propName.utf8(rt);
59
-
60
- if (name == "isValid") {
61
- return Value(_isValid);
62
- }
63
- // Invalidated host objects expose only `isValid` (returns false).
64
- // Every other access throws — matches vc FrameHostObject's contract.
65
- // Lets worklets that incorrectly retain a frame across dispatch
66
- // boundaries fail loudly rather than read garbage.
67
- if (!_isValid) {
68
- throw JSError(rt,
69
- "[StitcherFrame] cannot access property '" + name +
70
- "' after host object was invalidated. "
71
- "Frame data is only valid for the duration of the worklet call.");
72
- }
73
-
74
- if (name == "width") return Value(static_cast<double>(_data.width));
75
- if (name == "height") return Value(static_cast<double>(_data.height));
76
- if (name == "pixelFormat") return String::createFromUtf8(rt, _data.pixelFormat);
77
- if (name == "orientation") return String::createFromUtf8(rt, _data.orientation);
78
- if (name == "timestamp") return Value(_data.timestampNs);
79
- if (name == "source") return String::createFromUtf8(rt, _data.source);
80
-
81
- if (name == "pose") {
82
- Object pose(rt);
83
- Array rotation(rt, 4);
84
- rotation.setValueAtIndex(rt, 0, Value(_data.qx));
85
- rotation.setValueAtIndex(rt, 1, Value(_data.qy));
86
- rotation.setValueAtIndex(rt, 2, Value(_data.qz));
87
- rotation.setValueAtIndex(rt, 3, Value(_data.qw));
88
- pose.setProperty(rt, "rotation", rotation);
89
- if (_data.hasTranslation) {
90
- Array translation(rt, 3);
91
- translation.setValueAtIndex(rt, 0, Value(_data.tx));
92
- translation.setValueAtIndex(rt, 1, Value(_data.ty));
93
- translation.setValueAtIndex(rt, 2, Value(_data.tz));
94
- pose.setProperty(rt, "translation", translation);
95
- }
96
- return pose;
97
- }
98
-
99
- if (name == "arTrackingState") {
100
- if (_data.arTrackingState.empty()) return Value::undefined();
101
- return String::createFromUtf8(rt, _data.arTrackingState);
102
- }
103
-
104
- if (name == "toArrayBuffer") {
105
- // Capture a weak self so the lambda doesn't extend the host
106
- // object's lifetime beyond what the runtime intended. When the
107
- // runtime releases its shared_ptr (after dispatch), the weak
108
- // ref expires and toArrayBuffer() throws on next call.
109
- auto weakSelf = std::weak_ptr<StitcherFrameJsiHostObject>(shared_from_this());
110
- HostFunctionType fn = [weakSelf](Runtime& runtime,
111
- const Value& thisVal,
112
- const Value* args,
113
- size_t count) -> Value {
114
- auto self = weakSelf.lock();
115
- if (!self || !self->_isValid || !self->_data.pixelReader) {
116
- throw JSError(runtime,
117
- "[StitcherFrame] toArrayBuffer() called on invalidated frame "
118
- "(host object was released after the worklet dispatch returned)");
119
- }
120
- const std::size_t bufSize = self->_data.pixelReader->byteSize();
121
-
122
- // Per-runtime ArrayBuffer cache. Pattern from vision-camera's
123
- // FrameHostObject.mm:124-149. Without this, every per-frame
124
- // worklet call to toArrayBuffer() allocates a fresh ~2MB
125
- // vector (1920x1080 NV12 Y-plane) — ~60 MB/s of GC churn at
126
- // 30 fps that defeats the point of having a worklet at all.
127
- // Caching on `runtime.global()` is safe because (a) each
128
- // worklet runtime has its own global, and (b) every call
129
- // overwrites the cached buffer before returning, so there's
130
- // no time-window for cross-worklet data leaks.
131
- static constexpr const char* kCacheKey =
132
- "__stitcherFrameArrayBufferCache";
133
- auto global = runtime.global();
134
- std::shared_ptr<OwningPixelBuffer> owning;
135
-
136
- bool needsAlloc = true;
137
- if (global.hasProperty(runtime, kCacheKey)) {
138
- auto cached = global.getPropertyAsObject(runtime, kCacheKey);
139
- if (cached.isArrayBuffer(runtime)) {
140
- auto cachedBuffer = cached.getArrayBuffer(runtime);
141
- // Hermes JSI exposes the underlying MutableBuffer via the
142
- // shared_ptr the ArrayBuffer was constructed with — but
143
- // there's no public getter once handed to JSI. We retain
144
- // a parallel shared_ptr below via a hidden global slot.
145
- if (cachedBuffer.size(runtime) == bufSize) {
146
- // Size matches — reuse. Pull the parallel
147
- // OwningPixelBuffer ref out of its hidden slot.
148
- static constexpr const char* kRefKey =
149
- "__stitcherFrameArrayBufferCacheRef";
150
- if (global.hasProperty(runtime, kRefKey)) {
151
- // The hidden ref is stored as a HostObject wrapping
152
- // the shared_ptr; pull it back. See alloc path below.
153
- auto refObj = global.getPropertyAsObject(runtime, kRefKey);
154
- if (refObj.isHostObject(runtime)) {
155
- struct RefHolder : facebook::jsi::HostObject {
156
- std::shared_ptr<OwningPixelBuffer> buf;
157
- explicit RefHolder(std::shared_ptr<OwningPixelBuffer> b)
158
- : buf(std::move(b)) {}
159
- };
160
- auto holder =
161
- refObj.getHostObject<RefHolder>(runtime);
162
- if (holder && holder->buf) {
163
- owning = holder->buf;
164
- needsAlloc = false;
165
- }
166
- }
167
- }
168
- }
169
- }
170
- }
171
-
172
- if (needsAlloc) {
173
- owning = std::make_shared<OwningPixelBuffer>(bufSize);
174
- // Store the ArrayBuffer + a parallel ref-holder on global.
175
- // The ArrayBuffer's MutableBuffer is the same `owning`; the
176
- // ref-holder lets us pull `owning` back out on cache hits.
177
- global.setProperty(runtime, kCacheKey,
178
- facebook::jsi::ArrayBuffer(runtime, owning));
179
- struct RefHolder : facebook::jsi::HostObject {
180
- std::shared_ptr<OwningPixelBuffer> buf;
181
- explicit RefHolder(std::shared_ptr<OwningPixelBuffer> b)
182
- : buf(std::move(b)) {}
183
- };
184
- global.setProperty(runtime, "__stitcherFrameArrayBufferCacheRef",
185
- facebook::jsi::Object::createFromHostObject(runtime,
186
- std::make_shared<RefHolder>(owning)));
187
- }
188
-
189
- std::size_t written =
190
- self->_data.pixelReader->copyTo(owning->bytes(), bufSize);
191
- if (written == 0 && bufSize > 0) {
192
- throw JSError(runtime,
193
- "[StitcherFrame] toArrayBuffer() pixel copy failed "
194
- "(reader returned 0 bytes — likely the underlying "
195
- "camera buffer was NULL or unreadable; see native log)");
196
- }
197
-
198
- // Re-fetch the cached ArrayBuffer to return. Cheap (just a
199
- // property lookup); avoids constructing a new jsi::ArrayBuffer
200
- // that wraps the same MutableBuffer (which would be wasteful).
201
- return global.getPropertyAsObject(runtime, kCacheKey)
202
- .getArrayBuffer(runtime);
203
- };
204
- return Function::createFromHostFunction(rt,
205
- PropNameID::forUtf8(rt, "toArrayBuffer"), 0, fn);
206
- }
207
-
208
- // Unknown property — return undefined (matches JS object
209
- // semantics). Worklets accessing arDepth / arAnchors hit this
210
- // path in v0.8.0 (stubbed to undefined; populated in v0.8.1+).
211
- return Value::undefined();
212
- }
213
-
214
- } // namespace retailens
@@ -1,108 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // stitcher_frame_jsi.hpp — shared C++ JSI host object for the v0.8.0
4
- // `StitcherFrame` contract. Compiles on both iOS and Android; each
5
- // platform provides only the PixelBufferReader implementation and
6
- // the construction call site (Obj-C++ on iOS; JNI on Android).
7
- //
8
- // The JSI dispatch logic (`get` / `getPropertyNames`) is identical
9
- // across platforms — the host object exposes the same JS-visible
10
- // surface regardless of frame source, by design of the
11
- // `StitcherFrame` contract.
12
-
13
- #pragma once
14
-
15
- #include <jsi/jsi.h>
16
-
17
- #include <cstdint>
18
- #include <memory>
19
- #include <vector>
20
-
21
- #include "stitcher_frame_data.hpp"
22
-
23
- namespace retailens {
24
-
25
- /// Owning byte buffer that satisfies the `jsi::MutableBuffer`
26
- /// contract. Backs the `ArrayBuffer` returned by
27
- /// `StitcherFrame.toArrayBuffer()`.
28
- ///
29
- /// **Lifetime:** tied to the JSI ArrayBuffer's GC root. The buffer
30
- /// persists until Hermes / JSC garbage-collects the ArrayBuffer
31
- /// (not deterministic with frame timing). To avoid per-frame
32
- /// allocation churn (30 fps × 2 MB = ~60 MB/s in the AR-mode pan
33
- /// case), `toArrayBuffer()` caches a single instance per JSI
34
- /// runtime on `runtime.global()` and reuses it across frames —
35
- /// reallocating only when the requested size changes. Pattern
36
- /// adopted from vision-camera's `FrameHostObject.mm:124-149`.
37
- class OwningPixelBuffer : public facebook::jsi::MutableBuffer {
38
- public:
39
- explicit OwningPixelBuffer(std::size_t sizeBytes)
40
- : _storage(sizeBytes, 0) {}
41
-
42
- // jsi::MutableBuffer interface
43
- uint8_t* data() override { return _storage.data(); }
44
- size_t size() const override { return _storage.size(); }
45
-
46
- /// Direct accessor for the native side to memcpy into before
47
- /// handing the buffer to JSI. Not part of jsi::MutableBuffer.
48
- uint8_t* bytes() { return _storage.data(); }
49
-
50
- private:
51
- std::vector<uint8_t> _storage;
52
- };
53
-
54
- /// v0.8.0 — JSI host object representing one `StitcherFrame`. See
55
- /// `src/stitching/StitcherFrame.ts` for the JS-visible contract.
56
- ///
57
- /// Construct on the worklet runtime's thread, hand to
58
- /// `jsi::Object::createFromHostObject`, dispatch to a registered
59
- /// worklet, then invalidate (typically immediately after dispatch
60
- /// returns — the underlying pixel buffer's lifetime is bound to
61
- /// the calling AR-session callback scope).
62
- class StitcherFrameJsiHostObject
63
- : public facebook::jsi::HostObject,
64
- public std::enable_shared_from_this<StitcherFrameJsiHostObject> {
65
- public:
66
- /// Factory. ALWAYS use this — `shared_from_this()` (called inside
67
- /// `get` for `toArrayBuffer`) requires the instance to be owned
68
- /// by a `shared_ptr` from the moment of construction. A raw
69
- /// `new StitcherFrameJsiHostObject(...)` would throw
70
- /// `std::bad_weak_ptr` on the first `toArrayBuffer()` JSI call.
71
- ///
72
- /// Private constructor + public factory enforces this at the
73
- /// language level; callers can't accidentally construct without
74
- /// `std::make_shared`.
75
- static std::shared_ptr<StitcherFrameJsiHostObject> create(
76
- StitcherFrameData data) {
77
- // `std::make_shared` would require a public ctor; route through
78
- // a tagged-dispatch private constructor instead.
79
- struct EnableMakeShared : StitcherFrameJsiHostObject {
80
- explicit EnableMakeShared(StitcherFrameData d)
81
- : StitcherFrameJsiHostObject(std::move(d)) {}
82
- };
83
- return std::make_shared<EnableMakeShared>(std::move(data));
84
- }
85
-
86
- // jsi::HostObject interface
87
- facebook::jsi::Value get(
88
- facebook::jsi::Runtime& rt,
89
- const facebook::jsi::PropNameID& name) override;
90
- std::vector<facebook::jsi::PropNameID> getPropertyNames(
91
- facebook::jsi::Runtime& rt) override;
92
-
93
- /// Mark the host object's backing data as no longer accessible.
94
- /// Subsequent JSI reads of valid-required properties throw.
95
- /// Releases the pixel reader (and its underlying ARFrame /
96
- /// ArImage retain) immediately. Idempotent.
97
- void invalidate();
98
-
99
- bool isValid() const { return _isValid; }
100
-
101
- private:
102
- explicit StitcherFrameJsiHostObject(StitcherFrameData data);
103
-
104
- StitcherFrameData _data;
105
- bool _isValid;
106
- };
107
-
108
- } // namespace retailens