react-native-image-stitcher 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +180 -1
- package/android/build.gradle +35 -1
- package/android/src/main/cpp/CMakeLists.txt +64 -2
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +227 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +30 -11
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +4 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +78 -3
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +103 -0
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +256 -0
- package/android/src/main/java/io/imagestitcher/rn/TransferredNV21.kt +100 -0
- package/cpp/stitcher_frame_data.hpp +141 -0
- package/cpp/stitcher_frame_jsi.cpp +214 -0
- package/cpp/stitcher_frame_jsi.hpp +108 -0
- package/cpp/stitcher_proxy_jsi.cpp +109 -0
- package/cpp/stitcher_proxy_jsi.hpp +46 -0
- package/cpp/stitcher_worklet_dispatch.cpp +103 -0
- package/cpp/stitcher_worklet_dispatch.hpp +71 -0
- package/cpp/stitcher_worklet_registry.cpp +81 -0
- package/cpp/stitcher_worklet_registry.hpp +136 -0
- package/dist/camera/Camera.d.ts +62 -12
- package/dist/camera/Camera.js +30 -15
- package/dist/index.d.ts +2 -0
- package/dist/index.js +11 -1
- package/dist/stitching/StitcherFrame.d.ts +170 -0
- package/dist/stitching/StitcherFrame.js +4 -0
- package/dist/stitching/StitcherWorkletRegistry.d.ts +117 -0
- package/dist/stitching/StitcherWorkletRegistry.js +78 -0
- package/dist/stitching/ensureStitcherProxyInstalled.d.ts +8 -0
- package/dist/stitching/ensureStitcherProxyInstalled.js +81 -0
- package/dist/stitching/useFrameProcessor.d.ts +119 -0
- package/dist/stitching/useFrameProcessor.js +196 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +46 -10
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +128 -0
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +313 -0
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +60 -0
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +214 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +42 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +103 -0
- package/package.json +1 -1
- package/src/camera/Camera.tsx +93 -28
- package/src/index.ts +16 -0
- package/src/stitching/StitcherFrame.ts +197 -0
- package/src/stitching/StitcherWorkletRegistry.ts +156 -0
- package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +176 -0
- package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +94 -0
- package/src/stitching/ensureStitcherProxyInstalled.ts +141 -0
- package/src/stitching/useFrameProcessor.ts +226 -0
|
@@ -287,8 +287,20 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
287
287
|
// contract was already in place for Phase 4.
|
|
288
288
|
appendPose(camera, frame.timestamp)
|
|
289
289
|
|
|
290
|
-
// Forward to the incremental stitcher if engaged
|
|
291
|
-
|
|
290
|
+
// Forward to the incremental stitcher if engaged, OR if any
|
|
291
|
+
// host worklets are registered (v0.8.0 Phase 4b.iii). iOS'
|
|
292
|
+
// `RNSARWorkletRuntime.dispatchFrame:pose:` fires on every
|
|
293
|
+
// AR frame regardless of capture state; Android needs the
|
|
294
|
+
// same semantic so host worklets see the AR-mode preview
|
|
295
|
+
// stream, not just capture frames.
|
|
296
|
+
//
|
|
297
|
+
// `hasHostWorklets()` is a microsecond atomic-read on the
|
|
298
|
+
// native registry — cheap enough to hit per frame. When
|
|
299
|
+
// no host worklets are registered AND no capture is active,
|
|
300
|
+
// the entire forwardToIncremental branch (including the
|
|
301
|
+
// ~3-5ms NV21 pack) is skipped — same cost envelope as
|
|
302
|
+
// before Phase 4b.iii.
|
|
303
|
+
if (ingestActive || StitcherWorkletRuntime.hasHostWorklets()) {
|
|
292
304
|
forwardToIncremental(frame, camera)
|
|
293
305
|
}
|
|
294
306
|
|
|
@@ -514,6 +526,23 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
514
526
|
// (Was: eager JPEG encode for non-batch-keyframe modes,
|
|
515
527
|
// written to `tmpJpegFile`, passed as `legacyJpegPath`.
|
|
516
528
|
// See the v0.3 / F8.6 entries in CHANGELOG.md.)
|
|
529
|
+
//
|
|
530
|
+
// v0.8.0 Phase 3c — route through the worklet runtime's
|
|
531
|
+
// `runFirstParty` indirection. The lambda body is the
|
|
532
|
+
// unchanged engine ingest call; the indirection sets up
|
|
533
|
+
// the seam where Phase 4 will fan out to host worklets
|
|
534
|
+
// without touching this first-party path. Synchronous
|
|
535
|
+
// invocation preserves the ARCore Image ownership contract
|
|
536
|
+
// — the engine consumes the TransferredNV21 inside the
|
|
537
|
+
// lambda before ARCore recycles the Image.
|
|
538
|
+
StitcherWorkletRuntime.installIfNeeded()
|
|
539
|
+
// v0.8.0 Phase 4b.iii — only run first-party stitching when
|
|
540
|
+
// the host has actively engaged capture (`setIncrementalIngestionActive(true)`).
|
|
541
|
+
// The host-worklet dispatch below runs regardless, so AR-mode
|
|
542
|
+
// preview frames stream through registered host worklets even
|
|
543
|
+
// before/after capture.
|
|
544
|
+
if (ingestActive) {
|
|
545
|
+
StitcherWorkletRuntime.runFirstParty {
|
|
517
546
|
module.ingestFromARCameraView(
|
|
518
547
|
tx = tArr[0].toDouble(),
|
|
519
548
|
ty = tArr[1].toDouble(),
|
|
@@ -538,7 +567,17 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
538
567
|
legacyJpegPath = null,
|
|
539
568
|
// F8.6 — pixel-data path for live engines. Batch-
|
|
540
569
|
// keyframe mode ignores these (bails earlier).
|
|
541
|
-
|
|
570
|
+
//
|
|
571
|
+
// v0.10.0 audit #4A — wrap `packed.nv21` in
|
|
572
|
+
// TransferredNV21 so ownership is enforced at runtime.
|
|
573
|
+
// The AR caller passes the SAME `packed.nv21` array as
|
|
574
|
+
// both `grayData` (sync, gate-eval read) and
|
|
575
|
+
// `nv21PixelData` (async, engine ingest). Today no race
|
|
576
|
+
// because grayData is consumed inside evaluateWithFrame
|
|
577
|
+
// before workScope.launch fires; the wrapper makes a
|
|
578
|
+
// future refactor that reorders consumption fail loudly
|
|
579
|
+
// instead of silently corrupting frames.
|
|
580
|
+
nv21PixelData = TransferredNV21(packed.nv21),
|
|
542
581
|
nv21PixelWidth = packed.width,
|
|
543
582
|
nv21PixelHeight = packed.height,
|
|
544
583
|
onAccept = { targetPath ->
|
|
@@ -555,6 +594,42 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
555
594
|
) != null
|
|
556
595
|
},
|
|
557
596
|
)
|
|
597
|
+
} // closes StitcherWorkletRuntime.runFirstParty { … } (v0.8.0 Phase 3c)
|
|
598
|
+
} // closes `if (ingestActive)` (v0.8.0 Phase 4b.iii)
|
|
599
|
+
|
|
600
|
+
// ── v0.8.0 Phase 4b.iii — host-worklet fan-out ─────────────
|
|
601
|
+
//
|
|
602
|
+
// Dispatch the AR frame to every host worklet registered via
|
|
603
|
+
// `globalThis.__stitcherProxy.install(workletFn)` (the
|
|
604
|
+
// `useFrameProcessor` hook's AR-mode path). The native side
|
|
605
|
+
// fast-path early-exits when the registry is empty (~ns
|
|
606
|
+
// cost), so this call is free for first-party-only deployments.
|
|
607
|
+
//
|
|
608
|
+
// Map the trackingState back to the JS-visible string set.
|
|
609
|
+
// `RNSARSession.TRACKING_*` are int codes; we re-derive the
|
|
610
|
+
// string here instead of plumbing it through. (Could be
|
|
611
|
+
// refactored into a helper if/when other call sites need
|
|
612
|
+
// it.)
|
|
613
|
+
val trackingStateStr = when (camera.trackingState) {
|
|
614
|
+
TrackingState.TRACKING -> "normal"
|
|
615
|
+
TrackingState.PAUSED -> "limited"
|
|
616
|
+
TrackingState.STOPPED -> "notAvailable"
|
|
617
|
+
else -> ""
|
|
618
|
+
}
|
|
619
|
+
StitcherWorkletRuntime.dispatchToHostWorklets(
|
|
620
|
+
nv21Bytes = packed.nv21,
|
|
621
|
+
width = packed.width,
|
|
622
|
+
height = packed.height,
|
|
623
|
+
qx = qarr[0].toDouble(),
|
|
624
|
+
qy = qarr[1].toDouble(),
|
|
625
|
+
qz = qarr[2].toDouble(),
|
|
626
|
+
qw = qarr[3].toDouble(),
|
|
627
|
+
tx = tArr[0].toDouble(),
|
|
628
|
+
ty = tArr[1].toDouble(),
|
|
629
|
+
tz = tArr[2].toDouble(),
|
|
630
|
+
timestampNs = frame.timestamp.toDouble(),
|
|
631
|
+
trackingState = trackingStateStr,
|
|
632
|
+
)
|
|
558
633
|
}
|
|
559
634
|
|
|
560
635
|
private fun applyDisplayGeometry() {
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
import android.util.Log
|
|
5
|
+
import com.facebook.react.bridge.ReactApplicationContext
|
|
6
|
+
import com.facebook.react.bridge.ReactContextBaseJavaModule
|
|
7
|
+
import com.facebook.react.bridge.ReactMethod
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* v0.8.0 Phase 4b.ii — Android-side JSI installer for the host
|
|
11
|
+
* worklet proxy. Mirror of iOS' `StitcherJsiInstaller`.
|
|
12
|
+
*
|
|
13
|
+
* The module exposes one synchronous method, `install()`, which JS
|
|
14
|
+
* calls once at lib bootstrap (via the
|
|
15
|
+
* `ensureStitcherProxyInstalled` helper in
|
|
16
|
+
* `src/stitching/ensureStitcherProxyInstalled.ts`). We reach into
|
|
17
|
+
* the main JS runtime via `ReactApplicationContext.getJavaScriptContextHolder().get()`
|
|
18
|
+
* — the canonical bridgeless-compatible accessor in modern RN
|
|
19
|
+
* (worklets-core's `WorkletsModule` uses the same pattern, verified
|
|
20
|
+
* working on RN 0.84.1 + new arch + Hermes).
|
|
21
|
+
*
|
|
22
|
+
* The native `nativeInstall(jsiRuntimeRef)` JNI then casts the long
|
|
23
|
+
* back to a `jsi::Runtime*` and calls into the shared C++
|
|
24
|
+
* `retailens::installStitcherProxy(runtime)` (in
|
|
25
|
+
* `cpp/stitcher_proxy_jsi.{hpp,cpp}`). Identical destination on
|
|
26
|
+
* both platforms — `globalThis.__stitcherProxy` exposes the same
|
|
27
|
+
* `install` / `uninstall` / `count` host functions.
|
|
28
|
+
*
|
|
29
|
+
* ## Returning `Boolean` (not `Promise`) from a sync method
|
|
30
|
+
*
|
|
31
|
+
* `isBlockingSynchronousMethod = true` + `Boolean` return is the
|
|
32
|
+
* documented pattern for "I'm doing one-shot native setup that
|
|
33
|
+
* needs to complete before the next JS line runs." Same shape as
|
|
34
|
+
* `WorkletsModule.install()`.
|
|
35
|
+
*
|
|
36
|
+
* ## What we DON'T do here (Phase 4b.ii follow-up)
|
|
37
|
+
*
|
|
38
|
+
* Phase 4b.ii's MVP installs the proxy ONLY. Host worklets that
|
|
39
|
+
* register through `__stitcherProxy.install` land in the native
|
|
40
|
+
* `retailens::StitcherWorkletRegistry`. Per-frame fan-out from
|
|
41
|
+
* Android's `StitcherWorkletRuntime` is a separate piece of work
|
|
42
|
+
* (Phase 4b.ii follow-up) — needs the Kotlin↔JNI bridge that
|
|
43
|
+
* constructs a `StitcherFrameJsiHostObject` from an `ArImage` +
|
|
44
|
+
* pose and posts it through a worklet runtime. Until that lands,
|
|
45
|
+
* Android-registered worklets behave exactly like iOS-registered
|
|
46
|
+
* worklets BEFORE Phase 4b.i: they exist in the registry but
|
|
47
|
+
* aren't invoked.
|
|
48
|
+
*
|
|
49
|
+
* The proxy install itself is still useful as a foundation —
|
|
50
|
+
* verifies the JNI handshake works, exercises the bridgeless
|
|
51
|
+
* runtime accessor, and gives us a `count()` smoke test for the
|
|
52
|
+
* device verification step.
|
|
53
|
+
*/
|
|
54
|
+
class StitcherJsiInstallerModule(
|
|
55
|
+
private val reactContext: ReactApplicationContext,
|
|
56
|
+
) : ReactContextBaseJavaModule(reactContext) {
|
|
57
|
+
override fun getName(): String = NAME
|
|
58
|
+
|
|
59
|
+
@ReactMethod(isBlockingSynchronousMethod = true)
|
|
60
|
+
fun install(): Boolean {
|
|
61
|
+
return try {
|
|
62
|
+
// `getJavaScriptContextHolder().get()` returns a raw
|
|
63
|
+
// `jsi::Runtime*` boxed as `Long`. Same accessor
|
|
64
|
+
// worklets-core's `WorkletsModule.install()` uses;
|
|
65
|
+
// documented to work in both legacy + bridgeless modes
|
|
66
|
+
// on RN 0.71+.
|
|
67
|
+
val holder = reactContext.javaScriptContextHolder
|
|
68
|
+
if (holder == null) {
|
|
69
|
+
Log.e(TAG, "getJavaScriptContextHolder() returned null; runtime unreachable")
|
|
70
|
+
return false
|
|
71
|
+
}
|
|
72
|
+
val runtimeRef = holder.get()
|
|
73
|
+
if (runtimeRef == 0L) {
|
|
74
|
+
Log.e(TAG, "JavaScriptContextHolder.get() returned 0; runtime not initialized yet")
|
|
75
|
+
return false
|
|
76
|
+
}
|
|
77
|
+
val ok = nativeInstall(runtimeRef)
|
|
78
|
+
if (!ok) {
|
|
79
|
+
Log.e(TAG, "nativeInstall(runtimeRef=$runtimeRef) returned false")
|
|
80
|
+
}
|
|
81
|
+
ok
|
|
82
|
+
} catch (t: Throwable) {
|
|
83
|
+
Log.e(TAG, "install() threw — falling back to JS-side registry", t)
|
|
84
|
+
false
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
private external fun nativeInstall(jsiRuntimeRef: Long): Boolean
|
|
89
|
+
|
|
90
|
+
companion object {
|
|
91
|
+
const val NAME = "StitcherJsiInstaller"
|
|
92
|
+
private const val TAG = "StitcherJsiInstaller"
|
|
93
|
+
|
|
94
|
+
init {
|
|
95
|
+
// The Phase 3a JNI shim (`libimage_stitcher.so`) absorbed
|
|
96
|
+
// the JSI-install JNI binding from Phase 4b.ii. Loading
|
|
97
|
+
// it once is enough — Android's loader deduplicates,
|
|
98
|
+
// so even if `IncrementalStitcher.kt`'s init block
|
|
99
|
+
// already loaded the lib, calling again is a cheap no-op.
|
|
100
|
+
System.loadLibrary("image_stitcher")
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
import android.os.HandlerThread
|
|
5
|
+
import android.util.Log
|
|
6
|
+
import java.util.concurrent.atomic.AtomicBoolean
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* v0.8.0 Phase 3b — Android twin of iOS' `RNSARWorkletRuntime`.
|
|
10
|
+
* Owns the per-AR-frame worklet runtime + the thread it dispatches
|
|
11
|
+
* on. Symmetric API shape to the iOS class so the cross-platform
|
|
12
|
+
* dispatch story (Phase 3c) lands in lockstep.
|
|
13
|
+
*
|
|
14
|
+
* ## Phase 3b scope (this commit)
|
|
15
|
+
*
|
|
16
|
+
* - Singleton accessor + lifecycle (installIfNeeded / isInstalled).
|
|
17
|
+
* - Dedicated `HandlerThread` for worklet dispatch — keeps work off
|
|
18
|
+
* the GLSurfaceView GL render thread (audit caveat #4 from the
|
|
19
|
+
* Phase-0 audit).
|
|
20
|
+
* - `dispatchFrame()` stub — Phase 3c will fill in:
|
|
21
|
+
* 1. Build `StitcherFrameHostObject` from ARCore Frame + pose
|
|
22
|
+
* (via the shared C++ JSI host object now linked into
|
|
23
|
+
* `libimage_stitcher.so` post-Phase-3a).
|
|
24
|
+
* 2. Run first-party stitching synchronously on the caller
|
|
25
|
+
* thread (`onDrawFrame`'s GL thread, today).
|
|
26
|
+
* 3. If host worklets are registered, dispatch the host object
|
|
27
|
+
* onto this runtime's `HandlerThread` + invoke each worklet
|
|
28
|
+
* via JNI → `RNWorklet::WorkletInvoker::call`.
|
|
29
|
+
* 4. Invalidate the host object after all worklets return.
|
|
30
|
+
*
|
|
31
|
+
* ## Worklet-runtime construction model
|
|
32
|
+
*
|
|
33
|
+
* Unlike iOS (where the lib's `.mm` directly `std::make_shared`s
|
|
34
|
+
* a `RNWorklet::JsiWorkletContext`), Android can't construct the
|
|
35
|
+
* context purely from native C++ without JNI plumbing. Phase 3c
|
|
36
|
+
* will choose between two paths:
|
|
37
|
+
*
|
|
38
|
+
* - **Option A:** JS-side code calls
|
|
39
|
+
* `Worklets.createContext("stitcher.ar")` at AR-mode start;
|
|
40
|
+
* hands the resulting context pointer to this Kotlin class via
|
|
41
|
+
* a small JSI plugin. Minimal new JNI. **Phase 3b's
|
|
42
|
+
* HandlerThread becomes dead code** under this option — the
|
|
43
|
+
* JS-side `Worklets.createContext` picks its own thread. We'd
|
|
44
|
+
* need to remove the HandlerThread + change `installIfNeeded`
|
|
45
|
+
* into a no-op until a `setContextHandle(Long)` setter lands.
|
|
46
|
+
* - **Option B:** Direct JNI binding to worklets-core's C++
|
|
47
|
+
* constructor. More native code but no JS dependency at runtime.
|
|
48
|
+
* **Phase 3b's HandlerThread is exactly the right scaffold**
|
|
49
|
+
* under this option — its looper becomes the JsiWorkletContext's
|
|
50
|
+
* `workletCallInvoker` target.
|
|
51
|
+
*
|
|
52
|
+
* **Phase 3b assumption: Option B is the more likely path.** The
|
|
53
|
+
* scaffolding below (HandlerThread + serial dispatch) fits Option
|
|
54
|
+
* B; if Phase 3c picks Option A instead, the HandlerThread becomes
|
|
55
|
+
* unused and Phase 3c will refactor accordingly.
|
|
56
|
+
*
|
|
57
|
+
* @see [RNSARWorkletRuntime] iOS equivalent
|
|
58
|
+
* @see docs/plans/handoff/2026-05-26-v0.8.0-phase-0-audit.md
|
|
59
|
+
* worklets-core API rationale (Audit 2: `JsiWorkletContext`).
|
|
60
|
+
* @see docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md
|
|
61
|
+
* Phase 3c implementation plan.
|
|
62
|
+
*/
|
|
63
|
+
object StitcherWorkletRuntime {
|
|
64
|
+
private const val TAG = "StitcherWorkletRuntime"
|
|
65
|
+
|
|
66
|
+
/// Single-flight install guard. `compareAndSet` makes the
|
|
67
|
+
/// runtime construction race-safe across concurrent first-mount
|
|
68
|
+
/// calls from multiple `<Camera>` instances.
|
|
69
|
+
private val installed = AtomicBoolean(false)
|
|
70
|
+
|
|
71
|
+
/// Dedicated dispatch thread. Constructed eagerly so we can
|
|
72
|
+
/// validate the thread starts cleanly during `installIfNeeded`.
|
|
73
|
+
/// Off the GLSurfaceView GL render thread (audit caveat #4)
|
|
74
|
+
/// + off the main thread. Phase 3c will configure the worklet
|
|
75
|
+
/// context's `workletCallInvoker` to post onto this thread's
|
|
76
|
+
/// looper.
|
|
77
|
+
private val dispatchThread: HandlerThread by lazy {
|
|
78
|
+
HandlerThread("io.imagestitcher.ar-worklet-runtime").apply { start() }
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// Construct the underlying worklet context if not yet installed.
|
|
82
|
+
/// Idempotent — repeated calls are no-ops.
|
|
83
|
+
///
|
|
84
|
+
/// Phase 3b: starts the dispatch thread; no JsiWorkletContext
|
|
85
|
+
/// construction yet (deferred to Phase 3c).
|
|
86
|
+
/// Phase 3c: also wires the JsiWorkletContext + binds it to the
|
|
87
|
+
/// dispatch thread's looper.
|
|
88
|
+
@JvmStatic
|
|
89
|
+
fun installIfNeeded() {
|
|
90
|
+
if (!installed.compareAndSet(false, true)) {
|
|
91
|
+
return
|
|
92
|
+
}
|
|
93
|
+
// Force the lazy `dispatchThread` to initialise. If the
|
|
94
|
+
// OS denies thread creation (extreme memory pressure on a
|
|
95
|
+
// budget device), `HandlerThread.start()` won't throw but
|
|
96
|
+
// the looper won't be available — the Phase 3c dispatch
|
|
97
|
+
// logic will need to defend against that. For Phase 3b
|
|
98
|
+
// we only care that this method returns without throwing.
|
|
99
|
+
//
|
|
100
|
+
// Log `Thread.id` (Java-side monotonic, always non-zero) —
|
|
101
|
+
// NOT `HandlerThread.threadId` (Linux tid set after first
|
|
102
|
+
// Looper-prepared message; reading from this caller thread
|
|
103
|
+
// immediately after .start() returns -1 until scheduled).
|
|
104
|
+
val javaThreadId = dispatchThread.id
|
|
105
|
+
Log.i(TAG, "installed runtime; dispatch java-thread id=$javaThreadId")
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/// Diagnostics + tests. Returns `true` after a successful
|
|
109
|
+
/// `installIfNeeded()`.
|
|
110
|
+
@JvmStatic
|
|
111
|
+
fun isInstalled(): Boolean = installed.get()
|
|
112
|
+
|
|
113
|
+
/// v0.8.0 Phase 3c — first-party stitching dispatch. Invokes
|
|
114
|
+
/// the supplied block synchronously on the caller thread
|
|
115
|
+
/// (`onDrawFrame`'s GL render thread today).
|
|
116
|
+
///
|
|
117
|
+
/// Phase 3c minimum-viable: this is the closure-based equivalent
|
|
118
|
+
/// of iOS' first-party callback. The block is the original
|
|
119
|
+
/// `module.ingestFromARCameraView(...)` call site moved
|
|
120
|
+
/// verbatim into a lambda — no behaviour change, just an
|
|
121
|
+
/// indirection so Phase 4 can interpose host-worklet fanout
|
|
122
|
+
/// without touching the engine ingest path.
|
|
123
|
+
///
|
|
124
|
+
/// **Why synchronous + on the caller thread:** the engine's
|
|
125
|
+
/// `ingestFromARCameraView` takes ownership of the ARCore
|
|
126
|
+
/// `Image`-derived NV21 buffer (via the v0.10.0 `TransferredNV21`
|
|
127
|
+
/// wrapper). ARCore's `Image.close()` happens after this call
|
|
128
|
+
/// returns, so the consumer must finish reading the bytes before
|
|
129
|
+
/// we return — exactly what synchronous block invocation
|
|
130
|
+
/// provides. Phase 4 will copy the buffer for off-thread
|
|
131
|
+
/// access in host worklets; Phase 3c keeps the sync contract.
|
|
132
|
+
///
|
|
133
|
+
/// If `installIfNeeded()` hasn't been called yet, the block
|
|
134
|
+
/// still runs (no-op on the registry side). Defensive — the
|
|
135
|
+
/// caller may call this method before `installIfNeeded` is
|
|
136
|
+
/// wired up.
|
|
137
|
+
@JvmStatic
|
|
138
|
+
fun runFirstParty(block: () -> Unit) {
|
|
139
|
+
// Synchronous invocation — Phase 4 will extend this to also
|
|
140
|
+
// post the registered host worklets onto `dispatchThread`.
|
|
141
|
+
// Not `inline`: Phase 4 will need to read `dispatchThread`
|
|
142
|
+
// (private) from inside this function body, and Kotlin's
|
|
143
|
+
// inline functions can't access private members from call
|
|
144
|
+
// sites outside the declaring class. Per-frame lambda
|
|
145
|
+
// alloc is ~ns and the alternative (callers passing a
|
|
146
|
+
// method reference) doesn't materially change cost.
|
|
147
|
+
block()
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/// v0.8.0 Phase 4b.iii — fan out one AR frame to every host
|
|
151
|
+
/// worklet registered in the shared C++ `StitcherWorkletRegistry`
|
|
152
|
+
/// (populated from JS via `__stitcherProxy.install(workletFn)`).
|
|
153
|
+
///
|
|
154
|
+
/// Called from `RNSARCameraView.onDrawFrame` immediately after
|
|
155
|
+
/// `runFirstParty { ... }` returns, with the already-extracted
|
|
156
|
+
/// AR frame data (pose + NV21 bytes + dimensions + tracking
|
|
157
|
+
/// state).
|
|
158
|
+
///
|
|
159
|
+
/// **Fast-path:** the native side queries the registry's count
|
|
160
|
+
/// FIRST and returns before copying any bytes when no host
|
|
161
|
+
/// worklets are registered. In the common first-party-only
|
|
162
|
+
/// deployment, this method costs one JNI call + one C++ atomic
|
|
163
|
+
/// read per frame — negligible.
|
|
164
|
+
///
|
|
165
|
+
/// **When host worklets ARE registered:** the JNI layer copies
|
|
166
|
+
/// the NV21 byte array into an owned C++ `std::vector` (so the
|
|
167
|
+
/// async dispatch can outlive ARCore's `Image.close()` scope),
|
|
168
|
+
/// builds a `StitcherFrameJsiHostObject`, and posts a lambda
|
|
169
|
+
/// onto worklets-core's default `JsiWorkletContext`'s worklet
|
|
170
|
+
/// thread. The lambda iterates the registry's
|
|
171
|
+
/// `WorkletInvoker`s, calls each with the JSI host object as
|
|
172
|
+
/// its argument, and invalidates the host object after the
|
|
173
|
+
/// last invoker returns. Per-worklet failure isolation: one
|
|
174
|
+
/// host worklet throwing does NOT stop the lib's stitching or
|
|
175
|
+
/// the other host worklets.
|
|
176
|
+
///
|
|
177
|
+
/// **Threading:** this method returns synchronously on the
|
|
178
|
+
/// caller's thread. The actual worklet invocations happen
|
|
179
|
+
/// asynchronously on the worklets-core thread; the caller does
|
|
180
|
+
/// NOT block on them.
|
|
181
|
+
///
|
|
182
|
+
/// **Caller-thread contract:** the caller (`RNSARCameraView`'s
|
|
183
|
+
/// `onDrawFrame`) MUST have already invoked `runFirstParty`
|
|
184
|
+
/// before calling this method. The first-party stitching
|
|
185
|
+
/// path holds the synchronous ARCore Image consumption
|
|
186
|
+
/// contract; the host-worklet dispatch does not.
|
|
187
|
+
///
|
|
188
|
+
/// @param nv21Bytes Pre-packed NV21 byte array. COPIED
|
|
189
|
+
/// into a native owned buffer; caller can
|
|
190
|
+
/// release the reference after return.
|
|
191
|
+
/// @param width Camera image width (pixels).
|
|
192
|
+
/// @param height Camera image height (pixels).
|
|
193
|
+
/// @param qx,qy,qz,qw Pose rotation quaternion (unit length).
|
|
194
|
+
/// @param tx,ty,tz Pose translation (metres, world coords).
|
|
195
|
+
/// @param timestampNs Frame timestamp in nanoseconds.
|
|
196
|
+
/// @param trackingState One of "" / "notAvailable" / "limited"
|
|
197
|
+
/// / "normal". Empty string ⇒ JS-side
|
|
198
|
+
/// `arTrackingState` is `undefined`.
|
|
199
|
+
@JvmStatic
|
|
200
|
+
fun dispatchToHostWorklets(
|
|
201
|
+
nv21Bytes: ByteArray,
|
|
202
|
+
width: Int,
|
|
203
|
+
height: Int,
|
|
204
|
+
qx: Double, qy: Double, qz: Double, qw: Double,
|
|
205
|
+
tx: Double, ty: Double, tz: Double,
|
|
206
|
+
timestampNs: Double,
|
|
207
|
+
trackingState: String,
|
|
208
|
+
) {
|
|
209
|
+
if (!installed.get()) return
|
|
210
|
+
nativeDispatchToHostWorklets(
|
|
211
|
+
nv21Bytes, width, height,
|
|
212
|
+
qx, qy, qz, qw,
|
|
213
|
+
tx, ty, tz,
|
|
214
|
+
timestampNs, trackingState,
|
|
215
|
+
)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/// v0.8.0 Phase 4b.iii — number of registered host worklets.
|
|
219
|
+
/// Cheap (microsecond) call into the native registry. Used by
|
|
220
|
+
/// `RNSARCameraView.onDrawFrame` to gate the per-frame
|
|
221
|
+
/// NV21-pack + dispatch path: when no worklets are registered
|
|
222
|
+
/// AND no capture is active, the entire `forwardToIncremental`
|
|
223
|
+
/// branch can be skipped, saving the ~3-5ms NV21 pack cost per
|
|
224
|
+
/// idle preview frame.
|
|
225
|
+
@JvmStatic
|
|
226
|
+
fun hasHostWorklets(): Boolean {
|
|
227
|
+
if (!installed.get()) return false
|
|
228
|
+
return nativeRegistryCount() > 0
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
@JvmStatic
|
|
232
|
+
private external fun nativeRegistryCount(): Int
|
|
233
|
+
|
|
234
|
+
/// JNI binding: `android/src/main/cpp/stitcher_jsi_install_jni.cpp`'s
|
|
235
|
+
/// `nativeDispatchToHostWorklets`. Fast-path early-exit lives
|
|
236
|
+
/// inside the native function — see its docstring.
|
|
237
|
+
@JvmStatic
|
|
238
|
+
private external fun nativeDispatchToHostWorklets(
|
|
239
|
+
nv21Bytes: ByteArray,
|
|
240
|
+
width: Int,
|
|
241
|
+
height: Int,
|
|
242
|
+
qx: Double, qy: Double, qz: Double, qw: Double,
|
|
243
|
+
tx: Double, ty: Double, tz: Double,
|
|
244
|
+
timestampNs: Double,
|
|
245
|
+
trackingState: String,
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
init {
|
|
249
|
+
// The JSI install module (`StitcherJsiInstallerModule`)
|
|
250
|
+
// already loads `libimage_stitcher` at class load. We
|
|
251
|
+
// load it again here defensively in case
|
|
252
|
+
// `StitcherWorkletRuntime` is referenced before the install
|
|
253
|
+
// module — `System.loadLibrary` is idempotent.
|
|
254
|
+
System.loadLibrary("image_stitcher")
|
|
255
|
+
}
|
|
256
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* v0.10.0 (audit #4A) — single-use NV21 byte-array handle that
|
|
6
|
+
* enforces the engine's pixel-data ownership contract at runtime.
|
|
7
|
+
*
|
|
8
|
+
* ## Why this exists
|
|
9
|
+
*
|
|
10
|
+
* `IncrementalStitcher.ingestFromARCameraView` accepts an
|
|
11
|
+
* `nv21PixelData` parameter that the engine retains for ~50 ms
|
|
12
|
+
* after the producer thread returns (until the `workScope`
|
|
13
|
+
* coroutine consumes it). The documented contract is
|
|
14
|
+
* "callers MUST treat the array as transferred — do not mutate it
|
|
15
|
+
* or return it to a buffer pool after calling this method."
|
|
16
|
+
*
|
|
17
|
+
* The v0.10.0 audit (`docs/plans/handoff/2026-05-26-autonomous-run-handoff.md`
|
|
18
|
+
* finding #4A) noted this is by-convention only. The current AR
|
|
19
|
+
* caller (`RNSARCameraView`) passes the same `packed.nv21` array
|
|
20
|
+
* as BOTH `grayData` (consumed synchronously inside the gate)
|
|
21
|
+
* AND `nv21PixelData` (consumed asynchronously). Today no race
|
|
22
|
+
* because the sync read finishes before the async coroutine reads,
|
|
23
|
+
* but a future refactor that reorders consumption would silently
|
|
24
|
+
* corrupt frames.
|
|
25
|
+
*
|
|
26
|
+
* Wrapping the bytes in `TransferredNV21` turns the documentation
|
|
27
|
+
* contract into a runtime contract: callers can only extract the
|
|
28
|
+
* bytes once via `takeOnce()`; the second call throws. The
|
|
29
|
+
* misuse is caught at the call site, not at the engine.
|
|
30
|
+
*
|
|
31
|
+
* ## Cost
|
|
32
|
+
*
|
|
33
|
+
* Construction: tens of ns (one heap allocation for the wrapper +
|
|
34
|
+
* one volatile write of the bytes reference). `takeOnce()`: tens
|
|
35
|
+
* of ns (one synchronized read + one null-out). Negligible vs the
|
|
36
|
+
* underlying NV21 array's KB-scale memory footprint and the
|
|
37
|
+
* ms-scale frame-processing cost — but not a free pointer hop.
|
|
38
|
+
*
|
|
39
|
+
* ## Thread-safety
|
|
40
|
+
*
|
|
41
|
+
* `takeOnce()` and `available` are `synchronized` on the wrapper
|
|
42
|
+
* itself. Producers should still extract on a single thread (the
|
|
43
|
+
* frame producer); the synchronization defends against the
|
|
44
|
+
* pathological case where two threads race to extract.
|
|
45
|
+
*/
|
|
46
|
+
class TransferredNV21(bytes: ByteArray) {
|
|
47
|
+
init {
|
|
48
|
+
// Empty arrays would propagate as "0 bytes of pixel data with
|
|
49
|
+
// a non-zero width/height" downstream and crash inside the
|
|
50
|
+
// C++ ingest with a far less actionable error. Catch at
|
|
51
|
+
// construction. Critic-finding [MAJOR][B].
|
|
52
|
+
require(bytes.isNotEmpty()) {
|
|
53
|
+
"TransferredNV21 requires a non-empty byte array " +
|
|
54
|
+
"(received zero-length)"
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
@Volatile
|
|
59
|
+
private var bytes: ByteArray? = bytes
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Take the wrapped bytes. Throws on second call.
|
|
63
|
+
*
|
|
64
|
+
* Consumers should call this exactly once — typically once per
|
|
65
|
+
* frame, on the producer thread, immediately before handing
|
|
66
|
+
* the bytes to the async work queue:
|
|
67
|
+
*
|
|
68
|
+
* ```kotlin
|
|
69
|
+
* val pixelBytes: ByteArray? = if (hasPixelData) nv21PixelData!!.takeOnce() else null
|
|
70
|
+
* workScope.launch {
|
|
71
|
+
* // pixelBytes is captured by value; no race.
|
|
72
|
+
* engine.addFramePixelData(nv21 = pixelBytes!!, ...)
|
|
73
|
+
* }
|
|
74
|
+
* ```
|
|
75
|
+
*
|
|
76
|
+
* Concurrency note: `@Volatile` on the bytes field plus the
|
|
77
|
+
* `synchronized(this)` block here together guarantee both
|
|
78
|
+
* visibility AND atomicity across threads. The `@Volatile` is
|
|
79
|
+
* defensive for any future non-synchronized read; today every
|
|
80
|
+
* accessor goes through the synchronized block.
|
|
81
|
+
*/
|
|
82
|
+
fun takeOnce(): ByteArray = synchronized(this) {
|
|
83
|
+
val b = bytes ?: error(
|
|
84
|
+
"TransferredNV21.takeOnce() called twice — bytes already transferred. " +
|
|
85
|
+
"Check that you're not passing the same TransferredNV21 instance to " +
|
|
86
|
+
"two consumers (e.g., a sync gate-eval call AND an async workScope.launch)."
|
|
87
|
+
)
|
|
88
|
+
bytes = null
|
|
89
|
+
b
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Note: an `available` property was considered and removed in
|
|
93
|
+
// pre-merge review (critic-finding [MAJOR][B]). Any
|
|
94
|
+
// `if (handle.available) handle.takeOnce()` pattern is
|
|
95
|
+
// inherently TOCTOU-racy — another thread could win the
|
|
96
|
+
// takeOnce() between the check and the use. Consumers should
|
|
97
|
+
// call `takeOnce()` directly and catch the `IllegalStateException`
|
|
98
|
+
// if they need recovery semantics. No internal caller used
|
|
99
|
+
// `available`; YAGNI removed it.
|
|
100
|
+
}
|