react-native-image-stitcher 0.16.2 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +154 -0
- package/RNImageStitcher.podspec +26 -1
- package/android/build.gradle +20 -0
- package/android/src/main/cpp/CMakeLists.txt +46 -3
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +436 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +6 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +711 -6
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +156 -0
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +103 -0
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +338 -0
- package/cpp/{stitcher_frame_data.hpp → camera_frame_data.hpp} +96 -13
- package/cpp/camera_frame_jsi.cpp +357 -0
- package/cpp/camera_frame_jsi.hpp +108 -0
- package/cpp/stitcher_proxy_jsi.cpp +140 -0
- package/cpp/stitcher_proxy_jsi.hpp +62 -0
- package/cpp/stitcher_worklet_dispatch.cpp +103 -0
- package/cpp/stitcher_worklet_dispatch.hpp +71 -0
- package/cpp/stitcher_worklet_registry.cpp +91 -0
- package/cpp/stitcher_worklet_registry.hpp +146 -0
- package/dist/camera/ARCameraView.d.ts +77 -0
- package/dist/camera/ARCameraView.js +90 -1
- package/dist/camera/Camera.d.ts +63 -4
- package/dist/camera/Camera.js +2 -2
- package/dist/camera/CaptureMemoryPill.d.ts +4 -3
- package/dist/camera/CaptureMemoryPill.js +4 -3
- package/dist/index.d.ts +2 -1
- package/dist/stitching/ARFrameMeta.d.ts +100 -0
- package/dist/stitching/{StitcherFrame.js → ARFrameMeta.js} +1 -1
- package/dist/stitching/{StitcherFrame.d.ts → CameraFrame.d.ts} +70 -11
- package/dist/stitching/CameraFrame.js +4 -0
- package/dist/stitching/ensureStitcherProxyInstalled.d.ts +8 -0
- package/dist/stitching/ensureStitcherProxyInstalled.js +81 -0
- package/dist/stitching/useStitcherWorklet.d.ts +4 -4
- package/dist/stitching/useStitcherWorklet.js +4 -4
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +23 -1
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +137 -2
- package/ios/Sources/RNImageStitcher/CameraFrameHostObject.h +83 -0
- package/ios/Sources/RNImageStitcher/CameraFrameHostObject.mm +760 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +336 -40
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +128 -0
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +313 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +42 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +160 -0
- package/package.json +1 -1
- package/src/camera/ARCameraView.tsx +211 -2
- package/src/camera/Camera.tsx +81 -4
- package/src/camera/CaptureMemoryPill.tsx +4 -3
- package/src/index.ts +7 -3
- package/src/stitching/ARFrameMeta.ts +107 -0
- package/src/stitching/{StitcherFrame.ts → CameraFrame.ts} +79 -11
- package/src/stitching/ensureStitcherProxyInstalled.ts +141 -0
- package/src/stitching/useStitcherWorklet.ts +9 -9
|
@@ -120,6 +120,72 @@ class RNSARSession(reactContext: ReactApplicationContext)
|
|
|
120
120
|
}
|
|
121
121
|
}
|
|
122
122
|
|
|
123
|
+
/**
|
|
124
|
+
* Android side of the JS `<Camera planeDetection=...>` prop. Sets
|
|
125
|
+
* which plane orientations are EMITTED into `arAnchors`
|
|
126
|
+
* (`"vertical"` | `"horizontal"` | `"both"`); an unrecognised value
|
|
127
|
+
* falls back to `"vertical"`.
|
|
128
|
+
*
|
|
129
|
+
* Unlike iOS — where the prop narrows ARKit's `planeDetection`
|
|
130
|
+
* option set — we deliberately do NOT narrow ARCore's
|
|
131
|
+
* `planeFindingMode`: it stays `HORIZONTAL_AND_VERTICAL` so ARCore
|
|
132
|
+
* keeps bootstrapping tracking from horizontal planes (a plain
|
|
133
|
+
* vertical wall alone leaves ARCore unable to establish a
|
|
134
|
+
* gravity-aligned world; see the start()/startForView() config
|
|
135
|
+
* comments). We only FILTER which plane orientations are surfaced
|
|
136
|
+
* into `arAnchors` (in [RNSARCameraView.collectTrackingAnchors]).
|
|
137
|
+
*
|
|
138
|
+
* Void (no Promise): a fire-and-forget setter mirroring the other
|
|
139
|
+
* config-prop bridge calls.
|
|
140
|
+
*/
|
|
141
|
+
@ReactMethod
|
|
142
|
+
fun setPlaneDetection(mode: String) {
|
|
143
|
+
planeDetectionMode = when (mode) {
|
|
144
|
+
"vertical", "horizontal", "both" -> mode
|
|
145
|
+
else -> "vertical"
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// ── onArFrame — LIGHT AR metadata channel (v0.18.0) ───────────────
|
|
150
|
+
//
|
|
151
|
+
// Android side of the shared `onArFrame` contract. Delivers light
|
|
152
|
+
// AR metadata (tracking state, pose, intrinsics, depth dims, anchor
|
|
153
|
+
// descriptors, mesh counts) to JS on the main thread as a normal
|
|
154
|
+
// RCTDeviceEventEmitter event — NO worklets, NO zero-copy buffers.
|
|
155
|
+
//
|
|
156
|
+
// This is the deliberate counterpart to the `arFrameProcessor`
|
|
157
|
+
// host-worklet path: worklets-core's closure-wrap crashes when an
|
|
158
|
+
// AR worklet captures host objects, so `onArFrame` bypasses worklets
|
|
159
|
+
// entirely and uses the bridge event channel the rest of this module
|
|
160
|
+
// already uses (see `IncrementalStitcher.emitState`).
|
|
161
|
+
//
|
|
162
|
+
// TS sets the gate via `setArFrameMetaEnabled(true, intervalMs)` when
|
|
163
|
+
// a host passes the `onArFrame` prop, and `setArFrameMetaEnabled(false, _)`
|
|
164
|
+
// on unmount / prop removal. The per-frame build + emit happens in
|
|
165
|
+
// `RNSARCameraView.onDrawFrame` (the only thread guaranteed to run
|
|
166
|
+
// once the AR session is live), gated + throttled by the companion
|
|
167
|
+
// state set here.
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* JS-facing gate for the `onArFrame` metadata channel.
|
|
171
|
+
*
|
|
172
|
+
* - `enabled` — true while a host supplies the `onArFrame` prop.
|
|
173
|
+
* - `intervalMs` — throttle floor in milliseconds (default contract
|
|
174
|
+
* value 100 ≈ 10 Hz). Clamped to ≥ 0; a 0 interval
|
|
175
|
+
* means "emit every render frame" (no throttle).
|
|
176
|
+
*
|
|
177
|
+
* Fire-and-forget (no Promise) — mirrors the other config-prop bridge
|
|
178
|
+
* setters (`setPlaneDetection`, `lockPortrait`).
|
|
179
|
+
*/
|
|
180
|
+
@ReactMethod
|
|
181
|
+
fun setArFrameMetaEnabled(enabled: Boolean, intervalMs: Double) {
|
|
182
|
+
arFrameMetaEnabled = enabled
|
|
183
|
+
arFrameMetaIntervalMs = if (intervalMs.isNaN()) 100L else intervalMs.toLong().coerceAtLeast(0L)
|
|
184
|
+
// Reset the throttle clock so the first frame after enabling emits
|
|
185
|
+
// immediately rather than waiting out a stale interval window.
|
|
186
|
+
arFrameMetaLastEmitNs = 0L
|
|
187
|
+
}
|
|
188
|
+
|
|
123
189
|
@ReactMethod
|
|
124
190
|
fun isSupported(promise: Promise) {
|
|
125
191
|
// `checkAvailability` can return UNKNOWN_CHECKING if the
|
|
@@ -832,6 +898,48 @@ class RNSARSession(reactContext: ReactApplicationContext)
|
|
|
832
898
|
if (attachedView === view) attachedView = null
|
|
833
899
|
}
|
|
834
900
|
|
|
901
|
+
/**
|
|
902
|
+
* Emit a pre-built [ARFrameMeta] WritableMap to JS over the shared
|
|
903
|
+
* `RNImageStitcherARFrame` device event. Called from
|
|
904
|
+
* [RNSARCameraView.maybeEmitArFrameMeta] on the GL render thread.
|
|
905
|
+
*
|
|
906
|
+
* Uses the same `DeviceEventManagerModule.RCTDeviceEventEmitter`
|
|
907
|
+
* channel as [IncrementalStitcher.emitState] — RN drops the event
|
|
908
|
+
* when no JS listener is attached, so no extra gating is needed
|
|
909
|
+
* beyond the enabled flag the caller already checked. The
|
|
910
|
+
* TS-required `addListener`/`removeListeners` no-op pair already
|
|
911
|
+
* exists on the `IncrementalStitcher` module; the `NativeEventEmitter`
|
|
912
|
+
* the TS layer constructs over `RNSARSession` needs the same pair, so
|
|
913
|
+
* they're declared below.
|
|
914
|
+
*/
|
|
915
|
+
internal fun emitArFrameMeta(meta: com.facebook.react.bridge.WritableMap) {
|
|
916
|
+
try {
|
|
917
|
+
reactApplicationContext
|
|
918
|
+
.getJSModule(
|
|
919
|
+
com.facebook.react.modules.core.DeviceEventManagerModule.RCTDeviceEventEmitter::class.java,
|
|
920
|
+
)
|
|
921
|
+
.emit(AR_FRAME_META_EVENT, meta)
|
|
922
|
+
} catch (t: Throwable) {
|
|
923
|
+
// Catalyst instance torn down mid-emit (reload / unmount race),
|
|
924
|
+
// or no JS context yet — drop the frame silently. AR metadata
|
|
925
|
+
// is best-effort and re-emitted every interval, so a dropped
|
|
926
|
+
// frame is harmless.
|
|
927
|
+
Log.d(TAG, "emitArFrameMeta: emit failed (ignoring): ${t.message}")
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
/// Required by RN's `NativeEventEmitter` contract — the TS
|
|
932
|
+
/// `onArFrame` wiring constructs a `NativeEventEmitter` over this
|
|
933
|
+
/// module, which calls `addListener`/`removeListeners` on subscribe /
|
|
934
|
+
/// unsubscribe. No-op on Android: `DeviceEventManagerModule` does its
|
|
935
|
+
/// own listener tracking and drops events when none are attached
|
|
936
|
+
/// (same rationale as `IncrementalStitcher.addListener`).
|
|
937
|
+
@ReactMethod
|
|
938
|
+
fun addListener(eventName: String) { /* no-op — see KDoc */ }
|
|
939
|
+
|
|
940
|
+
@ReactMethod
|
|
941
|
+
fun removeListeners(count: Int) { /* no-op — see KDoc */ }
|
|
942
|
+
|
|
835
943
|
private fun clearPoseLogInternal() {
|
|
836
944
|
poseLogLock.write { poseLog.clear() }
|
|
837
945
|
}
|
|
@@ -915,6 +1023,54 @@ class RNSARSession(reactContext: ReactApplicationContext)
|
|
|
915
1023
|
@Volatile
|
|
916
1024
|
var instance: RNSARSession? = null
|
|
917
1025
|
private set
|
|
1026
|
+
|
|
1027
|
+
/**
|
|
1028
|
+
* Which plane orientations reach `arAnchors`:
|
|
1029
|
+
* `"vertical"` | `"horizontal"` | `"both"`.
|
|
1030
|
+
*
|
|
1031
|
+
* Default `"vertical"` preserves the legacy plane-projected
|
|
1032
|
+
* stitch path (the shutter-gate / evaluatePlanesForFrame logic
|
|
1033
|
+
* only ever cared about vertical planes), so existing hosts see
|
|
1034
|
+
* no change unless they opt into a wider filter via the JS
|
|
1035
|
+
* `<Camera planeDetection=...>` prop (→ [setPlaneDetection]).
|
|
1036
|
+
*
|
|
1037
|
+
* Read on the GL render thread in
|
|
1038
|
+
* [RNSARCameraView.collectTrackingAnchors]; written from the JS
|
|
1039
|
+
* thread via [setPlaneDetection] — hence `@Volatile`.
|
|
1040
|
+
*/
|
|
1041
|
+
@JvmStatic
|
|
1042
|
+
@Volatile
|
|
1043
|
+
var planeDetectionMode: String = "vertical"
|
|
1044
|
+
|
|
1045
|
+
// ── onArFrame gate + throttle (v0.18.0) ──────────────────────
|
|
1046
|
+
//
|
|
1047
|
+
// Written from the JS thread via [setArFrameMetaEnabled];
|
|
1048
|
+
// read on the GL render thread in
|
|
1049
|
+
// [RNSARCameraView.maybeEmitArFrameMeta] — hence `@Volatile`.
|
|
1050
|
+
//
|
|
1051
|
+
// - `arFrameMetaEnabled` — gate: only build+emit when true.
|
|
1052
|
+
// - `arFrameMetaIntervalMs` — throttle floor (ms); contract
|
|
1053
|
+
// default 100 (≈10 Hz).
|
|
1054
|
+
// - `arFrameMetaLastEmitNs` — monotonic clock of the last emit
|
|
1055
|
+
// (System.nanoTime()); 0 = "never",
|
|
1056
|
+
// reset on every enable so the first
|
|
1057
|
+
// post-enable frame emits at once.
|
|
1058
|
+
@JvmStatic
|
|
1059
|
+
@Volatile
|
|
1060
|
+
var arFrameMetaEnabled: Boolean = false
|
|
1061
|
+
|
|
1062
|
+
@JvmStatic
|
|
1063
|
+
@Volatile
|
|
1064
|
+
var arFrameMetaIntervalMs: Long = 100L
|
|
1065
|
+
|
|
1066
|
+
@JvmStatic
|
|
1067
|
+
@Volatile
|
|
1068
|
+
var arFrameMetaLastEmitNs: Long = 0L
|
|
1069
|
+
|
|
1070
|
+
/// Event name carrying the [ARFrameMeta] payload to JS. MUST
|
|
1071
|
+
/// match the shared contract + the iOS `supportedEvents` entry +
|
|
1072
|
+
/// the TS `NativeEventEmitter` subscription string exactly.
|
|
1073
|
+
const val AR_FRAME_META_EVENT = "RNImageStitcherARFrame"
|
|
918
1074
|
}
|
|
919
1075
|
|
|
920
1076
|
init {
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
import android.util.Log
|
|
5
|
+
import com.facebook.react.bridge.ReactApplicationContext
|
|
6
|
+
import com.facebook.react.bridge.ReactContextBaseJavaModule
|
|
7
|
+
import com.facebook.react.bridge.ReactMethod
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* v0.8.0 Phase 4b.ii — Android-side JSI installer for the host
|
|
11
|
+
* worklet proxy. Mirror of iOS' `StitcherJsiInstaller`.
|
|
12
|
+
*
|
|
13
|
+
* The module exposes one synchronous method, `install()`, which JS
|
|
14
|
+
* calls once at lib bootstrap (via the
|
|
15
|
+
* `ensureStitcherProxyInstalled` helper in
|
|
16
|
+
* `src/stitching/ensureStitcherProxyInstalled.ts`). We reach into
|
|
17
|
+
* the main JS runtime via `ReactApplicationContext.getJavaScriptContextHolder().get()`
|
|
18
|
+
* — the canonical bridgeless-compatible accessor in modern RN
|
|
19
|
+
* (worklets-core's `WorkletsModule` uses the same pattern, verified
|
|
20
|
+
* working on RN 0.84.1 + new arch + Hermes).
|
|
21
|
+
*
|
|
22
|
+
* The native `nativeInstall(jsiRuntimeRef)` JNI then casts the long
|
|
23
|
+
* back to a `jsi::Runtime*` and calls into the shared C++
|
|
24
|
+
* `retailens::installStitcherProxy(runtime)` (in
|
|
25
|
+
* `cpp/stitcher_proxy_jsi.{hpp,cpp}`). Identical destination on
|
|
26
|
+
* both platforms — `globalThis.__stitcherProxy` exposes the same
|
|
27
|
+
* `install` / `uninstall` / `count` host functions.
|
|
28
|
+
*
|
|
29
|
+
* ## Returning `Boolean` (not `Promise`) from a sync method
|
|
30
|
+
*
|
|
31
|
+
* `isBlockingSynchronousMethod = true` + `Boolean` return is the
|
|
32
|
+
* documented pattern for "I'm doing one-shot native setup that
|
|
33
|
+
* needs to complete before the next JS line runs." Same shape as
|
|
34
|
+
* `WorkletsModule.install()`.
|
|
35
|
+
*
|
|
36
|
+
* ## What we DON'T do here (Phase 4b.ii follow-up)
|
|
37
|
+
*
|
|
38
|
+
* Phase 4b.ii's MVP installs the proxy ONLY. Host worklets that
|
|
39
|
+
* register through `__stitcherProxy.install` land in the native
|
|
40
|
+
* `retailens::StitcherWorkletRegistry`. Per-frame fan-out from
|
|
41
|
+
* Android's `StitcherWorkletRuntime` is a separate piece of work
|
|
42
|
+
* (Phase 4b.ii follow-up) — needs the Kotlin↔JNI bridge that
|
|
43
|
+
* constructs a `CameraFrameJsiHostObject` from an `ArImage` +
|
|
44
|
+
* pose and posts it through a worklet runtime. Until that lands,
|
|
45
|
+
* Android-registered worklets behave exactly like iOS-registered
|
|
46
|
+
* worklets BEFORE Phase 4b.i: they exist in the registry but
|
|
47
|
+
* aren't invoked.
|
|
48
|
+
*
|
|
49
|
+
* The proxy install itself is still useful as a foundation —
|
|
50
|
+
* verifies the JNI handshake works, exercises the bridgeless
|
|
51
|
+
* runtime accessor, and gives us a `count()` smoke test for the
|
|
52
|
+
* device verification step.
|
|
53
|
+
*/
|
|
54
|
+
class StitcherJsiInstallerModule(
|
|
55
|
+
private val reactContext: ReactApplicationContext,
|
|
56
|
+
) : ReactContextBaseJavaModule(reactContext) {
|
|
57
|
+
override fun getName(): String = NAME
|
|
58
|
+
|
|
59
|
+
@ReactMethod(isBlockingSynchronousMethod = true)
|
|
60
|
+
fun install(): Boolean {
|
|
61
|
+
return try {
|
|
62
|
+
// `getJavaScriptContextHolder().get()` returns a raw
|
|
63
|
+
// `jsi::Runtime*` boxed as `Long`. Same accessor
|
|
64
|
+
// worklets-core's `WorkletsModule.install()` uses;
|
|
65
|
+
// documented to work in both legacy + bridgeless modes
|
|
66
|
+
// on RN 0.71+.
|
|
67
|
+
val holder = reactContext.javaScriptContextHolder
|
|
68
|
+
if (holder == null) {
|
|
69
|
+
Log.e(TAG, "getJavaScriptContextHolder() returned null; runtime unreachable")
|
|
70
|
+
return false
|
|
71
|
+
}
|
|
72
|
+
val runtimeRef = holder.get()
|
|
73
|
+
if (runtimeRef == 0L) {
|
|
74
|
+
Log.e(TAG, "JavaScriptContextHolder.get() returned 0; runtime not initialized yet")
|
|
75
|
+
return false
|
|
76
|
+
}
|
|
77
|
+
val ok = nativeInstall(runtimeRef)
|
|
78
|
+
if (!ok) {
|
|
79
|
+
Log.e(TAG, "nativeInstall(runtimeRef=$runtimeRef) returned false")
|
|
80
|
+
}
|
|
81
|
+
ok
|
|
82
|
+
} catch (t: Throwable) {
|
|
83
|
+
Log.e(TAG, "install() threw — falling back to JS-side registry", t)
|
|
84
|
+
false
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
private external fun nativeInstall(jsiRuntimeRef: Long): Boolean
|
|
89
|
+
|
|
90
|
+
companion object {
|
|
91
|
+
const val NAME = "StitcherJsiInstaller"
|
|
92
|
+
private const val TAG = "StitcherJsiInstaller"
|
|
93
|
+
|
|
94
|
+
init {
|
|
95
|
+
// The Phase 3a JNI shim (`libimage_stitcher.so`) absorbed
|
|
96
|
+
// the JSI-install JNI binding from Phase 4b.ii. Loading
|
|
97
|
+
// it once is enough — Android's loader deduplicates,
|
|
98
|
+
// so even if `IncrementalStitcher.kt`'s init block
|
|
99
|
+
// already loaded the lib, calling again is a cheap no-op.
|
|
100
|
+
System.loadLibrary("image_stitcher")
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
import android.os.HandlerThread
|
|
5
|
+
import android.util.Log
|
|
6
|
+
import java.util.concurrent.atomic.AtomicBoolean
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* v0.8.0 Phase 3b — Android twin of iOS' `RNSARWorkletRuntime`.
|
|
10
|
+
* Owns the per-AR-frame worklet runtime + the thread it dispatches
|
|
11
|
+
* on. Symmetric API shape to the iOS class so the cross-platform
|
|
12
|
+
* dispatch story (Phase 3c) lands in lockstep.
|
|
13
|
+
*
|
|
14
|
+
* ## Phase 3b scope (this commit)
|
|
15
|
+
*
|
|
16
|
+
* - Singleton accessor + lifecycle (installIfNeeded / isInstalled).
|
|
17
|
+
* - Dedicated `HandlerThread` for worklet dispatch — keeps work off
|
|
18
|
+
* the GLSurfaceView GL render thread (audit caveat #4 from the
|
|
19
|
+
* Phase-0 audit).
|
|
20
|
+
* - `dispatchFrame()` stub — Phase 3c will fill in:
|
|
21
|
+
* 1. Build `CameraFrameHostObject` from ARCore Frame + pose
|
|
22
|
+
* (via the shared C++ JSI host object now linked into
|
|
23
|
+
* `libimage_stitcher.so` post-Phase-3a).
|
|
24
|
+
* 2. Run first-party stitching synchronously on the caller
|
|
25
|
+
* thread (`onDrawFrame`'s GL thread, today).
|
|
26
|
+
* 3. If host worklets are registered, dispatch the host object
|
|
27
|
+
* onto this runtime's `HandlerThread` + invoke each worklet
|
|
28
|
+
* via JNI → `RNWorklet::WorkletInvoker::call`.
|
|
29
|
+
* 4. Invalidate the host object after all worklets return.
|
|
30
|
+
*
|
|
31
|
+
* ## Worklet-runtime construction model
|
|
32
|
+
*
|
|
33
|
+
* Unlike iOS (where the lib's `.mm` directly `std::make_shared`s
|
|
34
|
+
* a `RNWorklet::JsiWorkletContext`), Android can't construct the
|
|
35
|
+
* context purely from native C++ without JNI plumbing. Phase 3c
|
|
36
|
+
* will choose between two paths:
|
|
37
|
+
*
|
|
38
|
+
* - **Option A:** JS-side code calls
|
|
39
|
+
* `Worklets.createContext("stitcher.ar")` at AR-mode start;
|
|
40
|
+
* hands the resulting context pointer to this Kotlin class via
|
|
41
|
+
* a small JSI plugin. Minimal new JNI. **Phase 3b's
|
|
42
|
+
* HandlerThread becomes dead code** under this option — the
|
|
43
|
+
* JS-side `Worklets.createContext` picks its own thread. We'd
|
|
44
|
+
* need to remove the HandlerThread + change `installIfNeeded`
|
|
45
|
+
* into a no-op until a `setContextHandle(Long)` setter lands.
|
|
46
|
+
* - **Option B:** Direct JNI binding to worklets-core's C++
|
|
47
|
+
* constructor. More native code but no JS dependency at runtime.
|
|
48
|
+
* **Phase 3b's HandlerThread is exactly the right scaffold**
|
|
49
|
+
* under this option — its looper becomes the JsiWorkletContext's
|
|
50
|
+
* `workletCallInvoker` target.
|
|
51
|
+
*
|
|
52
|
+
* **Phase 3b assumption: Option B is the more likely path.** The
|
|
53
|
+
* scaffolding below (HandlerThread + serial dispatch) fits Option
|
|
54
|
+
* B; if Phase 3c picks Option A instead, the HandlerThread becomes
|
|
55
|
+
* unused and Phase 3c will refactor accordingly.
|
|
56
|
+
*
|
|
57
|
+
* @see [RNSARWorkletRuntime] iOS equivalent
|
|
58
|
+
* @see docs/plans/handoff/2026-05-26-v0.8.0-phase-0-audit.md
|
|
59
|
+
* worklets-core API rationale (Audit 2: `JsiWorkletContext`).
|
|
60
|
+
* @see docs/plans/handoff/2026-05-26-v0.8.0-phases-2-5-implementation-guide.md
|
|
61
|
+
* Phase 3c implementation plan.
|
|
62
|
+
*/
|
|
63
|
+
object StitcherWorkletRuntime {
|
|
64
|
+
private const val TAG = "StitcherWorkletRuntime"
|
|
65
|
+
|
|
66
|
+
/// Single-flight install guard. `compareAndSet` makes the
|
|
67
|
+
/// runtime construction race-safe across concurrent first-mount
|
|
68
|
+
/// calls from multiple `<Camera>` instances.
|
|
69
|
+
private val installed = AtomicBoolean(false)
|
|
70
|
+
|
|
71
|
+
/// Dedicated dispatch thread. Constructed eagerly so we can
|
|
72
|
+
/// validate the thread starts cleanly during `installIfNeeded`.
|
|
73
|
+
/// Off the GLSurfaceView GL render thread (audit caveat #4)
|
|
74
|
+
/// + off the main thread. Phase 3c will configure the worklet
|
|
75
|
+
/// context's `workletCallInvoker` to post onto this thread's
|
|
76
|
+
/// looper.
|
|
77
|
+
private val dispatchThread: HandlerThread by lazy {
|
|
78
|
+
HandlerThread("io.imagestitcher.ar-worklet-runtime").apply { start() }
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// Construct the underlying worklet context if not yet installed.
|
|
82
|
+
/// Idempotent — repeated calls are no-ops.
|
|
83
|
+
///
|
|
84
|
+
/// Phase 3b: starts the dispatch thread; no JsiWorkletContext
|
|
85
|
+
/// construction yet (deferred to Phase 3c).
|
|
86
|
+
/// Phase 3c: also wires the JsiWorkletContext + binds it to the
|
|
87
|
+
/// dispatch thread's looper.
|
|
88
|
+
@JvmStatic
|
|
89
|
+
fun installIfNeeded() {
|
|
90
|
+
if (!installed.compareAndSet(false, true)) {
|
|
91
|
+
return
|
|
92
|
+
}
|
|
93
|
+
// Force the lazy `dispatchThread` to initialise. If the
|
|
94
|
+
// OS denies thread creation (extreme memory pressure on a
|
|
95
|
+
// budget device), `HandlerThread.start()` won't throw but
|
|
96
|
+
// the looper won't be available — the Phase 3c dispatch
|
|
97
|
+
// logic will need to defend against that. For Phase 3b
|
|
98
|
+
// we only care that this method returns without throwing.
|
|
99
|
+
//
|
|
100
|
+
// Log `Thread.id` (Java-side monotonic, always non-zero) —
|
|
101
|
+
// NOT `HandlerThread.threadId` (Linux tid set after first
|
|
102
|
+
// Looper-prepared message; reading from this caller thread
|
|
103
|
+
// immediately after .start() returns -1 until scheduled).
|
|
104
|
+
val javaThreadId = dispatchThread.id
|
|
105
|
+
Log.i(TAG, "installed runtime; dispatch java-thread id=$javaThreadId")
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/// Diagnostics + tests. Returns `true` after a successful
|
|
109
|
+
/// `installIfNeeded()`.
|
|
110
|
+
@JvmStatic
|
|
111
|
+
fun isInstalled(): Boolean = installed.get()
|
|
112
|
+
|
|
113
|
+
/// v0.8.0 Phase 3c — first-party stitching dispatch. Invokes
|
|
114
|
+
/// the supplied block synchronously on the caller thread
|
|
115
|
+
/// (`onDrawFrame`'s GL render thread today).
|
|
116
|
+
///
|
|
117
|
+
/// Phase 3c minimum-viable: this is the closure-based equivalent
|
|
118
|
+
/// of iOS' first-party callback. The block is the original
|
|
119
|
+
/// `module.ingestFromARCameraView(...)` call site moved
|
|
120
|
+
/// verbatim into a lambda — no behaviour change, just an
|
|
121
|
+
/// indirection so Phase 4 can interpose host-worklet fanout
|
|
122
|
+
/// without touching the engine ingest path.
|
|
123
|
+
///
|
|
124
|
+
/// **Why synchronous + on the caller thread:** the engine's
|
|
125
|
+
/// `ingestFromARCameraView` takes ownership of the ARCore
|
|
126
|
+
/// `Image`-derived NV21 buffer (via the v0.10.0 `TransferredNV21`
|
|
127
|
+
/// wrapper). ARCore's `Image.close()` happens after this call
|
|
128
|
+
/// returns, so the consumer must finish reading the bytes before
|
|
129
|
+
/// we return — exactly what synchronous block invocation
|
|
130
|
+
/// provides. Phase 4 will copy the buffer for off-thread
|
|
131
|
+
/// access in host worklets; Phase 3c keeps the sync contract.
|
|
132
|
+
///
|
|
133
|
+
/// If `installIfNeeded()` hasn't been called yet, the block
|
|
134
|
+
/// still runs (no-op on the registry side). Defensive — the
|
|
135
|
+
/// caller may call this method before `installIfNeeded` is
|
|
136
|
+
/// wired up.
|
|
137
|
+
@JvmStatic
|
|
138
|
+
fun runFirstParty(block: () -> Unit) {
|
|
139
|
+
// Synchronous invocation — Phase 4 will extend this to also
|
|
140
|
+
// post the registered host worklets onto `dispatchThread`.
|
|
141
|
+
// Not `inline`: Phase 4 will need to read `dispatchThread`
|
|
142
|
+
// (private) from inside this function body, and Kotlin's
|
|
143
|
+
// inline functions can't access private members from call
|
|
144
|
+
// sites outside the declaring class. Per-frame lambda
|
|
145
|
+
// alloc is ~ns and the alternative (callers passing a
|
|
146
|
+
// method reference) doesn't materially change cost.
|
|
147
|
+
block()
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/// v0.8.0 Phase 4b.iii — fan out one AR frame to every host
|
|
151
|
+
/// worklet registered in the shared C++ `StitcherWorkletRegistry`
|
|
152
|
+
/// (populated from JS via `__stitcherProxy.install(workletFn)`).
|
|
153
|
+
///
|
|
154
|
+
/// Called from `RNSARCameraView.onDrawFrame` immediately after
|
|
155
|
+
/// `runFirstParty { ... }` returns, with the already-extracted
|
|
156
|
+
/// AR frame data (pose + NV21 bytes + dimensions + tracking
|
|
157
|
+
/// state).
|
|
158
|
+
///
|
|
159
|
+
/// **Fast-path:** the native side queries the registry's count
|
|
160
|
+
/// FIRST and returns before copying any bytes when no host
|
|
161
|
+
/// worklets are registered. In the common first-party-only
|
|
162
|
+
/// deployment, this method costs one JNI call + one C++ atomic
|
|
163
|
+
/// read per frame — negligible.
|
|
164
|
+
///
|
|
165
|
+
/// **When host worklets ARE registered:** the JNI layer copies
|
|
166
|
+
/// the NV21 byte array into an owned C++ `std::vector` (so the
|
|
167
|
+
/// async dispatch can outlive ARCore's `Image.close()` scope),
|
|
168
|
+
/// builds a `CameraFrameJsiHostObject`, and posts a lambda
|
|
169
|
+
/// onto worklets-core's default `JsiWorkletContext`'s worklet
|
|
170
|
+
/// thread. The lambda iterates the registry's
|
|
171
|
+
/// `WorkletInvoker`s, calls each with the JSI host object as
|
|
172
|
+
/// its argument, and invalidates the host object after the
|
|
173
|
+
/// last invoker returns. Per-worklet failure isolation: one
|
|
174
|
+
/// host worklet throwing does NOT stop the lib's stitching or
|
|
175
|
+
/// the other host worklets.
|
|
176
|
+
///
|
|
177
|
+
/// **Threading:** this method returns synchronously on the
|
|
178
|
+
/// caller's thread. The actual worklet invocations happen
|
|
179
|
+
/// asynchronously on the worklets-core thread; the caller does
|
|
180
|
+
/// NOT block on them.
|
|
181
|
+
///
|
|
182
|
+
/// **Caller-thread contract:** the caller (`RNSARCameraView`'s
|
|
183
|
+
/// `onDrawFrame`) MUST have already invoked `runFirstParty`
|
|
184
|
+
/// before calling this method. The first-party stitching
|
|
185
|
+
/// path holds the synchronous ARCore Image consumption
|
|
186
|
+
/// contract; the host-worklet dispatch does not.
|
|
187
|
+
///
|
|
188
|
+
/// @param nv21Bytes Pre-packed NV21 byte array. COPIED
|
|
189
|
+
/// into a native owned buffer; caller can
|
|
190
|
+
/// release the reference after return.
|
|
191
|
+
/// @param width Camera image width (pixels).
|
|
192
|
+
/// @param height Camera image height (pixels).
|
|
193
|
+
/// @param qx,qy,qz,qw Pose rotation quaternion (unit length).
|
|
194
|
+
/// @param tx,ty,tz Pose translation (metres, world coords).
|
|
195
|
+
/// @param timestampNs Frame timestamp in nanoseconds.
|
|
196
|
+
/// @param trackingState One of "" / "notAvailable" / "limited"
|
|
197
|
+
/// / "normal". Empty string ⇒ JS-side
|
|
198
|
+
/// `arTrackingState` is `undefined`.
|
|
199
|
+
/// @param depthBytes Raw ARCore DEPTH16 bytes, dense row-packed
|
|
200
|
+
/// (`depthWidth*depthHeight*2` bytes, uint16/px,
|
|
201
|
+
/// low 13 bits = mm, high 3 bits = confidence).
|
|
202
|
+
/// `null` when depth is unavailable this frame —
|
|
203
|
+
/// the JNI then leaves `data.arDepth == nullopt`.
|
|
204
|
+
/// @param depthWidth Depth-map width (px); 0 when no depth.
|
|
205
|
+
/// @param depthHeight Depth-map height (px); 0 when no depth.
|
|
206
|
+
/// @param anchorIds Parallel arrays describing every TRACKING
|
|
207
|
+
/// @param anchorTypes anchor: stable id, coarse type
|
|
208
|
+
/// @param anchorTransforms ("plane"/"image"/"point"/"mesh"), and a
|
|
209
|
+
/// 16-element ROW-MAJOR (anchor->world) transform
|
|
210
|
+
/// (identity for the depth-derived "mesh" anchor —
|
|
211
|
+
/// its vertices are camera-local). Empty when no
|
|
212
|
+
/// anchors/mesh were collected.
|
|
213
|
+
/// @param anchorMeshVertices Parallel per-anchor mesh byte arrays: a
|
|
214
|
+
/// @param anchorMeshFaces Float32-xyz vertex buffer + a Uint32 index
|
|
215
|
+
/// buffer for the depth-derived mesh anchor, `null`
|
|
216
|
+
/// for every non-mesh anchor. Carried verbatim to
|
|
217
|
+
/// the JNI which sets `ArAnchor.hasMesh` + the
|
|
218
|
+
/// mesh vectors when both are non-null.
|
|
219
|
+
@JvmStatic
|
|
220
|
+
fun dispatchToHostWorklets(
|
|
221
|
+
nv21Bytes: ByteArray,
|
|
222
|
+
width: Int,
|
|
223
|
+
height: Int,
|
|
224
|
+
qx: Double, qy: Double, qz: Double, qw: Double,
|
|
225
|
+
tx: Double, ty: Double, tz: Double,
|
|
226
|
+
timestampNs: Double,
|
|
227
|
+
trackingState: String,
|
|
228
|
+
depthBytes: ByteArray?,
|
|
229
|
+
depthWidth: Int,
|
|
230
|
+
depthHeight: Int,
|
|
231
|
+
anchorIds: Array<String>,
|
|
232
|
+
anchorTypes: Array<String>,
|
|
233
|
+
anchorTransforms: Array<DoubleArray>,
|
|
234
|
+
anchorMeshVertices: Array<ByteArray?>,
|
|
235
|
+
anchorMeshFaces: Array<ByteArray?>,
|
|
236
|
+
fx: Double, fy: Double, cx: Double, cy: Double,
|
|
237
|
+
intrinsicsImageWidth: Int, intrinsicsImageHeight: Int,
|
|
238
|
+
anchorAlignments: Array<String>,
|
|
239
|
+
anchorExtents: Array<DoubleArray?>,
|
|
240
|
+
) {
|
|
241
|
+
if (!installed.get()) return
|
|
242
|
+
nativeDispatchToHostWorklets(
|
|
243
|
+
nv21Bytes, width, height,
|
|
244
|
+
qx, qy, qz, qw,
|
|
245
|
+
tx, ty, tz,
|
|
246
|
+
timestampNs, trackingState,
|
|
247
|
+
depthBytes, depthWidth, depthHeight,
|
|
248
|
+
anchorIds, anchorTypes, anchorTransforms,
|
|
249
|
+
anchorMeshVertices, anchorMeshFaces,
|
|
250
|
+
fx, fy, cx, cy,
|
|
251
|
+
intrinsicsImageWidth, intrinsicsImageHeight,
|
|
252
|
+
anchorAlignments, anchorExtents,
|
|
253
|
+
)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/// v0.8.0 Phase 4b.iii — number of registered host worklets.
|
|
257
|
+
/// Cheap (microsecond) call into the native registry. Used by
|
|
258
|
+
/// `RNSARCameraView.onDrawFrame` to gate the per-frame
|
|
259
|
+
/// NV21-pack + dispatch path: when no worklets are registered
|
|
260
|
+
/// AND no capture is active, the entire `forwardToIncremental`
|
|
261
|
+
/// branch can be skipped, saving the ~3-5ms NV21 pack cost per
|
|
262
|
+
/// idle preview frame.
|
|
263
|
+
@JvmStatic
|
|
264
|
+
fun hasHostWorklets(): Boolean {
|
|
265
|
+
if (!installed.get()) return false
|
|
266
|
+
return nativeRegistryCount() > 0
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/// Per-frame AR-metadata extraction toggles (the JS-driven
|
|
270
|
+
/// enableDepth/enableAnchors/enableMesh `<Camera>` props, written via
|
|
271
|
+
/// `__stitcherProxy.setExtractionConfig`). Read once per frame in
|
|
272
|
+
/// `RNSARCameraView.forwardToIncremental` to GATE the costly ARCore
|
|
273
|
+
/// depth-acquire / anchor-collect / mesh-build work — all default OFF,
|
|
274
|
+
/// so a host pays zero AR-metadata cost until it opts in.
|
|
275
|
+
///
|
|
276
|
+
/// Returns all-false (no extraction) before `installIfNeeded()` runs.
|
|
277
|
+
data class ExtractionFlags(
|
|
278
|
+
val depth: Boolean,
|
|
279
|
+
val anchors: Boolean,
|
|
280
|
+
val mesh: Boolean,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
@JvmStatic
|
|
284
|
+
fun extractionFlags(): ExtractionFlags {
|
|
285
|
+
if (!installed.get()) return ExtractionFlags(false, false, false)
|
|
286
|
+
val bits = nativeExtractionFlags()
|
|
287
|
+
return ExtractionFlags(
|
|
288
|
+
depth = (bits and 0x1) != 0,
|
|
289
|
+
anchors = (bits and 0x2) != 0,
|
|
290
|
+
mesh = (bits and 0x4) != 0,
|
|
291
|
+
)
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
@JvmStatic
|
|
295
|
+
private external fun nativeRegistryCount(): Int
|
|
296
|
+
|
|
297
|
+
/// JNI binding: `nativeExtractionFlags` in
|
|
298
|
+
/// `android/src/main/cpp/stitcher_jsi_install_jni.cpp`. Packs
|
|
299
|
+
/// `retailens::getExtractionConfig()` into a bitmask
|
|
300
|
+
/// (bit0=depth, bit1=anchors, bit2=mesh).
|
|
301
|
+
@JvmStatic
|
|
302
|
+
private external fun nativeExtractionFlags(): Int
|
|
303
|
+
|
|
304
|
+
/// JNI binding: `android/src/main/cpp/stitcher_jsi_install_jni.cpp`'s
|
|
305
|
+
/// `nativeDispatchToHostWorklets`. Fast-path early-exit lives
|
|
306
|
+
/// inside the native function — see its docstring.
|
|
307
|
+
@JvmStatic
|
|
308
|
+
private external fun nativeDispatchToHostWorklets(
|
|
309
|
+
nv21Bytes: ByteArray,
|
|
310
|
+
width: Int,
|
|
311
|
+
height: Int,
|
|
312
|
+
qx: Double, qy: Double, qz: Double, qw: Double,
|
|
313
|
+
tx: Double, ty: Double, tz: Double,
|
|
314
|
+
timestampNs: Double,
|
|
315
|
+
trackingState: String,
|
|
316
|
+
depthBytes: ByteArray?,
|
|
317
|
+
depthWidth: Int,
|
|
318
|
+
depthHeight: Int,
|
|
319
|
+
anchorIds: Array<String>,
|
|
320
|
+
anchorTypes: Array<String>,
|
|
321
|
+
anchorTransforms: Array<DoubleArray>,
|
|
322
|
+
anchorMeshVertices: Array<ByteArray?>,
|
|
323
|
+
anchorMeshFaces: Array<ByteArray?>,
|
|
324
|
+
fx: Double, fy: Double, cx: Double, cy: Double,
|
|
325
|
+
intrinsicsImageWidth: Int, intrinsicsImageHeight: Int,
|
|
326
|
+
anchorAlignments: Array<String>,
|
|
327
|
+
anchorExtents: Array<DoubleArray?>,
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
init {
|
|
331
|
+
// The JSI install module (`StitcherJsiInstallerModule`)
|
|
332
|
+
// already loads `libimage_stitcher` at class load. We
|
|
333
|
+
// load it again here defensively in case
|
|
334
|
+
// `StitcherWorkletRuntime` is referenced before the install
|
|
335
|
+
// module — `System.loadLibrary` is idempotent.
|
|
336
|
+
System.loadLibrary("image_stitcher")
|
|
337
|
+
}
|
|
338
|
+
}
|