react-native-image-stitcher 0.16.2 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/RNImageStitcher.podspec +26 -1
  3. package/android/build.gradle +20 -0
  4. package/android/src/main/cpp/CMakeLists.txt +46 -3
  5. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +227 -0
  6. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +6 -0
  7. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +55 -6
  8. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +103 -0
  9. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +256 -0
  10. package/cpp/stitcher_frame_jsi.cpp +214 -0
  11. package/cpp/stitcher_frame_jsi.hpp +108 -0
  12. package/cpp/stitcher_proxy_jsi.cpp +109 -0
  13. package/cpp/stitcher_proxy_jsi.hpp +46 -0
  14. package/cpp/stitcher_worklet_dispatch.cpp +103 -0
  15. package/cpp/stitcher_worklet_dispatch.hpp +71 -0
  16. package/cpp/stitcher_worklet_registry.cpp +91 -0
  17. package/cpp/stitcher_worklet_registry.hpp +146 -0
  18. package/dist/camera/ARCameraView.d.ts +20 -0
  19. package/dist/camera/ARCameraView.js +23 -1
  20. package/dist/camera/Camera.d.ts +12 -0
  21. package/dist/camera/Camera.js +2 -2
  22. package/dist/camera/CaptureMemoryPill.d.ts +4 -3
  23. package/dist/camera/CaptureMemoryPill.js +4 -3
  24. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +8 -0
  25. package/dist/stitching/ensureStitcherProxyInstalled.js +81 -0
  26. package/ios/Sources/RNImageStitcher/RNSARSession.swift +44 -6
  27. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +128 -0
  28. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +313 -0
  29. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +60 -0
  30. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +214 -0
  31. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +42 -0
  32. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +160 -0
  33. package/package.json +1 -1
  34. package/src/camera/ARCameraView.tsx +51 -2
  35. package/src/camera/Camera.tsx +15 -0
  36. package/src/camera/CaptureMemoryPill.tsx +4 -3
  37. package/src/stitching/ensureStitcherProxyInstalled.ts +141 -0
package/CHANGELOG.md CHANGED
@@ -14,6 +14,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
14
14
  > during 0.x are bumped to a new MINOR (e.g., 0.1 → 0.2), and the
15
15
  > upgrade path is documented in this CHANGELOG.
16
16
 
17
+ ## [0.17.0] — 2026-06-19
18
+
19
+ ### Added — `arFrameProcessor`: observe AR frames with a host worklet
20
+
21
+ `<Camera>` gains an **`arFrameProcessor`** prop — a `'worklet'` invoked once per
22
+ **ARKit / ARCore frame** while in AR capture, dispatched natively and running
23
+ *alongside* first-party stitching (composition, not replacement). The worklet
24
+ receives a `StitcherFrame` tagged `source: 'ar'` with the world-space `pose` and
25
+ `arTrackingState`. It fires during preview too (continuous observation), at zero
26
+ per-frame cost when no worklet is registered.
27
+
28
+ This restores the previously-archived AR host-worklet capability and re-exposes
29
+ it as an explicit prop (rather than the old auto-registering hook). Under the
30
+ hood it installs `globalThis.__stitcherProxy` (JSI) on first use and fans frames
31
+ out through a shared C++ proxy / registry / dispatch layer on both platforms
32
+ (verified against `react-native-worklets-core` 1.6.3).
33
+
34
+ The non-AR equivalent remains `frameProcessor` (vision-camera); the two modes use
35
+ different runtimes and frame shapes, hence the separate prop. The
36
+ `StitcherFrame` / `StitcherFrameProcessor` type names are unchanged.
37
+
38
+ Verified on device: the worklet fires per frame on **iOS (ARKit, iPhone 16 Pro)**
39
+ and **Android (ARCore, Galaxy A35)**.
40
+
41
+ ### Fixed
42
+
43
+ - **Example app crashed at launch on Android** (`PlatformConstants could not be
44
+ found`). The v0.16.2 OpenCV-reuse demo added an app-level `externalNativeBuild`
45
+ to `example/android/app/build.gradle` that displaced React Native's own
46
+ New-Architecture app native build (so core TurboModules weren't compiled in).
47
+ Removed it; React Native owns the app native build again. **Example-app only —
48
+ the published SDK was never affected.**
49
+
17
50
  ## [0.16.2] — 2026-06-17
18
51
 
19
52
  ### Added — reuse the bundled OpenCV from your host app's native code (Android)
@@ -60,6 +60,18 @@ Pod::Spec.new do |s|
60
60
 
61
61
  s.dependency 'React-Core'
62
62
 
63
+ # react-native-worklets-core — provides the `RNWorklet::WorkletInvoker`
64
+ # + `JsiWorkletContext` primitives the AR-mode JSI fan-out is built on
65
+ # (StitcherJsiInstaller.mm / RNSARWorkletRuntime.mm + the shared
66
+ # cpp/stitcher_worklet_{registry,dispatch}.cpp). In practice this pod
67
+ # is already in every host's graph (vision-camera depends on it), but
68
+ # declaring it here makes the dependency explicit and guarantees its
69
+ # headers are present even for a host that uses AR mode without
70
+ # vision-camera. The bare `WKTJsiWorklet.h` includes in the .mm files
71
+ # resolve via the HEADER_SEARCH_PATHS entry below (the package's own
72
+ # node_modules copy of the worklets-core cpp/ dir).
73
+ s.dependency 'react-native-worklets-core'
74
+
63
75
  # ─────────────────────────────────────────────────────────────────────
64
76
  # OpenCV — pre-built custom xcframework fetched by postinstall
65
77
  # ─────────────────────────────────────────────────────────────────────
@@ -84,6 +96,19 @@ Pod::Spec.new do |s|
84
96
  'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17',
85
97
  'CLANG_CXX_LIBRARY' => 'libc++',
86
98
  'OTHER_CPLUSPLUSFLAGS' => '$(inherited) -std=c++17',
87
- 'HEADER_SEARCH_PATHS' => '$(inherited) "${PODS_TARGET_SRCROOT}/cpp"',
99
+ # HEADER_SEARCH_PATHS:
100
+ # - "${PODS_TARGET_SRCROOT}/cpp" — the shared C++ port's own
101
+ # headers (keyframe_gate.hpp, stitcher_frame_jsi.hpp, …).
102
+ # - the worklets-core cpp/ dir — so the bare `#include
103
+ # "WKTJsiWorklet.h"` / "WKTJsiWorkletContext.h" lines in
104
+ # StitcherJsiInstaller.mm + RNSARWorkletRuntime.mm resolve.
105
+ # PODS_ROOT is `<host>/ios/Pods`; the package's worklets-core
106
+ # copy lives at `<host>/node_modules/react-native-worklets-core/
107
+ # cpp`, i.e. `${PODS_ROOT}/../node_modules/...`. (The shared
108
+ # cpp/*.cpp files instead use the namespace-prefixed
109
+ # `<react-native-worklets-core/WKTJsiWorklet.h>` form, which
110
+ # resolves against `${PODS_ROOT}/Headers/Public` — already on
111
+ # the inherited path — and works on Android's prefab too.)
112
+ 'HEADER_SEARCH_PATHS' => '$(inherited) "${PODS_TARGET_SRCROOT}/cpp" "${PODS_ROOT}/../node_modules/react-native-worklets-core/cpp"',
88
113
  }
89
114
  end
@@ -301,6 +301,26 @@ dependencies {
301
301
  android.sourceSets.main.java.exclude '**/CvFlowGateFrameProcessor.kt'
302
302
  }
303
303
 
304
+ // v0.8.0 Phase 4b.ii/iii — react-native-worklets-core, consumed
305
+ // for its `rnworklets` PREFAB module (RNWorklet::JsiWorkletContext /
306
+ // WorkletInvoker) by the AR frame-processor JSI fan-out
307
+ // (src/main/cpp/stitcher_jsi_install_jni.cpp + the shared cpp/
308
+ // stitcher_*_jsi.cpp). `find_package(react-native-worklets-core ...)`
309
+ // in CMakeLists.txt needs this gradle project on the path so AGP
310
+ // wires the prefab into the native build. Same consumption pattern
311
+ // react-native-vision-camera uses for Frame Processors.
312
+ //
313
+ // `findProject(...)` guard mirrors the vision-camera block above:
314
+ // the SDK still compiles in hosts that don't install worklets-core
315
+ // (those hosts simply don't use the AR frame processor — the JSI
316
+ // sources still compile, but the prefab link only happens when the
317
+ // host provides worklets-core). Autolinking adds the project for
318
+ // any host that depends on vision-camera (which transitively pulls
319
+ // in worklets-core).
320
+ if (findProject(':react-native-worklets-core') != null) {
321
+ implementation project(':react-native-worklets-core')
322
+ }
323
+
304
324
  // v0.10.0 audit #11A — Android JUnit test scaffold. JVM unit
305
325
  // tests for pure-Kotlin data wrappers + algorithm helpers that
306
326
  // don't need an Android device. Run via
@@ -66,6 +66,20 @@ add_library(opencv_stitching STATIC IMPORTED)
66
66
  set_target_properties(opencv_stitching PROPERTIES
67
67
  IMPORTED_LOCATION "${OPENCV_STITCHING_A}")
68
68
 
69
+ # ── React Native + worklets-core prefabs (v0.8.0 Phase 4b.ii/iii) ──
70
+ #
71
+ # The AR frame-processor's JSI fan-out (stitcher_jsi_install_jni.cpp +
72
+ # the shared cpp/stitcher_*_jsi.cpp) needs RN's JSI runtime, fbjni, and
73
+ # react-native-worklets-core's `RNWorklet::JsiWorkletContext` /
74
+ # `WorkletInvoker`. RN 0.71+ ships jsi + the native-modules umbrella as
75
+ # prefab packages; worklets-core ships a `rnworklets` prefab module.
76
+ # `buildFeatures.prefab true` (android/build.gradle) makes these
77
+ # discoverable. We mirror react-native-vision-camera's consumption of
78
+ # the SAME prefabs in this example app (verified working on RN 0.84.1).
79
+ find_package(ReactAndroid REQUIRED CONFIG)
80
+ find_package(fbjni REQUIRED CONFIG)
81
+ find_package(react-native-worklets-core REQUIRED CONFIG)
82
+
69
83
  # ── Shared C++ port (KeyframeGate) ────────────────────────────────
70
84
  #
71
85
  # `cpp/` at the SDK root holds C++ that's compiled into BOTH the iOS
@@ -91,12 +105,26 @@ add_library(image_stitcher SHARED
91
105
  # retry + dimension/memory instrumentation. Used to live in this
92
106
  # file (image_stitcher_jni.cpp). See cpp/stitcher.hpp for design
93
107
  # rationale.
94
- "${SHARED_CPP_DIR}/stitcher.cpp")
108
+ "${SHARED_CPP_DIR}/stitcher.cpp"
109
+ # v0.8.0 Phase 4b.ii/iii — AR frame-processor JSI fan-out. The JNI
110
+ # binding (stitcher_jsi_install_jni.cpp) installs
111
+ # `globalThis.__stitcherProxy` and fans each AR frame out to the
112
+ # registered host worklets. The 4 shared cpp/ JSI files below are
113
+ # the SAME source the iOS pod compiles — one cross-platform JSI
114
+ # surface (proxy host object + native worklet registry + per-frame
115
+ # dispatch helper + StitcherFrame JSI host object).
116
+ stitcher_jsi_install_jni.cpp
117
+ "${SHARED_CPP_DIR}/stitcher_proxy_jsi.cpp"
118
+ "${SHARED_CPP_DIR}/stitcher_worklet_registry.cpp"
119
+ "${SHARED_CPP_DIR}/stitcher_worklet_dispatch.cpp"
120
+ "${SHARED_CPP_DIR}/stitcher_frame_jsi.cpp")
95
121
 
96
122
  target_include_directories(image_stitcher PRIVATE
97
123
  "${OPENCV_INCLUDE_DIR}"
98
124
  # cpp/ holds keyframe_gate.hpp + ar_frame_pose.h that the JNI
99
- # bindings include without relative-path spelunking.
125
+ # bindings include without relative-path spelunking. Also the
126
+ # shared JSI sources (stitcher_proxy_jsi.hpp, stitcher_frame_data.hpp,
127
+ # stitcher_worklet_*.hpp, stitcher_frame_jsi.hpp).
100
128
  "${SHARED_CPP_DIR}")
101
129
 
102
130
  # Link order matters:
@@ -118,7 +146,22 @@ target_link_libraries(image_stitcher
118
146
  opencv_stitching
119
147
  -Wl,--no-whole-archive
120
148
  opencv_java
121
- log)
149
+ log
150
+ # v0.8.0 Phase 4b.ii/iii — JSI fan-out deps. Mirrors
151
+ # react-native-vision-camera's link set on RN 0.84.1:
152
+ # ReactAndroid::jsi — facebook::jsi::Runtime / Value / Object
153
+ # ReactAndroid::reactnative — RN's native-modules umbrella prefab
154
+ # (RN >= 0.76; CallInvoker + friends that
155
+ # worklets-core's context depends on)
156
+ # fbjni::fbjni — JNI helpers worklets-core links against
157
+ # react-native-worklets-core::rnworklets — JsiWorkletContext +
158
+ # WorkletInvoker. Carries its own
159
+ # include dir so <react-native-worklets-core/
160
+ # WKTJsiWorkletContext.h> resolves.
161
+ ReactAndroid::jsi
162
+ ReactAndroid::reactnative
163
+ fbjni::fbjni
164
+ react-native-worklets-core::rnworklets)
122
165
 
123
166
  target_compile_options(image_stitcher PRIVATE
124
167
  -fvisibility=hidden
@@ -0,0 +1,227 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ //
3
+ // stitcher_jsi_install_jni.cpp — JNI binding for the Android-side
4
+ // JSI install (v0.8.0 Phase 4b.ii).
5
+ //
6
+ // Kotlin's `StitcherJsiInstallerModule.nativeInstall(jsiRuntimeRef)`
7
+ // calls into this file. We unbox the `jsi::Runtime*` from the
8
+ // Java `long` and hand it to the shared
9
+ // `retailens::installStitcherProxy(runtime)` function which sets
10
+ // `globalThis.__stitcherProxy`. Same destination as iOS — the
11
+ // host object class lives in `cpp/stitcher_proxy_jsi.{hpp,cpp}`.
12
+ //
13
+ // ## Why a `long` ref, not a JSI handle wrapper class
14
+ //
15
+ // `ReactApplicationContext.getJavaScriptContextHolder()` returns a
16
+ // `JavaScriptContextHolder` whose `.get()` returns a Java `long`
17
+ // that's the raw pointer to the C++ `jsi::Runtime*`. Same
18
+ // contract as worklets-core's `WorkletsModule.nativeInstall`
19
+ // (verified at the same call site). Caller is responsible for
20
+ // ensuring the runtime outlives this call — in practice, the
21
+ // runtime IS the JS thread's runtime which lives the whole
22
+ // process lifetime, so this is structurally always safe in our
23
+ // usage.
24
+ //
25
+ // ## Threading
26
+ //
27
+ // Kotlin invokes this from a `@ReactMethod(isBlockingSynchronousMethod
28
+ // = true)` so we're already on the JS thread. Synchronous JSI
29
+ // access is safe.
30
+
31
+ #include "stitcher_proxy_jsi.hpp"
32
+
33
+ // v0.8.0 Phase 4b.iii — per-frame fan-out support. The shared
34
+ // `dispatchToHostWorklets` posts to worklets-core's default context;
35
+ // this JNI file's `nativeDispatchToHostWorklets` constructs the
36
+ // `StitcherFrameData` from raw bytes + pose + dims and forwards it.
37
+ #include "stitcher_frame_data.hpp"
38
+ #include "stitcher_worklet_dispatch.hpp"
39
+ #include "stitcher_worklet_registry.hpp"
40
+
41
+ #include <react-native-worklets-core/WKTJsiWorkletContext.h>
42
+
43
+ #include <jni.h>
44
+ #include <jsi/jsi.h>
45
+
46
+ #include <android/log.h>
47
+
48
+ #include <cstdint>
49
+ #include <cstring>
50
+ #include <memory>
51
+ #include <utility>
52
+ #include <vector>
53
+
54
+ #define LOG_TAG "StitcherJsiInstaller"
55
+ #define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
56
+ #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
57
+
58
+ extern "C" JNIEXPORT jboolean JNICALL
59
+ Java_io_imagestitcher_rn_StitcherJsiInstallerModule_nativeInstall(
60
+ JNIEnv* /*env*/, jobject /*thiz*/, jlong jsiRuntimeRef) {
61
+ if (jsiRuntimeRef == 0) {
62
+ // ReactApplicationContext.getJavaScriptContextHolder().get()
63
+ // returns 0 when the runtime isn't ready (rare — JS would have
64
+ // had to call us before its own runtime was up; impossible in
65
+ // practice). Defensive.
66
+ return JNI_FALSE;
67
+ }
68
+ auto* runtime = reinterpret_cast<facebook::jsi::Runtime*>(jsiRuntimeRef);
69
+ retailens::installStitcherProxy(*runtime);
70
+ LOGI("installed globalThis.__stitcherProxy on main JS runtime.");
71
+ return JNI_TRUE;
72
+ }
73
+
74
+ // ─── v0.8.0 Phase 4b.iii — Android NV21 PixelBufferReader ──────────
75
+ //
76
+ // Owns a heap-allocated `std::vector<uint8_t>` of pre-copied NV21
77
+ // bytes. Constructed by `nativeDispatchToHostWorklets` after one
78
+ // JNI byte-array copy from Kotlin; outlives the AR render thread
79
+ // scope via `StitcherFrameData::pixelReader`'s `shared_ptr` —
80
+ // dropped when the host object is invalidated.
81
+
82
+ namespace {
83
+
84
+ class AndroidNV21BufferReader : public retailens::PixelBufferReader {
85
+ public:
86
+ explicit AndroidNV21BufferReader(std::vector<uint8_t>&& bytes)
87
+ : _bytes(std::move(bytes)) {}
88
+
89
+ std::size_t byteSize() const override { return _bytes.size(); }
90
+
91
+ std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) override {
92
+ if (dst == nullptr) return 0;
93
+ std::size_t n = std::min(maxBytes, _bytes.size());
94
+ if (n > 0) {
95
+ std::memcpy(dst, _bytes.data(), n);
96
+ }
97
+ return n;
98
+ }
99
+
100
+ private:
101
+ std::vector<uint8_t> _bytes;
102
+ };
103
+
104
+ } // namespace
105
+
106
+ // ─── v0.8.0 Phase 4b.iii — registry count accessor ─────────────────
107
+ //
108
+ // Cheap (microsecond) accessor for the per-frame gate in
109
+ // `RNSARCameraView.onDrawFrame`. Avoids the NV21 byte-pack cost
110
+ // when no host worklets are registered AND no capture is active.
111
+ // Same atomic-read the JSI host object's `count()` host function
112
+ // goes through.
113
+ extern "C" JNIEXPORT jint JNICALL
114
+ Java_io_imagestitcher_rn_StitcherWorkletRuntime_nativeRegistryCount(
115
+ JNIEnv* /*env*/, jobject /*thiz*/) {
116
+ return static_cast<jint>(
117
+ retailens::StitcherWorkletRegistry::shared().count());
118
+ }
119
+
120
+ // ─── v0.8.0 Phase 4b.iii — per-frame dispatch JNI binding ──────────
121
+ //
122
+ // Called from Kotlin's `StitcherWorkletRuntime.dispatchToHostWorklets`
123
+ // after the first-party stitching block has returned (the AR-frame
124
+ // data is still in scope on the Kotlin side because
125
+ // `RNSARCameraView.onDrawFrame` reads the ARCore Frame, builds the
126
+ // NV21 byte[], invokes first-party via `runFirstParty { ... }`,
127
+ // THEN calls into here).
128
+ //
129
+ // The byte[] is COPIED into our owned vector — ARCore's pixel data
130
+ // becomes inaccessible shortly after `onDrawFrame` returns, and our
131
+ // async dispatch must outlive that scope. Cost: one ~3MB memcpy
132
+ // per frame at 1080p NV21 (~90 MB/s at 30 fps; <5 ms on a mid-range
133
+ // Android device). Fast-path early-exit when the registry is empty
134
+ // skips the copy entirely.
135
+ //
136
+ // trackingState: Kotlin passes one of "" / "notAvailable" / "limited"
137
+ // / "normal" (empty string = field unset → JS sees undefined).
138
+ extern "C" JNIEXPORT void JNICALL
139
+ Java_io_imagestitcher_rn_StitcherWorkletRuntime_nativeDispatchToHostWorklets(
140
+ JNIEnv* env, jobject /*thiz*/,
141
+ jbyteArray nv21Bytes,
142
+ jint width, jint height,
143
+ jdouble qx, jdouble qy, jdouble qz, jdouble qw,
144
+ jdouble tx, jdouble ty, jdouble tz,
145
+ jdouble timestampNs,
146
+ jstring trackingState) {
147
+ // Fast-path early-exit BEFORE the JNI byte-array copy. Saves the
148
+ // ~3MB memcpy + JSI host object alloc on every frame in the
149
+ // common first-party-only case.
150
+ if (retailens::StitcherWorkletRegistry::shared().count() == 0) {
151
+ return;
152
+ }
153
+
154
+ if (nv21Bytes == nullptr) {
155
+ LOGE("nativeDispatchToHostWorklets: nv21Bytes is null");
156
+ return;
157
+ }
158
+
159
+ const jsize byteLen = env->GetArrayLength(nv21Bytes);
160
+ if (byteLen <= 0) {
161
+ LOGE("nativeDispatchToHostWorklets: nv21Bytes is empty");
162
+ return;
163
+ }
164
+
165
+ // Copy into our owned vector. `GetByteArrayRegion` is the
166
+ // canonical "copy" path — `GetByteArrayElements + Release` MAY
167
+ // pin the JVM array (zero-copy) but the contract isn't
168
+ // guaranteed; we need our own buffer for the async dispatch
169
+ // anyway, so the explicit copy is cleaner.
170
+ std::vector<uint8_t> bytes(static_cast<std::size_t>(byteLen));
171
+ env->GetByteArrayRegion(
172
+ nv21Bytes, 0, byteLen,
173
+ reinterpret_cast<jbyte*>(bytes.data()));
174
+
175
+ // Extract trackingState string (may be null on the Kotlin side
176
+ // for non-AR or pre-tracking frames — guard accordingly).
177
+ std::string trackingStateStr;
178
+ if (trackingState != nullptr) {
179
+ const char* cs = env->GetStringUTFChars(trackingState, nullptr);
180
+ if (cs != nullptr) {
181
+ trackingStateStr = cs;
182
+ env->ReleaseStringUTFChars(trackingState, cs);
183
+ }
184
+ }
185
+
186
+ // Build StitcherFrameData. Field semantics match the iOS
187
+ // `StitcherFrameHostObject::fromARFrame:pose:` factory; this is
188
+ // the Android equivalent path.
189
+ retailens::StitcherFrameData data;
190
+ data.source = "ar";
191
+ data.width = static_cast<int32_t>(width);
192
+ data.height = static_cast<int32_t>(height);
193
+ // ARCore's camera image is YUV_420_888 on Android, mapped to NV21
194
+ // by the existing `YuvImageConverter.packNV21` path — the byte[]
195
+ // we receive is interleaved Y then VU. Worklets gate on this
196
+ // string identifier (`'yuv'` vs `'unknown'`); v0.8.0 always
197
+ // emits `'yuv'` for AR mode on Android (NV21).
198
+ data.pixelFormat = "yuv";
199
+ // Android AR-mode camera image is always landscape-natural; the
200
+ // mapping matches iOS' coarse two-value set. Hosts that need
201
+ // exact display orientation read it from the device-orientation
202
+ // sensors (see `useDeviceOrientation` hook).
203
+ data.orientation = (width >= height) ? "landscape-right" : "portrait";
204
+ data.timestampNs = timestampNs;
205
+ data.qx = qx;
206
+ data.qy = qy;
207
+ data.qz = qz;
208
+ data.qw = qw;
209
+ data.tx = tx;
210
+ data.ty = ty;
211
+ data.tz = tz;
212
+ data.hasTranslation = true; // AR mode always has translation
213
+ data.arTrackingState = trackingStateStr;
214
+ data.pixelReader =
215
+ std::make_shared<AndroidNV21BufferReader>(std::move(bytes));
216
+
217
+ // Dispatch on worklets-core's default context. That context is
218
+ // initialised by JS' `Worklets.install()` (which runs at lib
219
+ // bootstrap when worklets-core's module is imported); by the
220
+ // time host worklets are registered, the default context is up.
221
+ // The shared dispatch helper handles the registry snapshot,
222
+ // host-object construction (inside the worklet thread), per-
223
+ // worklet failure isolation, and invalidation.
224
+ retailens::dispatchToHostWorklets(
225
+ RNWorklet::JsiWorkletContext::getDefaultInstance(),
226
+ std::move(data));
227
+ }
@@ -101,6 +101,12 @@ class RNImageStitcherPackage : ReactPackage {
101
101
  RNSARSession(reactContext),
102
102
  IncrementalStitcher(reactContext),
103
103
  FileBridge(reactContext),
104
+ // v0.8.0 Phase 4b.ii — surfaces `NativeModules.StitcherJsiInstaller`
105
+ // so JS' `ensureStitcherProxyInstalled()` can call its
106
+ // blocking-sync `install()` to install `globalThis.__stitcherProxy`
107
+ // on the main JS runtime (AR frame-processor host-worklet
108
+ // registration). Mirror of iOS' StitcherJsiInstaller.
109
+ StitcherJsiInstallerModule(reactContext),
104
110
  )
105
111
  }
106
112
 
@@ -383,17 +383,30 @@ class RNSARCameraView @JvmOverloads constructor(
383
383
  cameraPosWorld,
384
384
  )
385
385
 
386
+ // v0.8.0 Phase 4b.iii — ensure the host-worklet runtime is
387
+ // installed before any per-frame fan-out can run. Idempotent
388
+ // (AtomicBoolean CAS): the first frame starts the dispatch
389
+ // thread; every later frame is a single atomic read. Kept on
390
+ // the GL thread because that's the only thread guaranteed to
391
+ // run once the AR session is live.
392
+ StitcherWorkletRuntime.installIfNeeded()
393
+
386
394
  // Push pose into the AR session log. Mirrors iOS' delegate
387
395
  // path; the existing RNSARFramePose / appendPose
388
396
  // contract was already in place for Phase 4.
389
397
  appendPose(camera, frame.timestamp)
390
398
 
391
- // Forward to the incremental stitcher only when capture is
392
- // engaged. (The v0.8.0 host-worklet dispatch which also
393
- // forwarded preview frames whenever host worklets were
394
- // registered was archived in the 2026-06 batch-keyframe
395
- // cleanup.)
396
- if (ingestActive) {
399
+ // Forward to the incremental stitcher when capture is engaged,
400
+ // OR when an AR frame-processor host worklet is registered (the
401
+ // v0.8.0 Phase 4b.iii fan-out forwards preview frames whenever
402
+ // host worklets exist, even with capture off — the host worklet
403
+ // observes the live AR stream). `forwardToIncremental` does the
404
+ // NV21 pack once and gates the first-party ingest internally on
405
+ // `ingestActive`; the host-worklet dispatch is gated on the
406
+ // native registry count. `hasHostWorklets()` is a cheap atomic
407
+ // read (microseconds) so the common capture-off / no-worklet
408
+ // preview path stays near-free.
409
+ if (ingestActive || StitcherWorkletRuntime.hasHostWorklets()) {
397
410
  forwardToIncremental(frame, camera)
398
411
  }
399
412
 
@@ -656,6 +669,42 @@ class RNSARCameraView @JvmOverloads constructor(
656
669
  },
657
670
  )
658
671
  } // closes `if (ingestActive)` (v0.8.0 Phase 4b.iii)
672
+
673
+ // ── v0.8.0 Phase 4b.iii — AR frame-processor host-worklet fan-out ──
674
+ //
675
+ // After the first-party stitching ingest (above), fan the SAME
676
+ // already-packed NV21 frame + pose out to every host worklet the
677
+ // JS `arFrameProcessor` registered via `__stitcherProxy.install`.
678
+ // This is independent of `ingestActive`: a host worklet observes
679
+ // the live AR stream whether or not the user has engaged capture
680
+ // (the onDrawFrame gate already let us in when host worklets
681
+ // exist). `dispatchToHostWorklets` does a cheap native
682
+ // registry-count fast-path early-exit + (only when worklets are
683
+ // registered) copies the bytes into an owned native buffer and
684
+ // dispatches asynchronously on worklets-core's default context,
685
+ // so the GL render thread is NOT blocked on worklet execution.
686
+ //
687
+ // We reuse `packed.nv21` (full NV21: Y plane then interleaved
688
+ // VU) + `qarr` / `tArr` (already read above) — no extra Image
689
+ // hold, no second pack. ARCore camera pose is full 6DoF, so
690
+ // translation is always valid.
691
+ val arTracking = when (camera.trackingState) {
692
+ TrackingState.TRACKING -> "normal"
693
+ TrackingState.PAUSED -> "limited"
694
+ TrackingState.STOPPED -> "notAvailable"
695
+ else -> "notAvailable"
696
+ }
697
+ StitcherWorkletRuntime.dispatchToHostWorklets(
698
+ nv21Bytes = packed.nv21,
699
+ width = packed.width,
700
+ height = packed.height,
701
+ qx = qarr[0].toDouble(), qy = qarr[1].toDouble(),
702
+ qz = qarr[2].toDouble(), qw = qarr[3].toDouble(),
703
+ tx = tArr[0].toDouble(), ty = tArr[1].toDouble(),
704
+ tz = tArr[2].toDouble(),
705
+ timestampNs = frame.timestamp.toDouble(),
706
+ trackingState = arTracking,
707
+ )
659
708
  }
660
709
 
661
710
  /// v0.13.2 — map the JS physical device orientation to the
@@ -0,0 +1,103 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ package io.imagestitcher.rn
3
+
4
+ import android.util.Log
5
+ import com.facebook.react.bridge.ReactApplicationContext
6
+ import com.facebook.react.bridge.ReactContextBaseJavaModule
7
+ import com.facebook.react.bridge.ReactMethod
8
+
9
+ /**
10
+ * v0.8.0 Phase 4b.ii — Android-side JSI installer for the host
11
+ * worklet proxy. Mirror of iOS' `StitcherJsiInstaller`.
12
+ *
13
+ * The module exposes one synchronous method, `install()`, which JS
14
+ * calls once at lib bootstrap (via the
15
+ * `ensureStitcherProxyInstalled` helper in
16
+ * `src/stitching/ensureStitcherProxyInstalled.ts`). We reach into
17
+ * the main JS runtime via `ReactApplicationContext.getJavaScriptContextHolder().get()`
18
+ * — the canonical bridgeless-compatible accessor in modern RN
19
+ * (worklets-core's `WorkletsModule` uses the same pattern, verified
20
+ * working on RN 0.84.1 + new arch + Hermes).
21
+ *
22
+ * The native `nativeInstall(jsiRuntimeRef)` JNI then casts the long
23
+ * back to a `jsi::Runtime*` and calls into the shared C++
24
+ * `retailens::installStitcherProxy(runtime)` (in
25
+ * `cpp/stitcher_proxy_jsi.{hpp,cpp}`). Identical destination on
26
+ * both platforms — `globalThis.__stitcherProxy` exposes the same
27
+ * `install` / `uninstall` / `count` host functions.
28
+ *
29
+ * ## Returning `Boolean` (not `Promise`) from a sync method
30
+ *
31
+ * `isBlockingSynchronousMethod = true` + `Boolean` return is the
32
+ * documented pattern for "I'm doing one-shot native setup that
33
+ * needs to complete before the next JS line runs." Same shape as
34
+ * `WorkletsModule.install()`.
35
+ *
36
+ * ## What we DON'T do here (Phase 4b.ii follow-up)
37
+ *
38
+ * Phase 4b.ii's MVP installs the proxy ONLY. Host worklets that
39
+ * register through `__stitcherProxy.install` land in the native
40
+ * `retailens::StitcherWorkletRegistry`. Per-frame fan-out from
41
+ * Android's `StitcherWorkletRuntime` is a separate piece of work
42
+ * (Phase 4b.ii follow-up) — needs the Kotlin↔JNI bridge that
43
+ * constructs a `StitcherFrameJsiHostObject` from an `ArImage` +
44
+ * pose and posts it through a worklet runtime. Until that lands,
45
+ * Android-registered worklets behave exactly like iOS-registered
46
+ * worklets BEFORE Phase 4b.i: they exist in the registry but
47
+ * aren't invoked.
48
+ *
49
+ * The proxy install itself is still useful as a foundation —
50
+ * verifies the JNI handshake works, exercises the bridgeless
51
+ * runtime accessor, and gives us a `count()` smoke test for the
52
+ * device verification step.
53
+ */
54
+ class StitcherJsiInstallerModule(
55
+ private val reactContext: ReactApplicationContext,
56
+ ) : ReactContextBaseJavaModule(reactContext) {
57
+ override fun getName(): String = NAME
58
+
59
+ @ReactMethod(isBlockingSynchronousMethod = true)
60
+ fun install(): Boolean {
61
+ return try {
62
+ // `getJavaScriptContextHolder().get()` returns a raw
63
+ // `jsi::Runtime*` boxed as `Long`. Same accessor
64
+ // worklets-core's `WorkletsModule.install()` uses;
65
+ // documented to work in both legacy + bridgeless modes
66
+ // on RN 0.71+.
67
+ val holder = reactContext.javaScriptContextHolder
68
+ if (holder == null) {
69
+ Log.e(TAG, "getJavaScriptContextHolder() returned null; runtime unreachable")
70
+ return false
71
+ }
72
+ val runtimeRef = holder.get()
73
+ if (runtimeRef == 0L) {
74
+ Log.e(TAG, "JavaScriptContextHolder.get() returned 0; runtime not initialized yet")
75
+ return false
76
+ }
77
+ val ok = nativeInstall(runtimeRef)
78
+ if (!ok) {
79
+ Log.e(TAG, "nativeInstall(runtimeRef=$runtimeRef) returned false")
80
+ }
81
+ ok
82
+ } catch (t: Throwable) {
83
+ Log.e(TAG, "install() threw — falling back to JS-side registry", t)
84
+ false
85
+ }
86
+ }
87
+
88
+ private external fun nativeInstall(jsiRuntimeRef: Long): Boolean
89
+
90
+ companion object {
91
+ const val NAME = "StitcherJsiInstaller"
92
+ private const val TAG = "StitcherJsiInstaller"
93
+
94
+ init {
95
+ // The Phase 3a JNI shim (`libimage_stitcher.so`) absorbed
96
+ // the JSI-install JNI binding from Phase 4b.ii. Loading
97
+ // it once is enough — Android's loader deduplicates,
98
+ // so even if `IncrementalStitcher.kt`'s init block
99
+ // already loaded the lib, calling again is a cheap no-op.
100
+ System.loadLibrary("image_stitcher")
101
+ }
102
+ }
103
+ }