react-native-image-stitcher 0.7.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +122 -0
  2. package/android/build.gradle +35 -1
  3. package/android/src/main/cpp/CMakeLists.txt +64 -2
  4. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +227 -0
  5. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +30 -11
  6. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +4 -0
  7. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +78 -3
  8. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +103 -0
  9. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +256 -0
  10. package/android/src/main/java/io/imagestitcher/rn/TransferredNV21.kt +100 -0
  11. package/cpp/stitcher_frame_data.hpp +141 -0
  12. package/cpp/stitcher_frame_jsi.cpp +214 -0
  13. package/cpp/stitcher_frame_jsi.hpp +108 -0
  14. package/cpp/stitcher_proxy_jsi.cpp +109 -0
  15. package/cpp/stitcher_proxy_jsi.hpp +46 -0
  16. package/cpp/stitcher_worklet_dispatch.cpp +103 -0
  17. package/cpp/stitcher_worklet_dispatch.hpp +71 -0
  18. package/cpp/stitcher_worklet_registry.cpp +81 -0
  19. package/cpp/stitcher_worklet_registry.hpp +136 -0
  20. package/dist/camera/Camera.d.ts +62 -12
  21. package/dist/camera/Camera.js +30 -15
  22. package/dist/index.d.ts +2 -0
  23. package/dist/index.js +11 -1
  24. package/dist/stitching/StitcherFrame.d.ts +170 -0
  25. package/dist/stitching/StitcherFrame.js +4 -0
  26. package/dist/stitching/StitcherWorkletRegistry.d.ts +117 -0
  27. package/dist/stitching/StitcherWorkletRegistry.js +78 -0
  28. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +8 -0
  29. package/dist/stitching/ensureStitcherProxyInstalled.js +81 -0
  30. package/dist/stitching/useFrameProcessor.d.ts +119 -0
  31. package/dist/stitching/useFrameProcessor.js +196 -0
  32. package/ios/Sources/RNImageStitcher/RNSARSession.swift +46 -10
  33. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +128 -0
  34. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +313 -0
  35. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +60 -0
  36. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +214 -0
  37. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +42 -0
  38. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +103 -0
  39. package/package.json +1 -1
  40. package/src/camera/Camera.tsx +93 -28
  41. package/src/index.ts +16 -0
  42. package/src/stitching/StitcherFrame.ts +197 -0
  43. package/src/stitching/StitcherWorkletRegistry.ts +156 -0
  44. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +176 -0
  45. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +94 -0
  46. package/src/stitching/ensureStitcherProxyInstalled.ts +141 -0
  47. package/src/stitching/useFrameProcessor.ts +226 -0
@@ -0,0 +1,103 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ //
3
+ // StitcherJsiInstaller.mm — implementation. Installs
4
+ // `globalThis.__stitcherProxy` on the main JS runtime.
5
+ //
6
+ // ## Why a host object rather than two globalThis functions
7
+ //
8
+ // We could install `__stitcherProxy_install` + `__stitcherProxy_uninstall`
9
+ // directly on `globalThis`. Wrapping them in a host object is
10
+ // slightly more code but:
11
+ // - Namespaces the proxy under a single global property
12
+ // (easier to feature-detect; one `if (globalThis.__stitcherProxy)`
13
+ // instead of two).
14
+ // - Matches vc's pattern (`global.VisionCameraProxy`), so future
15
+ // readers recognise the shape.
16
+ // - Keeps room to grow (e.g., add `__stitcherProxy.snapshot()` for
17
+ // diagnostics) without polluting globalThis further.
18
+
19
+ #import "StitcherJsiInstaller.h"
20
+
21
+ #import <Foundation/Foundation.h>
22
+ #import <React/RCTBridge.h>
23
+ #import <React/RCTBridge+Private.h>
24
+ #import <React/RCTUtils.h>
25
+ #import <os/log.h>
26
+
27
+ #include <jsi/jsi.h>
28
+
29
+ #include "stitcher_proxy_jsi.hpp"
30
+
31
+ using namespace facebook;
32
+
33
+ // The host object class + install logic moved to shared C++ in
34
+ // `cpp/stitcher_proxy_jsi.{hpp,cpp}` (v0.8.0 Phase 4b.ii). The
35
+ // Android JNI installer reuses the same `install` / `uninstall` /
36
+ // `count` host functions verbatim — the JSI dispatch is identical
37
+ // across platforms (matches the StitcherFrame host object's design).
38
+
39
+ #pragma mark - RN module
40
+
41
+ @implementation StitcherJsiInstaller
42
+
43
+ // RN injects `_bridge` at module init (legacy bridge → RCTBridge*;
44
+ // bridgeless / new arch → RCTBridgeProxy*, which forwards `runtime`
45
+ // access via NSProxy `forwardInvocation:`). Using the injected
46
+ // `_bridge` instead of `[RCTBridge currentBridge]` is the
47
+ // bridgeless-compatible idiom — `currentBridge` is nil under new
48
+ // arch. Pattern lifted from `react-native-worklets-core/ios/Worklets.mm`.
49
+ @synthesize bridge = _bridge;
50
+
51
+ RCT_EXPORT_MODULE()
52
+
53
+ + (BOOL)requiresMainQueueSetup {
54
+ return YES;
55
+ }
56
+
57
+ - (void)setBridge:(RCTBridge*)bridge {
58
+ _bridge = bridge;
59
+ }
60
+
61
+ // Synchronous install method. JS calls this once at lib bootstrap
62
+ // to install the global proxy on the main JS runtime. Returns
63
+ // `@YES` on success or `@NO` if the JSI runtime wasn't reachable
64
+ // (remote debug mode pre-Hermes; bridge not yet ready; etc.).
65
+ //
66
+ // `RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD` is the documented
67
+ // pattern for "run native code synchronously on the JS thread to
68
+ // install JSI bindings." Same pattern worklets-core + vision-camera
69
+ // use for their installs.
70
+ //
71
+ // **Bridgeless mode:** `_bridge` is an `RCTBridgeProxy` (NSProxy
72
+ // subclass) that forwards `-runtime` / `-jsCallInvoker` invocations
73
+ // to the underlying RCTHost-backed runtime. The `(RCTCxxBridge*)`
74
+ // cast is a no-op at runtime (NSProxy ignores static type) but
75
+ // keeps the Obj-C compiler happy about property access.
76
+ RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(install) {
77
+ if (_bridge == nil) {
78
+ os_log_error(OS_LOG_DEFAULT,
79
+ "[StitcherJsiInstaller] _bridge is nil; the module was "
80
+ "instantiated without bridge injection. Cannot install "
81
+ "__stitcherProxy.");
82
+ return @NO;
83
+ }
84
+
85
+ RCTCxxBridge* cxxBridge = (RCTCxxBridge*)_bridge;
86
+ if (cxxBridge.runtime == nullptr) {
87
+ os_log_error(OS_LOG_DEFAULT,
88
+ "[StitcherJsiInstaller] _bridge.runtime is nullptr; the JS "
89
+ "runtime hasn't been initialized yet OR remote debugger is "
90
+ "attached. Cannot install __stitcherProxy.");
91
+ return @NO;
92
+ }
93
+
94
+ jsi::Runtime& runtime = *(jsi::Runtime*)cxxBridge.runtime;
95
+ retailens::installStitcherProxy(runtime);
96
+
97
+ os_log_info(OS_LOG_DEFAULT,
98
+ "[StitcherJsiInstaller] installed globalThis.__stitcherProxy "
99
+ "on main JS runtime.");
100
+ return @YES;
101
+ }
102
+
103
+ @end
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "react-native-image-stitcher",
3
- "version": "0.7.1",
3
+ "version": "0.8.0",
4
4
  "description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -294,21 +294,71 @@ export interface CameraProps {
294
294
  onError?: (err: CameraError) => void;
295
295
 
296
296
  /**
297
- * Optional vision-camera frame processor. Only attached to the
298
- * non-AR preview (AR mode uses ARCameraView, which doesn't expose
299
- * a worklet seam). Build the worklet on the host side with
300
- * `useFrameProcessor` from `react-native-vision-camera`.
297
+ * Optional host-supplied vision-camera frame processor.
301
298
  *
302
- * Introduced for F8 (FrameProcessor port) — see
303
- * `docs/f8-frame-processor-plan.md`.
299
+ * ## When to set this prop
304
300
  *
305
- * The SDK installs its own frame processor via
306
- * `useFrameProcessorDriver`. Setting this prop is ignored with
307
- * a one-time `console.warn` — supplying a host worklet would
308
- * race with the SDK's pixel-buffer feed. Either remove the prop
309
- * or fork the SDK if you genuinely need a custom worklet.
301
+ * v0.8.0+ canonical answer: use the lib's own `useFrameProcessor`
302
+ * hook, NOT `react-native-vision-camera`'s. The lib's hook:
310
303
  *
311
- * AR mode is irrelevant: vision-camera's Camera isn't mounted.
304
+ * - **AR mode**: auto-registers the worklet in the native
305
+ * `__stitcherProxy` registry; the AR session's per-frame
306
+ * dispatch fans out to it alongside the lib's first-party
307
+ * stitching. No prop wiring needed — just mount the hook
308
+ * anywhere in the tree.
309
+ * - **Non-AR mode**: returns a vc processor object that this
310
+ * prop accepts. Wiring it through enables the host's
311
+ * worklet to fire on vc's Frame Processor runtime.
312
+ *
313
+ * ```tsx
314
+ * import { Camera, useFrameProcessor, type StitcherFrame }
315
+ * from 'react-native-image-stitcher';
316
+ *
317
+ * function MyScreen() {
318
+ * const fp = useFrameProcessor((frame: StitcherFrame) => {
319
+ * 'worklet';
320
+ * // ...
321
+ * }, []);
322
+ * return <Camera frameProcessor={fp} ... />;
323
+ * }
324
+ * ```
325
+ *
326
+ * ## Non-AR mode tradeoff (HONEST)
327
+ *
328
+ * vision-camera's `<Camera>` accepts ONLY ONE frame processor.
329
+ * The lib's internal `useFrameProcessorDriver` produces the
330
+ * processor that drives first-party panorama stitching in non-AR
331
+ * mode. If you supply your own via this prop, **the lib's
332
+ * first-party stitching is replaced** — panorama capture in
333
+ * non-AR mode will not produce stitched output until you remove
334
+ * the prop or fork the SDK to compose both worklets manually.
335
+ *
336
+ * For the common case (host wants worklet + lib wants stitching
337
+ * concurrently), prefer AR mode: the AR-mode path natively fans
338
+ * out to both the lib's first-party stitching AND every
339
+ * registered host worklet on every frame, with per-worklet
340
+ * failure isolation.
341
+ *
342
+ * Composition for non-AR mode (lib stitching + host worklet on
343
+ * the same vc processor) is tracked as a v0.9+ follow-up;
344
+ * needs the lib's first-party logic exposed as a vc Frame
345
+ * Processor plugin the host's worklet can call.
346
+ *
347
+ * ## AR mode behaviour
348
+ *
349
+ * In AR mode (`defaultCaptureSource="ar"` or runtime-toggled),
350
+ * vc's `<Camera>` isn't mounted; this prop has no effect.
351
+ * Host worklets registered via the lib's `useFrameProcessor`
352
+ * fire automatically through the AR-session dispatch path
353
+ * (iOS Phase 4b.i / Android Phase 4b.iii).
354
+ *
355
+ * ## Backwards compatibility
356
+ *
357
+ * The pre-v0.8.0 behaviour (warn + ignore) is preserved when the
358
+ * supplied processor is recognisably from
359
+ * `react-native-vision-camera`'s `useFrameProcessor` directly
360
+ * (no `__stitcherFrame` marker). Hosts should migrate to the
361
+ * lib's `useFrameProcessor` to benefit from AR-mode dispatch.
312
362
  *
313
363
  * (v0.5 had a `legacyDriver` escape hatch that routed back to
314
364
  * `useIncrementalJSDriver`. That hook + prop were removed in
@@ -791,29 +841,44 @@ export function Camera(props: CameraProps): React.JSX.Element {
791
841
  // eslint-disable-next-line react-hooks/exhaustive-deps
792
842
  useEffect(() => () => { fpDriver.stop(); }, []);
793
843
 
794
- // One-shot deprecation warning when the host supplies their own
795
- // `frameProcessor` prop. Two worklets racing on the same
796
- // producer thread would corrupt the engine's workQueue ordering,
797
- // so the SDK's own worklet wins and the host's is silently
798
- // ignored. (v0.5 had a `legacyDriver` opt-out for hosts that
799
- // wanted to route around the SDK driver; that was removed in
800
- // v0.6 along with `useIncrementalJSDriver`.)
801
- const hostFrameProcessorIgnoredWarnedRef = useRef(false);
844
+ // v0.8.0 Phase 5 frameProcessor prop semantics:
845
+ //
846
+ // - Host supplied? use host's processor; lib's first-party
847
+ // stitching is DISABLED in non-AR mode (vc accepts only one
848
+ // processor). One-shot console.info documents the tradeoff
849
+ // so the host isn't surprised by "panorama capture stopped
850
+ // producing output" in non-AR mode. AR-mode capture is
851
+ // unaffected the AR-session dispatch path fans out to BOTH
852
+ // first-party and host worklets independently.
853
+ //
854
+ // - No host processor? → use `fpDriver.frameProcessor` which is
855
+ // the lib's internal worklet driving first-party stitching
856
+ // via `useFrameProcessorDriver`. Default behaviour for the
857
+ // common "I just want panorama capture" case.
858
+ //
859
+ // The pre-v0.8.0 behaviour (host's prop silently ignored with
860
+ // a warning) is gone — Phase 5 plumbs the prop through. The
861
+ // tradeoff is honestly documented in the CameraProps docstring.
862
+ const hostFrameProcessorAcceptedWarnedRef = useRef(false);
802
863
  if (
803
864
  hostFrameProcessor != null
804
- && !hostFrameProcessorIgnoredWarnedRef.current
865
+ && !hostFrameProcessorAcceptedWarnedRef.current
805
866
  ) {
806
- hostFrameProcessorIgnoredWarnedRef.current = true;
867
+ hostFrameProcessorAcceptedWarnedRef.current = true;
807
868
  // eslint-disable-next-line no-console
808
- console.warn(
809
- '[react-native-image-stitcher] The `frameProcessor` prop on '
810
- + '<Camera> is ignored the SDK installs its own worklet '
811
- + 'via useFrameProcessorDriver. Remove the prop, or fork '
812
- + 'the SDK if you genuinely need a custom worklet.',
869
+ console.info(
870
+ '[react-native-image-stitcher] Host frameProcessor supplied '
871
+ + 'non-AR mode will run YOUR worklet instead of the lib\'s '
872
+ + 'first-party stitching plugin (vc accepts only one frame '
873
+ + 'processor). Non-AR panorama capture will not produce '
874
+ + 'stitched output until this prop is removed. AR-mode '
875
+ + 'capture is unaffected (AR-session dispatch fans out to '
876
+ + 'both first-party and host worklets independently).',
813
877
  );
814
878
  }
815
879
  // The Frame Processor worklet bound to vision-camera's Camera.
816
- const effectiveFrameProcessor = fpDriver.frameProcessor;
880
+ // Host's wins if supplied; lib's internal driver otherwise.
881
+ const effectiveFrameProcessor = hostFrameProcessor ?? fpDriver.frameProcessor;
817
882
 
818
883
  // ── Subscribe to engine state for live keyframe thumbs ──────────
819
884
  useEffect(() => {
package/src/index.ts CHANGED
@@ -182,6 +182,22 @@ export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
182
182
  // caveat). Foundation for plugin-pattern host features (OCR per
183
183
  // keyframe, packet detection, server-side analysis, etc.).
184
184
  export { useKeyframeStream } from './stitching/useKeyframeStream';
185
+ // v0.8.0 — unified frame contract for the worklet processor. Same
186
+ // JS-visible shape regardless of capture mode (AR vs non-AR).
187
+ export type {
188
+ StitcherFrame,
189
+ StitcherFrameProcessor,
190
+ ARAnchor,
191
+ } from './stitching/StitcherFrame';
192
+ // v0.8.0 Phase 4a — public host-worklet hook. Hosts that want a
193
+ // per-frame callback (OCR overlay, packet detection, ML inference)
194
+ // use this to attach a `'worklet'`-prefixed function that fires
195
+ // on the camera producer thread. Non-AR mode is fully wired
196
+ // today via vision-camera passthrough; AR-mode dispatch is
197
+ // API-stable but registration-only until Phase 4b lands the
198
+ // cross-runtime handoff (the AR runtime iterating the registry).
199
+ // See the hook's docstring + StitcherFrame.ts for the contract.
200
+ export { useFrameProcessor } from './stitching/useFrameProcessor';
185
201
  // vision-camera Frame Processor driver for non-AR captures. As
186
202
  // of v0.6 the only non-AR driver exported (the legacy
187
203
  // `useIncrementalJSDriver` was removed; was deprecated in v0.5).
@@ -0,0 +1,197 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+
3
+ /**
4
+ * v0.8.0 — unified frame contract for the lib's worklet processor.
5
+ *
6
+ * Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
7
+ * this directory) receive a `StitcherFrame` regardless of capture mode.
8
+ * The lib-owned worklet runtime guarantees the same JS-visible shape
9
+ * whether the underlying source is a vision-camera `Frame` (non-AR
10
+ * mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
11
+ * `Frame` (AR mode, sourced from a lib-managed delegate that the AR
12
+ * worklet runtime drives).
13
+ *
14
+ * ## Why structural (NOT `extends Frame`)
15
+ *
16
+ * vision-camera's iOS `Frame` is `CMSampleBufferRef`-shaped; ARFrame's
17
+ * `capturedImage` (a `CVPixelBufferRef`) can be wrapped into one
18
+ * (Phase-0 audit confirmed the iOS path). But vision-camera's
19
+ * **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
20
+ * ARCore does NOT produce `ImageProxy` instances. Forcing
21
+ * `StitcherFrame extends Frame` would either (a) require reverse-
22
+ * engineering ImageProxy on Android (intractable + fragile), or
23
+ * (b) make the type asymmetric per platform. Both are worse than
24
+ * making `StitcherFrame` a structural sibling type that vc Frames
25
+ * happen to satisfy (because vc Frames carry the same width / height /
26
+ * orientation / pixelFormat / timestamp / toArrayBuffer surface).
27
+ *
28
+ * The `__source: 'vc' | 'ar'` discriminator lets worklets gate on
29
+ * mode without a typeof / try-catch dance — e.g., skip work that
30
+ * needs AR tracking state when the source is `'vc'`.
31
+ *
32
+ * ## Buffer lifetime
33
+ *
34
+ * The underlying camera buffer (CMSampleBufferRef / ImageProxy /
35
+ * ARFrame.capturedImage) is valid only for the duration of the worklet
36
+ * call. Worklets that need to retain frame data MUST copy
37
+ * synchronously inside the worklet body (via `toArrayBuffer()` or via
38
+ * a JPEG-encode frame-processor plugin). Returning a reference and
39
+ * reading it later will read into freed memory.
40
+ */
41
+ export interface StitcherFrame {
42
+ // ── vision-camera-shaped fields (structural compat) ─────────────
43
+ // Worklets written against a vc `Frame` work unchanged against a
44
+ // `StitcherFrame` (the fields below are a strict subset of vc
45
+ // Frame's JS-visible surface).
46
+
47
+ /** Pixel width of the camera image. */
48
+ width: number;
49
+
50
+ /** Pixel height of the camera image. */
51
+ height: number;
52
+
53
+ /**
54
+ * Pixel format identifier. Both modes today emit `'yuv'` (NV12 on
55
+ * iOS, NV21 on Android). Other vision-camera formats may appear
56
+ * in future releases.
57
+ *
58
+ * **`'unknown'` semantics:** the lib reached a code path that
59
+ * doesn't recognise the underlying camera buffer's pixel format
60
+ * (e.g., a future ARKit version emits BGRA when historically it
61
+ * only emitted NV12). Worklets that depend on a known layout
62
+ * should treat `'unknown'` as "skip this frame". `toArrayBuffer()`
63
+ * still returns bytes when the format is `'unknown'`, but the
64
+ * layout is undefined — the bytes are the underlying buffer's
65
+ * first plane and may not be interpretable. When this happens
66
+ * the native side also emits an `os_log` / logcat warning.
67
+ */
68
+ pixelFormat: 'yuv' | 'rgb' | 'unknown';
69
+
70
+ /**
71
+ * Display orientation tag, matching vision-camera's
72
+ * `Frame.orientation`.
73
+ *
74
+ * **AR-mode limitation (v0.8.0):** AR-source frames return only
75
+ * the coarse two-value set `'landscape-right' | 'portrait'` (the
76
+ * lib reads `pose.imageWidth >= pose.imageHeight` as the
77
+ * discriminator since ARKit's `capturedImage` is always in the
78
+ * camera's native landscape-right orientation regardless of
79
+ * device pose). Worklets that need to distinguish
80
+ * `landscape-left` (upside-down landscape) or
81
+ * `portrait-upside-down` should consult device-orientation sensors
82
+ * separately while running in AR mode. Non-AR frames (vc source)
83
+ * return the full four-value set. Fixing the AR side requires
84
+ * threading `UIDevice.current.orientation` through; deferred to
85
+ * v0.8.1+ unless a consumer hits it.
86
+ */
87
+ orientation:
88
+ | 'portrait'
89
+ | 'portrait-upside-down'
90
+ | 'landscape-left'
91
+ | 'landscape-right';
92
+
93
+ /**
94
+ * Monotonic timestamp in **nanoseconds** (matches vision-camera's
95
+ * `Frame.timestamp` convention). Use timestamp deltas for
96
+ * inter-frame timing; the absolute value is implementation-defined
97
+ * and not comparable to `Date.now()`.
98
+ */
99
+ timestamp: number;
100
+
101
+ /**
102
+ * Copies the underlying pixel buffer into a JSI `ArrayBuffer`.
103
+ * Worklet-callable. Allocates O(width × height × bytesPerPixel)
104
+ * each call — avoid in tight inner loops; prefer plugin-side
105
+ * processing where possible.
106
+ */
107
+ toArrayBuffer(): ArrayBuffer;
108
+
109
+ // ── Lib additions ─────────────────────────────────────────────
110
+
111
+ /**
112
+ * Camera pose at frame-capture time. Always present.
113
+ *
114
+ * Rotation quaternion order is `(x, y, z, w)`; the lib uses
115
+ * `q = q_yaw * q_pitch * q_roll` throughout the engine + sensor
116
+ * fusion. Same convention surfaced by the v0.7.0
117
+ * `AcceptedKeyframe.pose` field.
118
+ *
119
+ * Translation is metres in world coordinates. Populated by AR
120
+ * mode (real ARKit / ARCore camera transform); undefined in
121
+ * non-AR mode (gyro provides only rotation — no spatial anchor).
122
+ */
123
+ pose: {
124
+ rotation: [number, number, number, number];
125
+ translation?: [number, number, number];
126
+ };
127
+
128
+ /**
129
+ * Discriminator for the frame source. Worklets branch on this to
130
+ * gate AR-only field access without try/catch. Standard TS
131
+ * discriminated-union pattern.
132
+ *
133
+ * - `'vc'` — vision-camera Frame Processor (non-AR mode)
134
+ * - `'ar'` — AR-session frame (AR mode); `arDepth` / `arAnchors` /
135
+ * `arTrackingState` fields may be populated
136
+ */
137
+ source: 'vc' | 'ar';
138
+
139
+ // ── AR-only optional fields ───────────────────────────────────
140
+ // Always undefined in `__source === 'vc'` mode.
141
+
142
+ /**
143
+ * Depth data when available — AR mode + a device that supports
144
+ * the AR framework's depth API (iPhone Pro LiDAR; ARCore Depth
145
+ * API on supported Android devices).
146
+ *
147
+ * Resolution is typically lower than the camera image (e.g.,
148
+ * 256×192 on iPhone Pro LiDAR). `confidenceMap` is per-pixel:
149
+ * `0` = low, `1` = medium, `2` = high confidence. `Float32`
150
+ * depth in metres; `Uint8` confidence.
151
+ */
152
+ arDepth?: {
153
+ width: number;
154
+ height: number;
155
+ depthMap: ArrayBuffer;
156
+ confidenceMap?: ArrayBuffer;
157
+ };
158
+
159
+ /**
160
+ * Tracked AR anchors visible in this frame. Empty array if AR
161
+ * is active but no anchors are tracked. Undefined in non-AR mode.
162
+ */
163
+ arAnchors?: ARAnchor[];
164
+
165
+ /**
166
+ * AR tracking quality. Worklets that should skip work when
167
+ * tracking is degraded check this. Undefined in non-AR mode.
168
+ */
169
+ arTrackingState?: 'notAvailable' | 'limited' | 'normal';
170
+ }
171
+
172
+ /**
173
+ * v0.8.0 — public AR anchor type. Subset of ARKit/ARCore anchor info
174
+ * exposed to JS worklets. Extend with plane-extent / image-name
175
+ * fields as the JSI binding learns them.
176
+ */
177
+ export interface ARAnchor {
178
+ /** Stable per-session anchor identifier. */
179
+ id: string;
180
+ /** Anchor kind. `'point'` is Android (ARCore) only. */
181
+ type: 'plane' | 'image' | 'point';
182
+ /**
183
+ * 4×4 row-major transform from anchor space to world space.
184
+ * 16 numbers.
185
+ */
186
+ transform: number[];
187
+ }
188
+
189
+ /**
190
+ * v0.8.0 — worklet function signature for the unified frame processor.
191
+ *
192
+ * Must be a `'worklet'`-prefixed function (so it can run on the
193
+ * worklet runtime). Receives a `StitcherFrame` per camera frame; the
194
+ * return value is ignored (use `runOnJS` / shared values to surface
195
+ * results back to the JS thread).
196
+ */
197
+ export type StitcherFrameProcessor = (frame: StitcherFrame) => void;
@@ -0,0 +1,156 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+
3
+ import type { StitcherFrameProcessor } from './StitcherFrame';
4
+
5
+ /**
6
+ * v0.8.0 Phase 4a — process-scope registry of host-supplied worklets
7
+ * that the v0.8.0 `useFrameProcessor` hook registers into.
8
+ *
9
+ * ## What this is (Phase 4a)
10
+ *
11
+ * A plain JS singleton holding an ordered list of registered
12
+ * worklets. Hosts mount the `useFrameProcessor` hook (in this
13
+ * directory); the hook registers its worklet into this singleton
14
+ * on mount and unregisters on unmount. Each entry carries:
15
+ *
16
+ * - `id`: stable identifier issued by `register`; passed to
17
+ * `unregister`.
18
+ * - `worklet`: the host's `StitcherFrameProcessor` function.
19
+ * MUST be `'worklet'`-prefixed at the call site (TS can't
20
+ * enforce that — convention).
21
+ * - `isFirstParty`: `false` for host-supplied worklets;
22
+ * reserved for the lib's own first-party stitching path which
23
+ * today is wired natively (not through this registry).
24
+ *
25
+ * Order is stable: first-party entries (none in Phase 4a) come
26
+ * first, then host entries by registration order. Re-registration
27
+ * of the same worklet by identity yields a new entry — hosts that
28
+ * re-render and call `register` again ARE responsible for calling
29
+ * `unregister` first. The `useFrameProcessor` hook handles this
30
+ * via its `deps` dependency array.
31
+ *
32
+ * ## What this is NOT (Phase 4b)
33
+ *
34
+ * **The native AR worklet runtime does NOT yet read this registry.**
35
+ * Worklets registered here for AR-mode captures will not fire
36
+ * until Phase 4b lands the cross-runtime handoff (a
37
+ * worklets-core `SharedValue` mirror that `RNSARWorkletRuntime`
38
+ * reads on each `dispatchFrame:pose:` call; the runtime then
39
+ * constructs a `StitcherFrameHostObject` + invokes each
40
+ * registered worklet via `RNWorklet::WorkletInvoker::call`).
41
+ *
42
+ * In non-AR mode the host-supplied worklet IS invoked, but via
43
+ * vision-camera's Frame Processor runtime directly (the
44
+ * `useFrameProcessor` hook returns vc's processor object which
45
+ * `<Camera>` passes to vision-camera). So Phase 4a's public API
46
+ * is fully functional for non-AR; AR is API-stable but
47
+ * runtime-deferred.
48
+ *
49
+ * ## Singleton lifetime
50
+ *
51
+ * The registry is a module-level instance. It lives for the
52
+ * lifetime of the JS runtime (= until app reload). Entries
53
+ * accumulate only via `register` and shed only via `unregister`
54
+ * — no GC / weak-ref logic. Hosts that mount `useFrameProcessor`
55
+ * inside React components MUST rely on the hook's effect cleanup
56
+ * to unregister on unmount, or they'll leak entries until
57
+ * reload. The hook handles this correctly today.
58
+ *
59
+ * ## Why a singleton (vs context provider)
60
+ *
61
+ * The native AR worklet runtime is itself a process-scope
62
+ * singleton (`RNSARWorkletRuntime`, `StitcherWorkletRuntime`).
63
+ * The Phase 4b handoff between TS and native is necessarily
64
+ * process-scope. Wrapping the registry in a React context
65
+ * would force every consumer to be in the same provider tree
66
+ * which is friction for layer-2 hosts that compose
67
+ * `<ARCameraView>` / `useIncrementalStitcher` themselves. The
68
+ * singleton is the right shape; the React-level ergonomics are
69
+ * provided by the `useFrameProcessor` hook.
70
+ */
71
+ export interface StitcherWorkletEntry {
72
+ readonly id: string;
73
+ readonly worklet: StitcherFrameProcessor;
74
+ readonly isFirstParty: boolean;
75
+ }
76
+
77
+ class Registry {
78
+ private entries: StitcherWorkletEntry[] = [];
79
+ private nextHostCounter = 0;
80
+
81
+ /**
82
+ * Register a worklet. Returns a stable ID for `unregister`.
83
+ *
84
+ * Entries are appended in registration order; first-party
85
+ * entries (if any are added in future) sort to the front.
86
+ */
87
+ register(opts: {
88
+ worklet: StitcherFrameProcessor;
89
+ isFirstParty?: boolean;
90
+ }): string {
91
+ const isFirstParty = opts.isFirstParty ?? false;
92
+ const id = isFirstParty
93
+ ? `fp-${this.nextHostCounter++}`
94
+ : `host-${this.nextHostCounter++}`;
95
+ const entry: StitcherWorkletEntry = {
96
+ id,
97
+ worklet: opts.worklet,
98
+ isFirstParty,
99
+ };
100
+ this.entries.push(entry);
101
+ // Re-sort so first-party always runs before host entries.
102
+ // Stable sort: registration order is preserved within each
103
+ // partition. Single-pass O(n log n) is fine — registration
104
+ // is rare (per-`<Camera>`-mount, not per-frame).
105
+ this.entries.sort((a, b) => {
106
+ if (a.isFirstParty !== b.isFirstParty) {
107
+ return a.isFirstParty ? -1 : 1;
108
+ }
109
+ return 0;
110
+ });
111
+ return id;
112
+ }
113
+
114
+ /**
115
+ * Remove a previously-registered worklet by ID. No-op if the ID
116
+ * isn't found. Hosts call this in their effect's cleanup.
117
+ */
118
+ unregister(id: string): void {
119
+ this.entries = this.entries.filter((e) => e.id !== id);
120
+ }
121
+
122
+ /**
123
+ * Snapshot the current entries. Returned array is a copy —
124
+ * mutations don't affect the registry. Phase 4b's native
125
+ * handoff will read a `SharedValue` mirror of this list so the
126
+ * AR runtime doesn't need a JS-thread hop on the hot per-frame
127
+ * path; for Phase 4a this method is the JS-side accessor.
128
+ */
129
+ getEntries(): readonly StitcherWorkletEntry[] {
130
+ return [...this.entries];
131
+ }
132
+
133
+ /**
134
+ * Total number of registered worklets (first-party + host).
135
+ * Useful for diagnostics + tests.
136
+ */
137
+ get count(): number {
138
+ return this.entries.length;
139
+ }
140
+
141
+ /**
142
+ * Test-only — clear all entries. NOT exported from
143
+ * `src/index.ts`. Used in unit tests to reset state between
144
+ * cases.
145
+ */
146
+ _resetForTests(): void {
147
+ this.entries = [];
148
+ this.nextHostCounter = 0;
149
+ }
150
+ }
151
+
152
+ /**
153
+ * Process-scope singleton. Imported by `useFrameProcessor` (in
154
+ * this directory) + by the Phase 4b native-handoff code (TBD).
155
+ */
156
+ export const StitcherWorkletRegistry = new Registry();