react-native-image-stitcher 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +122 -0
- package/android/build.gradle +35 -1
- package/android/src/main/cpp/CMakeLists.txt +64 -2
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +227 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +30 -11
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +4 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +78 -3
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +103 -0
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +256 -0
- package/android/src/main/java/io/imagestitcher/rn/TransferredNV21.kt +100 -0
- package/cpp/stitcher_frame_data.hpp +141 -0
- package/cpp/stitcher_frame_jsi.cpp +214 -0
- package/cpp/stitcher_frame_jsi.hpp +108 -0
- package/cpp/stitcher_proxy_jsi.cpp +109 -0
- package/cpp/stitcher_proxy_jsi.hpp +46 -0
- package/cpp/stitcher_worklet_dispatch.cpp +103 -0
- package/cpp/stitcher_worklet_dispatch.hpp +71 -0
- package/cpp/stitcher_worklet_registry.cpp +81 -0
- package/cpp/stitcher_worklet_registry.hpp +136 -0
- package/dist/camera/Camera.d.ts +62 -12
- package/dist/camera/Camera.js +30 -15
- package/dist/index.d.ts +2 -0
- package/dist/index.js +11 -1
- package/dist/stitching/StitcherFrame.d.ts +170 -0
- package/dist/stitching/StitcherFrame.js +4 -0
- package/dist/stitching/StitcherWorkletRegistry.d.ts +117 -0
- package/dist/stitching/StitcherWorkletRegistry.js +78 -0
- package/dist/stitching/ensureStitcherProxyInstalled.d.ts +8 -0
- package/dist/stitching/ensureStitcherProxyInstalled.js +81 -0
- package/dist/stitching/useFrameProcessor.d.ts +119 -0
- package/dist/stitching/useFrameProcessor.js +196 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +46 -10
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +128 -0
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +313 -0
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +60 -0
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +214 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +42 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +103 -0
- package/package.json +1 -1
- package/src/camera/Camera.tsx +93 -28
- package/src/index.ts +16 -0
- package/src/stitching/StitcherFrame.ts +197 -0
- package/src/stitching/StitcherWorkletRegistry.ts +156 -0
- package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +176 -0
- package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +94 -0
- package/src/stitching/ensureStitcherProxyInstalled.ts +141 -0
- package/src/stitching/useFrameProcessor.ts +226 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// StitcherJsiInstaller.mm — implementation. Installs
|
|
4
|
+
// `globalThis.__stitcherProxy` on the main JS runtime.
|
|
5
|
+
//
|
|
6
|
+
// ## Why a host object rather than two globalThis functions
|
|
7
|
+
//
|
|
8
|
+
// We could install `__stitcherProxy_install` + `__stitcherProxy_uninstall`
|
|
9
|
+
// directly on `globalThis`. Wrapping them in a host object is
|
|
10
|
+
// slightly more code but:
|
|
11
|
+
// - Namespaces the proxy under a single global property
|
|
12
|
+
// (easier to feature-detect; one `if (globalThis.__stitcherProxy)`
|
|
13
|
+
// instead of two).
|
|
14
|
+
// - Matches vc's pattern (`global.VisionCameraProxy`), so future
|
|
15
|
+
// readers recognise the shape.
|
|
16
|
+
// - Keeps room to grow (e.g., add `__stitcherProxy.snapshot()` for
|
|
17
|
+
// diagnostics) without polluting globalThis further.
|
|
18
|
+
|
|
19
|
+
#import "StitcherJsiInstaller.h"
|
|
20
|
+
|
|
21
|
+
#import <Foundation/Foundation.h>
|
|
22
|
+
#import <React/RCTBridge.h>
|
|
23
|
+
#import <React/RCTBridge+Private.h>
|
|
24
|
+
#import <React/RCTUtils.h>
|
|
25
|
+
#import <os/log.h>
|
|
26
|
+
|
|
27
|
+
#include <jsi/jsi.h>
|
|
28
|
+
|
|
29
|
+
#include "stitcher_proxy_jsi.hpp"
|
|
30
|
+
|
|
31
|
+
using namespace facebook;
|
|
32
|
+
|
|
33
|
+
// The host object class + install logic moved to shared C++ in
|
|
34
|
+
// `cpp/stitcher_proxy_jsi.{hpp,cpp}` (v0.8.0 Phase 4b.ii). The
|
|
35
|
+
// Android JNI installer reuses the same `install` / `uninstall` /
|
|
36
|
+
// `count` host functions verbatim — the JSI dispatch is identical
|
|
37
|
+
// across platforms (matches the StitcherFrame host object's design).
|
|
38
|
+
|
|
39
|
+
#pragma mark - RN module
|
|
40
|
+
|
|
41
|
+
@implementation StitcherJsiInstaller
|
|
42
|
+
|
|
43
|
+
// RN injects `_bridge` at module init (legacy bridge → RCTBridge*;
|
|
44
|
+
// bridgeless / new arch → RCTBridgeProxy*, which forwards `runtime`
|
|
45
|
+
// access via NSProxy `forwardInvocation:`). Using the injected
|
|
46
|
+
// `_bridge` instead of `[RCTBridge currentBridge]` is the
|
|
47
|
+
// bridgeless-compatible idiom — `currentBridge` is nil under new
|
|
48
|
+
// arch. Pattern lifted from `react-native-worklets-core/ios/Worklets.mm`.
|
|
49
|
+
@synthesize bridge = _bridge;
|
|
50
|
+
|
|
51
|
+
RCT_EXPORT_MODULE()
|
|
52
|
+
|
|
53
|
+
+ (BOOL)requiresMainQueueSetup {
|
|
54
|
+
return YES;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
- (void)setBridge:(RCTBridge*)bridge {
|
|
58
|
+
_bridge = bridge;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Synchronous install method. JS calls this once at lib bootstrap
|
|
62
|
+
// to install the global proxy on the main JS runtime. Returns
|
|
63
|
+
// `@YES` on success or `@NO` if the JSI runtime wasn't reachable
|
|
64
|
+
// (remote debug mode pre-Hermes; bridge not yet ready; etc.).
|
|
65
|
+
//
|
|
66
|
+
// `RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD` is the documented
|
|
67
|
+
// pattern for "run native code synchronously on the JS thread to
|
|
68
|
+
// install JSI bindings." Same pattern worklets-core + vision-camera
|
|
69
|
+
// use for their installs.
|
|
70
|
+
//
|
|
71
|
+
// **Bridgeless mode:** `_bridge` is an `RCTBridgeProxy` (NSProxy
|
|
72
|
+
// subclass) that forwards `-runtime` / `-jsCallInvoker` invocations
|
|
73
|
+
// to the underlying RCTHost-backed runtime. The `(RCTCxxBridge*)`
|
|
74
|
+
// cast is a no-op at runtime (NSProxy ignores static type) but
|
|
75
|
+
// keeps the Obj-C compiler happy about property access.
|
|
76
|
+
RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(install) {
|
|
77
|
+
if (_bridge == nil) {
|
|
78
|
+
os_log_error(OS_LOG_DEFAULT,
|
|
79
|
+
"[StitcherJsiInstaller] _bridge is nil; the module was "
|
|
80
|
+
"instantiated without bridge injection. Cannot install "
|
|
81
|
+
"__stitcherProxy.");
|
|
82
|
+
return @NO;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
RCTCxxBridge* cxxBridge = (RCTCxxBridge*)_bridge;
|
|
86
|
+
if (cxxBridge.runtime == nullptr) {
|
|
87
|
+
os_log_error(OS_LOG_DEFAULT,
|
|
88
|
+
"[StitcherJsiInstaller] _bridge.runtime is nullptr; the JS "
|
|
89
|
+
"runtime hasn't been initialized yet OR remote debugger is "
|
|
90
|
+
"attached. Cannot install __stitcherProxy.");
|
|
91
|
+
return @NO;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
jsi::Runtime& runtime = *(jsi::Runtime*)cxxBridge.runtime;
|
|
95
|
+
retailens::installStitcherProxy(runtime);
|
|
96
|
+
|
|
97
|
+
os_log_info(OS_LOG_DEFAULT,
|
|
98
|
+
"[StitcherJsiInstaller] installed globalThis.__stitcherProxy "
|
|
99
|
+
"on main JS runtime.");
|
|
100
|
+
return @YES;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
@end
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "react-native-image-stitcher",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
package/src/camera/Camera.tsx
CHANGED
|
@@ -294,21 +294,71 @@ export interface CameraProps {
|
|
|
294
294
|
onError?: (err: CameraError) => void;
|
|
295
295
|
|
|
296
296
|
/**
|
|
297
|
-
* Optional vision-camera frame processor.
|
|
298
|
-
* non-AR preview (AR mode uses ARCameraView, which doesn't expose
|
|
299
|
-
* a worklet seam). Build the worklet on the host side with
|
|
300
|
-
* `useFrameProcessor` from `react-native-vision-camera`.
|
|
297
|
+
* Optional host-supplied vision-camera frame processor.
|
|
301
298
|
*
|
|
302
|
-
*
|
|
303
|
-
* `docs/f8-frame-processor-plan.md`.
|
|
299
|
+
* ## When to set this prop
|
|
304
300
|
*
|
|
305
|
-
*
|
|
306
|
-
* `
|
|
307
|
-
* a one-time `console.warn` — supplying a host worklet would
|
|
308
|
-
* race with the SDK's pixel-buffer feed. Either remove the prop
|
|
309
|
-
* or fork the SDK if you genuinely need a custom worklet.
|
|
301
|
+
* v0.8.0+ canonical answer: use the lib's own `useFrameProcessor`
|
|
302
|
+
* hook, NOT `react-native-vision-camera`'s. The lib's hook:
|
|
310
303
|
*
|
|
311
|
-
* AR mode
|
|
304
|
+
* - **AR mode**: auto-registers the worklet in the native
|
|
305
|
+
* `__stitcherProxy` registry; the AR session's per-frame
|
|
306
|
+
* dispatch fans out to it alongside the lib's first-party
|
|
307
|
+
* stitching. No prop wiring needed — just mount the hook
|
|
308
|
+
* anywhere in the tree.
|
|
309
|
+
* - **Non-AR mode**: returns a vc processor object that this
|
|
310
|
+
* prop accepts. Wiring it through enables the host's
|
|
311
|
+
* worklet to fire on vc's Frame Processor runtime.
|
|
312
|
+
*
|
|
313
|
+
* ```tsx
|
|
314
|
+
* import { Camera, useFrameProcessor, type StitcherFrame }
|
|
315
|
+
* from 'react-native-image-stitcher';
|
|
316
|
+
*
|
|
317
|
+
* function MyScreen() {
|
|
318
|
+
* const fp = useFrameProcessor((frame: StitcherFrame) => {
|
|
319
|
+
* 'worklet';
|
|
320
|
+
* // ...
|
|
321
|
+
* }, []);
|
|
322
|
+
* return <Camera frameProcessor={fp} ... />;
|
|
323
|
+
* }
|
|
324
|
+
* ```
|
|
325
|
+
*
|
|
326
|
+
* ## Non-AR mode tradeoff (HONEST)
|
|
327
|
+
*
|
|
328
|
+
* vision-camera's `<Camera>` accepts ONLY ONE frame processor.
|
|
329
|
+
* The lib's internal `useFrameProcessorDriver` produces the
|
|
330
|
+
* processor that drives first-party panorama stitching in non-AR
|
|
331
|
+
* mode. If you supply your own via this prop, **the lib's
|
|
332
|
+
* first-party stitching is replaced** — panorama capture in
|
|
333
|
+
* non-AR mode will not produce stitched output until you remove
|
|
334
|
+
* the prop or fork the SDK to compose both worklets manually.
|
|
335
|
+
*
|
|
336
|
+
* For the common case (host wants worklet + lib wants stitching
|
|
337
|
+
* concurrently), prefer AR mode: the AR-mode path natively fans
|
|
338
|
+
* out to both the lib's first-party stitching AND every
|
|
339
|
+
* registered host worklet on every frame, with per-worklet
|
|
340
|
+
* failure isolation.
|
|
341
|
+
*
|
|
342
|
+
* Composition for non-AR mode (lib stitching + host worklet on
|
|
343
|
+
* the same vc processor) is tracked as a v0.9+ follow-up;
|
|
344
|
+
* needs the lib's first-party logic exposed as a vc Frame
|
|
345
|
+
* Processor plugin the host's worklet can call.
|
|
346
|
+
*
|
|
347
|
+
* ## AR mode behaviour
|
|
348
|
+
*
|
|
349
|
+
* In AR mode (`defaultCaptureSource="ar"` or runtime-toggled),
|
|
350
|
+
* vc's `<Camera>` isn't mounted; this prop has no effect.
|
|
351
|
+
* Host worklets registered via the lib's `useFrameProcessor`
|
|
352
|
+
* fire automatically through the AR-session dispatch path
|
|
353
|
+
* (iOS Phase 4b.i / Android Phase 4b.iii).
|
|
354
|
+
*
|
|
355
|
+
* ## Backwards compatibility
|
|
356
|
+
*
|
|
357
|
+
* The pre-v0.8.0 behaviour (warn + ignore) is preserved when the
|
|
358
|
+
* supplied processor is recognisably from
|
|
359
|
+
* `react-native-vision-camera`'s `useFrameProcessor` directly
|
|
360
|
+
* (no `__stitcherFrame` marker). Hosts should migrate to the
|
|
361
|
+
* lib's `useFrameProcessor` to benefit from AR-mode dispatch.
|
|
312
362
|
*
|
|
313
363
|
* (v0.5 had a `legacyDriver` escape hatch that routed back to
|
|
314
364
|
* `useIncrementalJSDriver`. That hook + prop were removed in
|
|
@@ -791,29 +841,44 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
791
841
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
792
842
|
useEffect(() => () => { fpDriver.stop(); }, []);
|
|
793
843
|
|
|
794
|
-
//
|
|
795
|
-
//
|
|
796
|
-
//
|
|
797
|
-
//
|
|
798
|
-
//
|
|
799
|
-
//
|
|
800
|
-
//
|
|
801
|
-
|
|
844
|
+
// v0.8.0 Phase 5 — frameProcessor prop semantics:
|
|
845
|
+
//
|
|
846
|
+
// - Host supplied? → use host's processor; lib's first-party
|
|
847
|
+
// stitching is DISABLED in non-AR mode (vc accepts only one
|
|
848
|
+
// processor). One-shot console.info documents the tradeoff
|
|
849
|
+
// so the host isn't surprised by "panorama capture stopped
|
|
850
|
+
// producing output" in non-AR mode. AR-mode capture is
|
|
851
|
+
// unaffected — the AR-session dispatch path fans out to BOTH
|
|
852
|
+
// first-party and host worklets independently.
|
|
853
|
+
//
|
|
854
|
+
// - No host processor? → use `fpDriver.frameProcessor` which is
|
|
855
|
+
// the lib's internal worklet driving first-party stitching
|
|
856
|
+
// via `useFrameProcessorDriver`. Default behaviour for the
|
|
857
|
+
// common "I just want panorama capture" case.
|
|
858
|
+
//
|
|
859
|
+
// The pre-v0.8.0 behaviour (host's prop silently ignored with
|
|
860
|
+
// a warning) is gone — Phase 5 plumbs the prop through. The
|
|
861
|
+
// tradeoff is honestly documented in the CameraProps docstring.
|
|
862
|
+
const hostFrameProcessorAcceptedWarnedRef = useRef(false);
|
|
802
863
|
if (
|
|
803
864
|
hostFrameProcessor != null
|
|
804
|
-
&& !
|
|
865
|
+
&& !hostFrameProcessorAcceptedWarnedRef.current
|
|
805
866
|
) {
|
|
806
|
-
|
|
867
|
+
hostFrameProcessorAcceptedWarnedRef.current = true;
|
|
807
868
|
// eslint-disable-next-line no-console
|
|
808
|
-
console.
|
|
809
|
-
'[react-native-image-stitcher]
|
|
810
|
-
+ '
|
|
811
|
-
+ '
|
|
812
|
-
+ '
|
|
869
|
+
console.info(
|
|
870
|
+
'[react-native-image-stitcher] Host frameProcessor supplied — '
|
|
871
|
+
+ 'non-AR mode will run YOUR worklet instead of the lib\'s '
|
|
872
|
+
+ 'first-party stitching plugin (vc accepts only one frame '
|
|
873
|
+
+ 'processor). Non-AR panorama capture will not produce '
|
|
874
|
+
+ 'stitched output until this prop is removed. AR-mode '
|
|
875
|
+
+ 'capture is unaffected (AR-session dispatch fans out to '
|
|
876
|
+
+ 'both first-party and host worklets independently).',
|
|
813
877
|
);
|
|
814
878
|
}
|
|
815
879
|
// The Frame Processor worklet bound to vision-camera's Camera.
|
|
816
|
-
|
|
880
|
+
// Host's wins if supplied; lib's internal driver otherwise.
|
|
881
|
+
const effectiveFrameProcessor = hostFrameProcessor ?? fpDriver.frameProcessor;
|
|
817
882
|
|
|
818
883
|
// ── Subscribe to engine state for live keyframe thumbs ──────────
|
|
819
884
|
useEffect(() => {
|
package/src/index.ts
CHANGED
|
@@ -182,6 +182,22 @@ export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
|
|
|
182
182
|
// caveat). Foundation for plugin-pattern host features (OCR per
|
|
183
183
|
// keyframe, packet detection, server-side analysis, etc.).
|
|
184
184
|
export { useKeyframeStream } from './stitching/useKeyframeStream';
|
|
185
|
+
// v0.8.0 — unified frame contract for the worklet processor. Same
|
|
186
|
+
// JS-visible shape regardless of capture mode (AR vs non-AR).
|
|
187
|
+
export type {
|
|
188
|
+
StitcherFrame,
|
|
189
|
+
StitcherFrameProcessor,
|
|
190
|
+
ARAnchor,
|
|
191
|
+
} from './stitching/StitcherFrame';
|
|
192
|
+
// v0.8.0 Phase 4a — public host-worklet hook. Hosts that want a
|
|
193
|
+
// per-frame callback (OCR overlay, packet detection, ML inference)
|
|
194
|
+
// use this to attach a `'worklet'`-prefixed function that fires
|
|
195
|
+
// on the camera producer thread. Non-AR mode is fully wired
|
|
196
|
+
// today via vision-camera passthrough; AR-mode dispatch is
|
|
197
|
+
// API-stable but registration-only until Phase 4b lands the
|
|
198
|
+
// cross-runtime handoff (the AR runtime iterating the registry).
|
|
199
|
+
// See the hook's docstring + StitcherFrame.ts for the contract.
|
|
200
|
+
export { useFrameProcessor } from './stitching/useFrameProcessor';
|
|
185
201
|
// vision-camera Frame Processor driver for non-AR captures. As
|
|
186
202
|
// of v0.6 the only non-AR driver exported (the legacy
|
|
187
203
|
// `useIncrementalJSDriver` was removed; was deprecated in v0.5).
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* v0.8.0 — unified frame contract for the lib's worklet processor.
|
|
5
|
+
*
|
|
6
|
+
* Worklets registered via the v0.8.0 `useFrameProcessor` hook (also in
|
|
7
|
+
* this directory) receive a `StitcherFrame` regardless of capture mode.
|
|
8
|
+
* The lib-owned worklet runtime guarantees the same JS-visible shape
|
|
9
|
+
* whether the underlying source is a vision-camera `Frame` (non-AR
|
|
10
|
+
* mode, sourced from the FP plugin) or an ARKit `ARFrame` / ARCore
|
|
11
|
+
* `Frame` (AR mode, sourced from a lib-managed delegate that the AR
|
|
12
|
+
* worklet runtime drives).
|
|
13
|
+
*
|
|
14
|
+
* ## Why structural (NOT `extends Frame`)
|
|
15
|
+
*
|
|
16
|
+
* vision-camera's iOS `Frame` is `CMSampleBufferRef`-shaped; ARFrame's
|
|
17
|
+
* `capturedImage` (a `CVPixelBufferRef`) can be wrapped into one
|
|
18
|
+
* (Phase-0 audit confirmed the iOS path). But vision-camera's
|
|
19
|
+
* **Android** `Frame` is `androidx.camera.core.ImageProxy`-coupled —
|
|
20
|
+
* ARCore does NOT produce `ImageProxy` instances. Forcing
|
|
21
|
+
* `StitcherFrame extends Frame` would either (a) require reverse-
|
|
22
|
+
* engineering ImageProxy on Android (intractable + fragile), or
|
|
23
|
+
* (b) make the type asymmetric per platform. Both are worse than
|
|
24
|
+
* making `StitcherFrame` a structural sibling type that vc Frames
|
|
25
|
+
* happen to satisfy (because vc Frames carry the same width / height /
|
|
26
|
+
* orientation / pixelFormat / timestamp / toArrayBuffer surface).
|
|
27
|
+
*
|
|
28
|
+
* The `__source: 'vc' | 'ar'` discriminator lets worklets gate on
|
|
29
|
+
* mode without a typeof / try-catch dance — e.g., skip work that
|
|
30
|
+
* needs AR tracking state when the source is `'vc'`.
|
|
31
|
+
*
|
|
32
|
+
* ## Buffer lifetime
|
|
33
|
+
*
|
|
34
|
+
* The underlying camera buffer (CMSampleBufferRef / ImageProxy /
|
|
35
|
+
* ARFrame.capturedImage) is valid only for the duration of the worklet
|
|
36
|
+
* call. Worklets that need to retain frame data MUST copy
|
|
37
|
+
* synchronously inside the worklet body (via `toArrayBuffer()` or via
|
|
38
|
+
* a JPEG-encode frame-processor plugin). Returning a reference and
|
|
39
|
+
* reading it later will read into freed memory.
|
|
40
|
+
*/
|
|
41
|
+
export interface StitcherFrame {
|
|
42
|
+
// ── vision-camera-shaped fields (structural compat) ─────────────
|
|
43
|
+
// Worklets written against a vc `Frame` work unchanged against a
|
|
44
|
+
// `StitcherFrame` (the fields below are a strict subset of vc
|
|
45
|
+
// Frame's JS-visible surface).
|
|
46
|
+
|
|
47
|
+
/** Pixel width of the camera image. */
|
|
48
|
+
width: number;
|
|
49
|
+
|
|
50
|
+
/** Pixel height of the camera image. */
|
|
51
|
+
height: number;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Pixel format identifier. Both modes today emit `'yuv'` (NV12 on
|
|
55
|
+
* iOS, NV21 on Android). Other vision-camera formats may appear
|
|
56
|
+
* in future releases.
|
|
57
|
+
*
|
|
58
|
+
* **`'unknown'` semantics:** the lib reached a code path that
|
|
59
|
+
* doesn't recognise the underlying camera buffer's pixel format
|
|
60
|
+
* (e.g., a future ARKit version emits BGRA when historically it
|
|
61
|
+
* only emitted NV12). Worklets that depend on a known layout
|
|
62
|
+
* should treat `'unknown'` as "skip this frame". `toArrayBuffer()`
|
|
63
|
+
* still returns bytes when the format is `'unknown'`, but the
|
|
64
|
+
* layout is undefined — the bytes are the underlying buffer's
|
|
65
|
+
* first plane and may not be interpretable. When this happens
|
|
66
|
+
* the native side also emits an `os_log` / logcat warning.
|
|
67
|
+
*/
|
|
68
|
+
pixelFormat: 'yuv' | 'rgb' | 'unknown';
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Display orientation tag, matching vision-camera's
|
|
72
|
+
* `Frame.orientation`.
|
|
73
|
+
*
|
|
74
|
+
* **AR-mode limitation (v0.8.0):** AR-source frames return only
|
|
75
|
+
* the coarse two-value set `'landscape-right' | 'portrait'` (the
|
|
76
|
+
* lib reads `pose.imageWidth >= pose.imageHeight` as the
|
|
77
|
+
* discriminator since ARKit's `capturedImage` is always in the
|
|
78
|
+
* camera's native landscape-right orientation regardless of
|
|
79
|
+
* device pose). Worklets that need to distinguish
|
|
80
|
+
* `landscape-left` (upside-down landscape) or
|
|
81
|
+
* `portrait-upside-down` should consult device-orientation sensors
|
|
82
|
+
* separately while running in AR mode. Non-AR frames (vc source)
|
|
83
|
+
* return the full four-value set. Fixing the AR side requires
|
|
84
|
+
* threading `UIDevice.current.orientation` through; deferred to
|
|
85
|
+
* v0.8.1+ unless a consumer hits it.
|
|
86
|
+
*/
|
|
87
|
+
orientation:
|
|
88
|
+
| 'portrait'
|
|
89
|
+
| 'portrait-upside-down'
|
|
90
|
+
| 'landscape-left'
|
|
91
|
+
| 'landscape-right';
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Monotonic timestamp in **nanoseconds** (matches vision-camera's
|
|
95
|
+
* `Frame.timestamp` convention). Use timestamp deltas for
|
|
96
|
+
* inter-frame timing; the absolute value is implementation-defined
|
|
97
|
+
* and not comparable to `Date.now()`.
|
|
98
|
+
*/
|
|
99
|
+
timestamp: number;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Copies the underlying pixel buffer into a JSI `ArrayBuffer`.
|
|
103
|
+
* Worklet-callable. Allocates O(width × height × bytesPerPixel)
|
|
104
|
+
* each call — avoid in tight inner loops; prefer plugin-side
|
|
105
|
+
* processing where possible.
|
|
106
|
+
*/
|
|
107
|
+
toArrayBuffer(): ArrayBuffer;
|
|
108
|
+
|
|
109
|
+
// ── Lib additions ─────────────────────────────────────────────
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Camera pose at frame-capture time. Always present.
|
|
113
|
+
*
|
|
114
|
+
* Rotation quaternion order is `(x, y, z, w)`; the lib uses
|
|
115
|
+
* `q = q_yaw * q_pitch * q_roll` throughout the engine + sensor
|
|
116
|
+
* fusion. Same convention surfaced by the v0.7.0
|
|
117
|
+
* `AcceptedKeyframe.pose` field.
|
|
118
|
+
*
|
|
119
|
+
* Translation is metres in world coordinates. Populated by AR
|
|
120
|
+
* mode (real ARKit / ARCore camera transform); undefined in
|
|
121
|
+
* non-AR mode (gyro provides only rotation — no spatial anchor).
|
|
122
|
+
*/
|
|
123
|
+
pose: {
|
|
124
|
+
rotation: [number, number, number, number];
|
|
125
|
+
translation?: [number, number, number];
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Discriminator for the frame source. Worklets branch on this to
|
|
130
|
+
* gate AR-only field access without try/catch. Standard TS
|
|
131
|
+
* discriminated-union pattern.
|
|
132
|
+
*
|
|
133
|
+
* - `'vc'` — vision-camera Frame Processor (non-AR mode)
|
|
134
|
+
* - `'ar'` — AR-session frame (AR mode); `arDepth` / `arAnchors` /
|
|
135
|
+
* `arTrackingState` fields may be populated
|
|
136
|
+
*/
|
|
137
|
+
source: 'vc' | 'ar';
|
|
138
|
+
|
|
139
|
+
// ── AR-only optional fields ───────────────────────────────────
|
|
140
|
+
// Always undefined in `__source === 'vc'` mode.
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Depth data when available — AR mode + a device that supports
|
|
144
|
+
* the AR framework's depth API (iPhone Pro LiDAR; ARCore Depth
|
|
145
|
+
* API on supported Android devices).
|
|
146
|
+
*
|
|
147
|
+
* Resolution is typically lower than the camera image (e.g.,
|
|
148
|
+
* 256×192 on iPhone Pro LiDAR). `confidenceMap` is per-pixel:
|
|
149
|
+
* `0` = low, `1` = medium, `2` = high confidence. `Float32`
|
|
150
|
+
* depth in metres; `Uint8` confidence.
|
|
151
|
+
*/
|
|
152
|
+
arDepth?: {
|
|
153
|
+
width: number;
|
|
154
|
+
height: number;
|
|
155
|
+
depthMap: ArrayBuffer;
|
|
156
|
+
confidenceMap?: ArrayBuffer;
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Tracked AR anchors visible in this frame. Empty array if AR
|
|
161
|
+
* is active but no anchors are tracked. Undefined in non-AR mode.
|
|
162
|
+
*/
|
|
163
|
+
arAnchors?: ARAnchor[];
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* AR tracking quality. Worklets that should skip work when
|
|
167
|
+
* tracking is degraded check this. Undefined in non-AR mode.
|
|
168
|
+
*/
|
|
169
|
+
arTrackingState?: 'notAvailable' | 'limited' | 'normal';
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* v0.8.0 — public AR anchor type. Subset of ARKit/ARCore anchor info
|
|
174
|
+
* exposed to JS worklets. Extend with plane-extent / image-name
|
|
175
|
+
* fields as the JSI binding learns them.
|
|
176
|
+
*/
|
|
177
|
+
export interface ARAnchor {
|
|
178
|
+
/** Stable per-session anchor identifier. */
|
|
179
|
+
id: string;
|
|
180
|
+
/** Anchor kind. `'point'` is Android (ARCore) only. */
|
|
181
|
+
type: 'plane' | 'image' | 'point';
|
|
182
|
+
/**
|
|
183
|
+
* 4×4 row-major transform from anchor space to world space.
|
|
184
|
+
* 16 numbers.
|
|
185
|
+
*/
|
|
186
|
+
transform: number[];
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* v0.8.0 — worklet function signature for the unified frame processor.
|
|
191
|
+
*
|
|
192
|
+
* Must be a `'worklet'`-prefixed function (so it can run on the
|
|
193
|
+
* worklet runtime). Receives a `StitcherFrame` per camera frame; the
|
|
194
|
+
* return value is ignored (use `runOnJS` / shared values to surface
|
|
195
|
+
* results back to the JS thread).
|
|
196
|
+
*/
|
|
197
|
+
export type StitcherFrameProcessor = (frame: StitcherFrame) => void;
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
import type { StitcherFrameProcessor } from './StitcherFrame';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* v0.8.0 Phase 4a — process-scope registry of host-supplied worklets
|
|
7
|
+
* that the v0.8.0 `useFrameProcessor` hook registers into.
|
|
8
|
+
*
|
|
9
|
+
* ## What this is (Phase 4a)
|
|
10
|
+
*
|
|
11
|
+
* A plain JS singleton holding an ordered list of registered
|
|
12
|
+
* worklets. Hosts mount the `useFrameProcessor` hook (in this
|
|
13
|
+
* directory); the hook registers its worklet into this singleton
|
|
14
|
+
* on mount and unregisters on unmount. Each entry carries:
|
|
15
|
+
*
|
|
16
|
+
* - `id`: stable identifier issued by `register`; passed to
|
|
17
|
+
* `unregister`.
|
|
18
|
+
* - `worklet`: the host's `StitcherFrameProcessor` function.
|
|
19
|
+
* MUST be `'worklet'`-prefixed at the call site (TS can't
|
|
20
|
+
* enforce that — convention).
|
|
21
|
+
* - `isFirstParty`: `false` for host-supplied worklets;
|
|
22
|
+
* reserved for the lib's own first-party stitching path which
|
|
23
|
+
* today is wired natively (not through this registry).
|
|
24
|
+
*
|
|
25
|
+
* Order is stable: first-party entries (none in Phase 4a) come
|
|
26
|
+
* first, then host entries by registration order. Re-registration
|
|
27
|
+
* of the same worklet by identity yields a new entry — hosts that
|
|
28
|
+
* re-render and call `register` again ARE responsible for calling
|
|
29
|
+
* `unregister` first. The `useFrameProcessor` hook handles this
|
|
30
|
+
* via its `deps` dependency array.
|
|
31
|
+
*
|
|
32
|
+
* ## What this is NOT (Phase 4b)
|
|
33
|
+
*
|
|
34
|
+
* **The native AR worklet runtime does NOT yet read this registry.**
|
|
35
|
+
* Worklets registered here for AR-mode captures will not fire
|
|
36
|
+
* until Phase 4b lands the cross-runtime handoff (a
|
|
37
|
+
* worklets-core `SharedValue` mirror that `RNSARWorkletRuntime`
|
|
38
|
+
* reads on each `dispatchFrame:pose:` call; the runtime then
|
|
39
|
+
* constructs a `StitcherFrameHostObject` + invokes each
|
|
40
|
+
* registered worklet via `RNWorklet::WorkletInvoker::call`).
|
|
41
|
+
*
|
|
42
|
+
* In non-AR mode the host-supplied worklet IS invoked, but via
|
|
43
|
+
* vision-camera's Frame Processor runtime directly (the
|
|
44
|
+
* `useFrameProcessor` hook returns vc's processor object which
|
|
45
|
+
* `<Camera>` passes to vision-camera). So Phase 4a's public API
|
|
46
|
+
* is fully functional for non-AR; AR is API-stable but
|
|
47
|
+
* runtime-deferred.
|
|
48
|
+
*
|
|
49
|
+
* ## Singleton lifetime
|
|
50
|
+
*
|
|
51
|
+
* The registry is a module-level instance. It lives for the
|
|
52
|
+
* lifetime of the JS runtime (= until app reload). Entries
|
|
53
|
+
* accumulate only via `register` and shed only via `unregister`
|
|
54
|
+
* — no GC / weak-ref logic. Hosts that mount `useFrameProcessor`
|
|
55
|
+
* inside React components MUST rely on the hook's effect cleanup
|
|
56
|
+
* to unregister on unmount, or they'll leak entries until
|
|
57
|
+
* reload. The hook handles this correctly today.
|
|
58
|
+
*
|
|
59
|
+
* ## Why a singleton (vs context provider)
|
|
60
|
+
*
|
|
61
|
+
* The native AR worklet runtime is itself a process-scope
|
|
62
|
+
* singleton (`RNSARWorkletRuntime`, `StitcherWorkletRuntime`).
|
|
63
|
+
* The Phase 4b handoff between TS and native is necessarily
|
|
64
|
+
* process-scope. Wrapping the registry in a React context
|
|
65
|
+
* would force every consumer to be in the same provider tree
|
|
66
|
+
* which is friction for layer-2 hosts that compose
|
|
67
|
+
* `<ARCameraView>` / `useIncrementalStitcher` themselves. The
|
|
68
|
+
* singleton is the right shape; the React-level ergonomics are
|
|
69
|
+
* provided by the `useFrameProcessor` hook.
|
|
70
|
+
*/
|
|
71
|
+
export interface StitcherWorkletEntry {
|
|
72
|
+
readonly id: string;
|
|
73
|
+
readonly worklet: StitcherFrameProcessor;
|
|
74
|
+
readonly isFirstParty: boolean;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
class Registry {
|
|
78
|
+
private entries: StitcherWorkletEntry[] = [];
|
|
79
|
+
private nextHostCounter = 0;
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Register a worklet. Returns a stable ID for `unregister`.
|
|
83
|
+
*
|
|
84
|
+
* Entries are appended in registration order; first-party
|
|
85
|
+
* entries (if any are added in future) sort to the front.
|
|
86
|
+
*/
|
|
87
|
+
register(opts: {
|
|
88
|
+
worklet: StitcherFrameProcessor;
|
|
89
|
+
isFirstParty?: boolean;
|
|
90
|
+
}): string {
|
|
91
|
+
const isFirstParty = opts.isFirstParty ?? false;
|
|
92
|
+
const id = isFirstParty
|
|
93
|
+
? `fp-${this.nextHostCounter++}`
|
|
94
|
+
: `host-${this.nextHostCounter++}`;
|
|
95
|
+
const entry: StitcherWorkletEntry = {
|
|
96
|
+
id,
|
|
97
|
+
worklet: opts.worklet,
|
|
98
|
+
isFirstParty,
|
|
99
|
+
};
|
|
100
|
+
this.entries.push(entry);
|
|
101
|
+
// Re-sort so first-party always runs before host entries.
|
|
102
|
+
// Stable sort: registration order is preserved within each
|
|
103
|
+
// partition. Single-pass O(n log n) is fine — registration
|
|
104
|
+
// is rare (per-`<Camera>`-mount, not per-frame).
|
|
105
|
+
this.entries.sort((a, b) => {
|
|
106
|
+
if (a.isFirstParty !== b.isFirstParty) {
|
|
107
|
+
return a.isFirstParty ? -1 : 1;
|
|
108
|
+
}
|
|
109
|
+
return 0;
|
|
110
|
+
});
|
|
111
|
+
return id;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Remove a previously-registered worklet by ID. No-op if the ID
|
|
116
|
+
* isn't found. Hosts call this in their effect's cleanup.
|
|
117
|
+
*/
|
|
118
|
+
unregister(id: string): void {
|
|
119
|
+
this.entries = this.entries.filter((e) => e.id !== id);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Snapshot the current entries. Returned array is a copy —
|
|
124
|
+
* mutations don't affect the registry. Phase 4b's native
|
|
125
|
+
* handoff will read a `SharedValue` mirror of this list so the
|
|
126
|
+
* AR runtime doesn't need a JS-thread hop on the hot per-frame
|
|
127
|
+
* path; for Phase 4a this method is the JS-side accessor.
|
|
128
|
+
*/
|
|
129
|
+
getEntries(): readonly StitcherWorkletEntry[] {
|
|
130
|
+
return [...this.entries];
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Total number of registered worklets (first-party + host).
|
|
135
|
+
* Useful for diagnostics + tests.
|
|
136
|
+
*/
|
|
137
|
+
get count(): number {
|
|
138
|
+
return this.entries.length;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Test-only — clear all entries. NOT exported from
|
|
143
|
+
* `src/index.ts`. Used in unit tests to reset state between
|
|
144
|
+
* cases.
|
|
145
|
+
*/
|
|
146
|
+
_resetForTests(): void {
|
|
147
|
+
this.entries = [];
|
|
148
|
+
this.nextHostCounter = 0;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Process-scope singleton. Imported by `useFrameProcessor` (in
|
|
154
|
+
* this directory) + by the Phase 4b native-handoff code (TBD).
|
|
155
|
+
*/
|
|
156
|
+
export const StitcherWorkletRegistry = new Registry();
|