react-native-image-stitcher 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +246 -0
- package/android/build.gradle +10 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +115 -10
- package/cpp/stitcher_worklet_registry.cpp +10 -0
- package/cpp/stitcher_worklet_registry.hpp +10 -0
- package/cpp/tests/CMakeLists.txt +98 -0
- package/cpp/tests/README.md +86 -0
- package/cpp/tests/pose_test.cpp +74 -0
- package/cpp/tests/stitcher_frame_data_test.cpp +132 -0
- package/cpp/tests/stitcher_worklet_registry_test.cpp +195 -0
- package/cpp/tests/stubs/jsi/jsi.h +33 -0
- package/cpp/tests/stubs/react-native-worklets-core/WKTJsiWorklet.h +34 -0
- package/dist/camera/Camera.d.ts +30 -14
- package/dist/camera/Camera.js +18 -18
- package/dist/camera/useCapture.d.ts +1 -1
- package/dist/camera/useCapture.js +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +9 -1
- package/dist/stitching/incremental.d.ts +41 -0
- package/dist/stitching/useFrameProcessorDriver.d.ts +50 -95
- package/dist/stitching/useFrameProcessorDriver.js +76 -294
- package/dist/stitching/useFrameStream.js +52 -37
- package/dist/stitching/useStitcherWorklet.d.ts +185 -0
- package/dist/stitching/useStitcherWorklet.js +275 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +138 -9
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +50 -14
- package/package.json +1 -1
- package/src/camera/Camera.tsx +48 -32
- package/src/camera/useCapture.ts +1 -1
- package/src/index.ts +13 -0
- package/src/stitching/__tests__/subscribeIncrementalState.refine.test.ts +276 -0
- package/src/stitching/incremental.ts +42 -0
- package/src/stitching/useFrameProcessorDriver.ts +79 -320
- package/src/stitching/useFrameStream.ts +55 -39
- package/src/stitching/useStitcherWorklet.ts +390 -0
|
@@ -61,10 +61,10 @@
|
|
|
61
61
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
62
62
|
exports.useFrameStream = useFrameStream;
|
|
63
63
|
const react_1 = require("react");
|
|
64
|
-
const react_native_1 = require("react-native");
|
|
65
64
|
const react_native_vision_camera_1 = require("react-native-vision-camera");
|
|
66
65
|
const react_native_worklets_core_1 = require("react-native-worklets-core");
|
|
67
66
|
const useThrottledFrameProcessor_1 = require("./useThrottledFrameProcessor");
|
|
67
|
+
const files_1 = require("../utils/files");
|
|
68
68
|
/**
|
|
69
69
|
* `useFrameStream` — Layer 3. See module docstring for the full
|
|
70
70
|
* design + use-case mapping. Quick start:
|
|
@@ -98,37 +98,38 @@ const useThrottledFrameProcessor_1 = require("./useThrottledFrameProcessor");
|
|
|
98
98
|
function useFrameStream(options, handler) {
|
|
99
99
|
const sampleHz = Math.max(0.5, Math.min(10, options.sampleHz));
|
|
100
100
|
const quality = options.quality ?? 75;
|
|
101
|
-
// Default output dir:
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
|
|
101
|
+
// Default output dir: the lib's canonical capture dir resolved
|
|
102
|
+
// via `FileBridge.defaultCaptureDir()`. Same dir the lib uses
|
|
103
|
+
// for panorama JPEGs / keyframe JPEGs — guaranteed writable on
|
|
104
|
+
// both platforms (iOS NSCachesDirectory + Android Context.cacheDir),
|
|
105
|
+
// created if missing. Resolved async on first mount; until
|
|
106
|
+
// resolution completes the worklet's `outputDir` is empty and
|
|
107
|
+
// the plugin call no-ops silently (a few frames missed at most;
|
|
108
|
+
// typical resolution time is <50ms).
|
|
109
|
+
//
|
|
110
|
+
// Hosts that want a specific path supply `options.outputDir`
|
|
111
|
+
// and skip the resolution entirely.
|
|
112
|
+
const [resolvedDefaultDir, setResolvedDefaultDir] = (0, react_1.useState)('');
|
|
113
|
+
(0, react_1.useEffect)(() => {
|
|
105
114
|
if (options.outputDir != null)
|
|
106
|
-
return
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
115
|
+
return;
|
|
116
|
+
let cancelled = false;
|
|
117
|
+
(0, files_1.getDefaultCaptureDir)()
|
|
118
|
+
.then((dir) => {
|
|
119
|
+
if (!cancelled)
|
|
120
|
+
setResolvedDefaultDir(dir);
|
|
121
|
+
})
|
|
122
|
+
.catch((err) => {
|
|
123
|
+
// eslint-disable-next-line no-console
|
|
124
|
+
console.warn('[useFrameStream] FileBridge.defaultCaptureDir() failed; ' +
|
|
125
|
+
'samples will not fire until `options.outputDir` is supplied. ' +
|
|
126
|
+
String(err));
|
|
127
|
+
});
|
|
128
|
+
return () => {
|
|
129
|
+
cancelled = true;
|
|
130
|
+
};
|
|
121
131
|
}, [options.outputDir]);
|
|
122
|
-
|
|
123
|
-
// react-native-fs but to keep the dep surface minimal, we just
|
|
124
|
-
// attempt to create via a tiny native call — or, simpler, accept
|
|
125
|
-
// that the plugin's write call will fail if the dir doesn't
|
|
126
|
-
// exist + log a clear error. For v0.9.0 baseline we defer
|
|
127
|
-
// mkdir to the host (document it in the option's JSDoc) OR fall
|
|
128
|
-
// back to the platform's tmpdir which already exists.
|
|
129
|
-
//
|
|
130
|
-
// The tmpdir defaults above always exist on iOS + Android, so
|
|
131
|
-
// the common case "host doesn't supply outputDir" Just Works.
|
|
132
|
+
const outputDir = options.outputDir ?? resolvedDefaultDir;
|
|
132
133
|
// Stable JS-side handler reference for `runOnJS`. The hook re-
|
|
133
134
|
// captures `handler` on every render but the ref keeps the
|
|
134
135
|
// worklet closure pointing at the latest callback (avoid stale
|
|
@@ -152,18 +153,33 @@ function useFrameStream(options, handler) {
|
|
|
152
153
|
// registry hasn't initialised yet (rare race on app start). We
|
|
153
154
|
// retry every 16ms (one display frame) until success — matches
|
|
154
155
|
// the pattern in `useFrameProcessorDriver`.
|
|
155
|
-
|
|
156
|
+
//
|
|
157
|
+
// Use `useState` (not `useRef`) so the eventual non-null value
|
|
158
|
+
// triggers a re-render — the worklet closure below captures
|
|
159
|
+
// `plugin` by value at render time, so without state we'd
|
|
160
|
+
// capture `null` forever.
|
|
161
|
+
const [plugin, setPlugin] = (0, react_1.useState)(null);
|
|
156
162
|
(0, react_1.useEffect)(() => {
|
|
157
163
|
let cancelled = false;
|
|
158
164
|
let timerId = null;
|
|
165
|
+
let attempts = 0;
|
|
159
166
|
const tryAcquire = () => {
|
|
160
167
|
if (cancelled)
|
|
161
168
|
return;
|
|
169
|
+
attempts += 1;
|
|
162
170
|
const p = react_native_vision_camera_1.VisionCameraProxy.initFrameProcessorPlugin('save_frame_as_jpeg', {});
|
|
163
171
|
if (p != null) {
|
|
164
|
-
|
|
172
|
+
setPlugin(p);
|
|
165
173
|
return;
|
|
166
174
|
}
|
|
175
|
+
// After ~1s of failed retries, warn once — the plugin should
|
|
176
|
+
// be registered by then; persistent failure means the host's
|
|
177
|
+
// native bundle doesn't include `save_frame_as_jpeg`.
|
|
178
|
+
if (attempts === 60) {
|
|
179
|
+
// eslint-disable-next-line no-console
|
|
180
|
+
console.warn('[useFrameStream] save_frame_as_jpeg plugin not found after 1s of retries. ' +
|
|
181
|
+
'Verify react-native-image-stitcher native module is installed in your host app.');
|
|
182
|
+
}
|
|
167
183
|
timerId = setTimeout(tryAcquire, 16);
|
|
168
184
|
};
|
|
169
185
|
tryAcquire();
|
|
@@ -173,15 +189,14 @@ function useFrameStream(options, handler) {
|
|
|
173
189
|
clearTimeout(timerId);
|
|
174
190
|
};
|
|
175
191
|
}, []);
|
|
176
|
-
// The worklet body — fires at sampleHz, calls the JPEG plugin,
|
|
177
|
-
// bridges the result to JS. Note we read `pluginRef.current`
|
|
178
|
-
// inside the worklet via the captured `plugin` value below;
|
|
179
|
-
// worklets-core handles the JS↔worklet reference.
|
|
180
|
-
const plugin = pluginRef.current;
|
|
181
192
|
return (0, useThrottledFrameProcessor_1.useThrottledFrameProcessor)((frame) => {
|
|
182
193
|
'worklet';
|
|
183
194
|
if (plugin == null)
|
|
184
195
|
return;
|
|
196
|
+
// Async outputDir resolution may not have completed yet on
|
|
197
|
+
// the first few frames after mount — bail until it does.
|
|
198
|
+
if (outputDir === '')
|
|
199
|
+
return;
|
|
185
200
|
// Slot rotation: compute slot from frame timestamp. At
|
|
186
201
|
// sampleHz=2 (500ms interval), the slot index changes every
|
|
187
202
|
// ~1s, giving each slot ~2 samples before being overwritten.
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* useStitcherWorklet — exposes the lib's first-party stitching as a
|
|
3
|
+
* callable worklet function for host-composed Frame Processors.
|
|
4
|
+
*
|
|
5
|
+
* v0.11.0 — closes the v0.8.0 Phase 5 either-or constraint by letting
|
|
6
|
+
* hosts COMPOSE: write ONE `useFrameProcessor` worklet body that calls
|
|
7
|
+
* BOTH your custom logic AND the lib's first-party stitching, instead
|
|
8
|
+
* of one displacing the other. See `docs/host-app-integration.md`
|
|
9
|
+
* § Tier 3 composition for the pattern.
|
|
10
|
+
*
|
|
11
|
+
* ## Why this is a separate hook
|
|
12
|
+
*
|
|
13
|
+
* vision-camera v4 lets a `<Camera>` mount accept exactly ONE frame
|
|
14
|
+
* processor. Pre-v0.11.0, hosts that passed a `frameProcessor` prop
|
|
15
|
+
* to the lib's `<Camera>` REPLACED the lib's first-party stitching
|
|
16
|
+
* processor in non-AR mode. Composing required hand-writing both
|
|
17
|
+
* worklet bodies in the host's processor. v0.11.0 extracts the
|
|
18
|
+
* lib's worklet body into this hook so hosts can compose with a
|
|
19
|
+
* single call:
|
|
20
|
+
*
|
|
21
|
+
* const stitcher = useStitcherWorklet();
|
|
22
|
+
* const fp = useFrameProcessor((frame) => {
|
|
23
|
+
* 'worklet';
|
|
24
|
+
* hostPreLogic(frame);
|
|
25
|
+
* stitcher.call(frame); // ← lib's first-party stitching
|
|
26
|
+
* hostPostLogic(frame);
|
|
27
|
+
* }, [stitcher.call]);
|
|
28
|
+
* return <Camera frameProcessor={fp} ... />;
|
|
29
|
+
*
|
|
30
|
+
* AR mode is unaffected — the AR-session dispatch path (v0.8.0 Phase
|
|
31
|
+
* 4b.i / 4b.iii) already composes natively.
|
|
32
|
+
*
|
|
33
|
+
* ## What this owns
|
|
34
|
+
*
|
|
35
|
+
* - vc Frame Processor plugin acquisition for
|
|
36
|
+
* `cv_flow_gate_process_frame` (the same plugin the legacy
|
|
37
|
+
* `useFrameProcessorDriver` used; reentrant by construction).
|
|
38
|
+
* - Shared values backing pose (yaw / pitch / roll), throttle
|
|
39
|
+
* counter, every-N gate, and FoV-derived intrinsics scalars.
|
|
40
|
+
* - Gyro subscription on the JS thread (always-on between mount
|
|
41
|
+
* and unmount; subscription cost is tiny).
|
|
42
|
+
* - The worklet body itself: throttle → pose synthesis →
|
|
43
|
+
* `plugin.call(frame, params)`.
|
|
44
|
+
*
|
|
45
|
+
* ## Lifecycle
|
|
46
|
+
*
|
|
47
|
+
* - Gyro auto-subscribes on mount, auto-unsubscribes on unmount.
|
|
48
|
+
* Composed hosts get pose tracking for free.
|
|
49
|
+
* - `reset()` zeros the accumulated yaw / pitch / roll between
|
|
50
|
+
* captures. `useFrameProcessorDriver` calls this on `start()` to
|
|
51
|
+
* preserve pre-v0.11.0 per-capture pose-reset behaviour;
|
|
52
|
+
* composed hosts should call it at the start of each capture too
|
|
53
|
+
* (otherwise pose drifts across captures).
|
|
54
|
+
*
|
|
55
|
+
* ## Behaviour delta from pre-v0.11.0
|
|
56
|
+
*
|
|
57
|
+
* Before: `useFrameProcessorDriver.start()` subscribed the gyro;
|
|
58
|
+
* `stop()` unsubscribed. The subscription was tied to the
|
|
59
|
+
* capture lifecycle.
|
|
60
|
+
*
|
|
61
|
+
* After: the gyro is subscribed for the lifetime of this hook
|
|
62
|
+
* (i.e., as long as the component using it is mounted). In the
|
|
63
|
+
* default `<Camera>` integration the hook mounts when the camera
|
|
64
|
+
* screen mounts, so the practical effect is the same; in
|
|
65
|
+
* custom-composed integrations the host controls mount/unmount
|
|
66
|
+
* by mounting/unmounting the component that calls
|
|
67
|
+
* `useStitcherWorklet`. The battery delta is small: gyroscope
|
|
68
|
+
* sampling at 33ms costs ≪1% CPU on every Android/iOS device
|
|
69
|
+
* the lib supports.
|
|
70
|
+
*
|
|
71
|
+
* `pose reset` semantics are preserved via the new explicit
|
|
72
|
+
* `reset()` method. Hosts that previously relied on `start()`
|
|
73
|
+
* to zero pose now call `stitcher.reset()` at the capture start.
|
|
74
|
+
*
|
|
75
|
+
* ## Pose synthesis (verbatim from `useFrameProcessorDriver`)
|
|
76
|
+
*
|
|
77
|
+
* Quaternion: q = q_yaw * q_pitch * q_roll (Tait-Bryan YPR, body
|
|
78
|
+
* frame). Expanded:
|
|
79
|
+
* qx = cy*sp*cr + sy*cp*sr
|
|
80
|
+
* qy = sy*cp*cr - cy*sp*sr
|
|
81
|
+
* qz = cy*cp*sr - sy*sp*cr
|
|
82
|
+
* qw = cy*cp*cr + sy*sp*sr
|
|
83
|
+
*
|
|
84
|
+
* When roll=0 this collapses to the legacy 2-axis form so captures
|
|
85
|
+
* held level produce bit-identical poses to the pre-v0.6 driver
|
|
86
|
+
* (and bit-identical to v0.10.x's `useFrameProcessorDriver`).
|
|
87
|
+
*
|
|
88
|
+
* ## Throttling (verbatim)
|
|
89
|
+
*
|
|
90
|
+
* `evalEveryNFrames` controls how often the worklet calls the
|
|
91
|
+
* plugin. Default 1. Independent of — and stacks on top of —
|
|
92
|
+
* the stitcher's own internal `flowEvalEveryNFrames` in
|
|
93
|
+
* `KeyframeGate.swift`; effective cadence is the product.
|
|
94
|
+
*
|
|
95
|
+
* ## Pairing with `IncrementalStitcher.start`
|
|
96
|
+
*
|
|
97
|
+
* The plugin's per-frame call into `consumeFrameFromPlugin` is
|
|
98
|
+
* gated by `IncrementalStitcher.frameProcessorIngestEnabled`,
|
|
99
|
+
* which is TRUE only when the stitcher was started with
|
|
100
|
+
* `frameSourceMode === 'frameProcessor'`. Hosts MUST call
|
|
101
|
+
* `incrementalStitcher.start({ frameSourceMode: 'frameProcessor',
|
|
102
|
+
* ... })` to actually get frames into the engine — otherwise the
|
|
103
|
+
* worklet runs to completion but the wrapper drops the call.
|
|
104
|
+
* `Camera.tsx` does this wiring automatically when the host opts
|
|
105
|
+
* into the lib's `useFrameProcessorDriver`. Hosts that compose
|
|
106
|
+
* their own worklet via this hook must do the wiring themselves.
|
|
107
|
+
*/
|
|
108
|
+
import type { Frame } from 'react-native-vision-camera';
|
|
109
|
+
import type { StitcherFrame } from './StitcherFrame';
|
|
110
|
+
/**
|
|
111
|
+
* Frames the lib's stitching worklet accepts. Accepting either a
|
|
112
|
+
* vc `Frame` (what the host's `useFrameProcessor` body sees) or the
|
|
113
|
+
* lib's `StitcherFrame` (what the lib's `useFrameProcessor` body
|
|
114
|
+
* sees) keeps the same `useStitcherWorklet` usable from both kinds
|
|
115
|
+
* of host worklet bodies without a cast on the call site. The
|
|
116
|
+
* worklet only reads `width` / `height`; the rest of the frame
|
|
117
|
+
* object is forwarded verbatim to the native plugin.
|
|
118
|
+
*/
|
|
119
|
+
export type StitcherWorkletInput = Frame | StitcherFrame;
|
|
120
|
+
export interface UseStitcherWorkletOptions {
|
|
121
|
+
/**
|
|
122
|
+
* Gyro sample interval in ms (~30 Hz default). Drives the JS-
|
|
123
|
+
* thread pose integration loop; not the producer-thread plugin
|
|
124
|
+
* call rate.
|
|
125
|
+
*/
|
|
126
|
+
gyroIntervalMs?: number;
|
|
127
|
+
/**
|
|
128
|
+
* Approximate horizontal FoV of the device camera, used to
|
|
129
|
+
* synthesise `fx` from frame width. Default 65° matches a typical
|
|
130
|
+
* mid-tier smartphone main camera.
|
|
131
|
+
*/
|
|
132
|
+
fovHorizDegrees?: number;
|
|
133
|
+
/**
|
|
134
|
+
* Approximate vertical FoV of the device camera, used to
|
|
135
|
+
* synthesise `fy` from frame height. Default 50° matches a typical
|
|
136
|
+
* 4:3 phone camera in landscape; for 16:9 portrait you probably
|
|
137
|
+
* want ~75°.
|
|
138
|
+
*/
|
|
139
|
+
fovVertDegrees?: number;
|
|
140
|
+
/**
|
|
141
|
+
* Evaluate the plugin every Nth producer-thread frame. Default 1
|
|
142
|
+
* (every frame).
|
|
143
|
+
*/
|
|
144
|
+
evalEveryNFrames?: number;
|
|
145
|
+
}
|
|
146
|
+
export interface StitcherWorkletHandle {
|
|
147
|
+
/**
|
|
148
|
+
* Worklet function: pass a `StitcherFrame` to perform one frame of
|
|
149
|
+
* the lib's first-party stitching (throttle + pose synthesis +
|
|
150
|
+
* native plugin call). Safe to call from inside another
|
|
151
|
+
* `'worklet'`-prefixed function (this is the canonical
|
|
152
|
+
* composition pattern).
|
|
153
|
+
*
|
|
154
|
+
* The returned function reference is stable across re-renders as
|
|
155
|
+
* long as the plugin reference doesn't change (which happens at
|
|
156
|
+
* most once — at the moment the JSI plugin finishes
|
|
157
|
+
* registering). Include `stitcher.call` in your `useFrameProcessor`
|
|
158
|
+
* deps so the host worklet rebuilds when the plugin acquires.
|
|
159
|
+
*
|
|
160
|
+
* Safe to invoke before the plugin is ready: the worklet
|
|
161
|
+
* internally short-circuits (the frame is silently skipped).
|
|
162
|
+
* Hosts that want to display a "stitcher initialising…" UI can
|
|
163
|
+
* read `isReady` to gate their own behaviour.
|
|
164
|
+
*/
|
|
165
|
+
call: (frame: StitcherWorkletInput) => void;
|
|
166
|
+
/**
|
|
167
|
+
* Zero accumulated yaw / pitch / roll. Call at the start of each
|
|
168
|
+
* capture so the pose stream starts from `(0, 0, 0)` instead of
|
|
169
|
+
* carrying drift from the previous capture or from idle time
|
|
170
|
+
* between captures. Idempotent; safe to call from JS.
|
|
171
|
+
*/
|
|
172
|
+
reset: () => void;
|
|
173
|
+
/**
|
|
174
|
+
* `true` once the JSI Frame Processor plugin
|
|
175
|
+
* (`cv_flow_gate_process_frame`) has resolved. Before this flips
|
|
176
|
+
* `true`, `call(frame)` is a no-op (the plugin reference is
|
|
177
|
+
* `null`). Hosts integrating via `useFrameProcessorDriver` use
|
|
178
|
+
* this to decide whether to render the frame-processor at all —
|
|
179
|
+
* the driver returns `null` for `frameProcessor` until ready, so
|
|
180
|
+
* `<Camera>` falls back gracefully.
|
|
181
|
+
*/
|
|
182
|
+
isReady: boolean;
|
|
183
|
+
}
|
|
184
|
+
export declare function useStitcherWorklet(options?: UseStitcherWorkletOptions): StitcherWorkletHandle;
|
|
185
|
+
//# sourceMappingURL=useStitcherWorklet.d.ts.map
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* useStitcherWorklet — exposes the lib's first-party stitching as a
|
|
5
|
+
* callable worklet function for host-composed Frame Processors.
|
|
6
|
+
*
|
|
7
|
+
* v0.11.0 — closes the v0.8.0 Phase 5 either-or constraint by letting
|
|
8
|
+
* hosts COMPOSE: write ONE `useFrameProcessor` worklet body that calls
|
|
9
|
+
* BOTH your custom logic AND the lib's first-party stitching, instead
|
|
10
|
+
* of one displacing the other. See `docs/host-app-integration.md`
|
|
11
|
+
* § Tier 3 composition for the pattern.
|
|
12
|
+
*
|
|
13
|
+
* ## Why this is a separate hook
|
|
14
|
+
*
|
|
15
|
+
* vision-camera v4 lets a `<Camera>` mount accept exactly ONE frame
|
|
16
|
+
* processor. Pre-v0.11.0, hosts that passed a `frameProcessor` prop
|
|
17
|
+
* to the lib's `<Camera>` REPLACED the lib's first-party stitching
|
|
18
|
+
* processor in non-AR mode. Composing required hand-writing both
|
|
19
|
+
* worklet bodies in the host's processor. v0.11.0 extracts the
|
|
20
|
+
* lib's worklet body into this hook so hosts can compose with a
|
|
21
|
+
* single call:
|
|
22
|
+
*
|
|
23
|
+
* const stitcher = useStitcherWorklet();
|
|
24
|
+
* const fp = useFrameProcessor((frame) => {
|
|
25
|
+
* 'worklet';
|
|
26
|
+
* hostPreLogic(frame);
|
|
27
|
+
* stitcher.call(frame); // ← lib's first-party stitching
|
|
28
|
+
* hostPostLogic(frame);
|
|
29
|
+
* }, [stitcher.call]);
|
|
30
|
+
* return <Camera frameProcessor={fp} ... />;
|
|
31
|
+
*
|
|
32
|
+
* AR mode is unaffected — the AR-session dispatch path (v0.8.0 Phase
|
|
33
|
+
* 4b.i / 4b.iii) already composes natively.
|
|
34
|
+
*
|
|
35
|
+
* ## What this owns
|
|
36
|
+
*
|
|
37
|
+
* - vc Frame Processor plugin acquisition for
|
|
38
|
+
* `cv_flow_gate_process_frame` (the same plugin the legacy
|
|
39
|
+
* `useFrameProcessorDriver` used; reentrant by construction).
|
|
40
|
+
* - Shared values backing pose (yaw / pitch / roll), throttle
|
|
41
|
+
* counter, every-N gate, and FoV-derived intrinsics scalars.
|
|
42
|
+
* - Gyro subscription on the JS thread (always-on between mount
|
|
43
|
+
* and unmount; subscription cost is tiny).
|
|
44
|
+
* - The worklet body itself: throttle → pose synthesis →
|
|
45
|
+
* `plugin.call(frame, params)`.
|
|
46
|
+
*
|
|
47
|
+
* ## Lifecycle
|
|
48
|
+
*
|
|
49
|
+
* - Gyro auto-subscribes on mount, auto-unsubscribes on unmount.
|
|
50
|
+
* Composed hosts get pose tracking for free.
|
|
51
|
+
* - `reset()` zeros the accumulated yaw / pitch / roll between
|
|
52
|
+
* captures. `useFrameProcessorDriver` calls this on `start()` to
|
|
53
|
+
* preserve pre-v0.11.0 per-capture pose-reset behaviour;
|
|
54
|
+
* composed hosts should call it at the start of each capture too
|
|
55
|
+
* (otherwise pose drifts across captures).
|
|
56
|
+
*
|
|
57
|
+
* ## Behaviour delta from pre-v0.11.0
|
|
58
|
+
*
|
|
59
|
+
* Before: `useFrameProcessorDriver.start()` subscribed the gyro;
|
|
60
|
+
* `stop()` unsubscribed. The subscription was tied to the
|
|
61
|
+
* capture lifecycle.
|
|
62
|
+
*
|
|
63
|
+
* After: the gyro is subscribed for the lifetime of this hook
|
|
64
|
+
* (i.e., as long as the component using it is mounted). In the
|
|
65
|
+
* default `<Camera>` integration the hook mounts when the camera
|
|
66
|
+
* screen mounts, so the practical effect is the same; in
|
|
67
|
+
* custom-composed integrations the host controls mount/unmount
|
|
68
|
+
* by mounting/unmounting the component that calls
|
|
69
|
+
* `useStitcherWorklet`. The battery delta is small: gyroscope
|
|
70
|
+
* sampling at 33ms costs ≪1% CPU on every Android/iOS device
|
|
71
|
+
* the lib supports.
|
|
72
|
+
*
|
|
73
|
+
* `pose reset` semantics are preserved via the new explicit
|
|
74
|
+
* `reset()` method. Hosts that previously relied on `start()`
|
|
75
|
+
* to zero pose now call `stitcher.reset()` at the capture start.
|
|
76
|
+
*
|
|
77
|
+
* ## Pose synthesis (verbatim from `useFrameProcessorDriver`)
|
|
78
|
+
*
|
|
79
|
+
* Quaternion: q = q_yaw * q_pitch * q_roll (Tait-Bryan YPR, body
|
|
80
|
+
* frame). Expanded:
|
|
81
|
+
* qx = cy*sp*cr + sy*cp*sr
|
|
82
|
+
* qy = sy*cp*cr - cy*sp*sr
|
|
83
|
+
* qz = cy*cp*sr - sy*sp*cr
|
|
84
|
+
* qw = cy*cp*cr + sy*sp*sr
|
|
85
|
+
*
|
|
86
|
+
* When roll=0 this collapses to the legacy 2-axis form so captures
|
|
87
|
+
* held level produce bit-identical poses to the pre-v0.6 driver
|
|
88
|
+
* (and bit-identical to v0.10.x's `useFrameProcessorDriver`).
|
|
89
|
+
*
|
|
90
|
+
* ## Throttling (verbatim)
|
|
91
|
+
*
|
|
92
|
+
* `evalEveryNFrames` controls how often the worklet calls the
|
|
93
|
+
* plugin. Default 1. Independent of — and stacks on top of —
|
|
94
|
+
* the stitcher's own internal `flowEvalEveryNFrames` in
|
|
95
|
+
* `KeyframeGate.swift`; effective cadence is the product.
|
|
96
|
+
*
|
|
97
|
+
* ## Pairing with `IncrementalStitcher.start`
|
|
98
|
+
*
|
|
99
|
+
* The plugin's per-frame call into `consumeFrameFromPlugin` is
|
|
100
|
+
* gated by `IncrementalStitcher.frameProcessorIngestEnabled`,
|
|
101
|
+
* which is TRUE only when the stitcher was started with
|
|
102
|
+
* `frameSourceMode === 'frameProcessor'`. Hosts MUST call
|
|
103
|
+
* `incrementalStitcher.start({ frameSourceMode: 'frameProcessor',
|
|
104
|
+
* ... })` to actually get frames into the engine — otherwise the
|
|
105
|
+
* worklet runs to completion but the wrapper drops the call.
|
|
106
|
+
* `Camera.tsx` does this wiring automatically when the host opts
|
|
107
|
+
* into the lib's `useFrameProcessorDriver`. Hosts that compose
|
|
108
|
+
* their own worklet via this hook must do the wiring themselves.
|
|
109
|
+
*/
|
|
110
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
111
|
+
exports.useStitcherWorklet = useStitcherWorklet;
|
|
112
|
+
const react_1 = require("react");
|
|
113
|
+
const react_native_sensors_1 = require("react-native-sensors");
|
|
114
|
+
// Reanimated's `useSharedValue` is the documented vision-camera
|
|
115
|
+
// idiom, but it's a heavy peer dep. `react-native-worklets-core`
|
|
116
|
+
// (already a transitive dep via vision-camera v4 on RN 0.84) exposes
|
|
117
|
+
// the same API surface (a `value` getter/setter readable from
|
|
118
|
+
// worklets and the JS thread) and is sufficient for our use.
|
|
119
|
+
const react_native_worklets_core_1 = require("react-native-worklets-core");
|
|
120
|
+
const react_native_vision_camera_1 = require("react-native-vision-camera");
|
|
121
|
+
function useStitcherWorklet(options = {}) {
|
|
122
|
+
const { gyroIntervalMs = 33, fovHorizDegrees = 65, fovVertDegrees = 50, evalEveryNFrames = 1, } = options;
|
|
123
|
+
// ── Plugin acquisition ──────────────────────────────────────────
|
|
124
|
+
//
|
|
125
|
+
// `initFrameProcessorPlugin` can return `undefined` if called
|
|
126
|
+
// before vision-camera's plugin registry has finished initialising
|
|
127
|
+
// (race observed in F8.1.a). Mount-once useEffect with a 16ms
|
|
128
|
+
// retry until success. Verbatim from `useFrameProcessorDriver`.
|
|
129
|
+
const [plugin, setPlugin] = (0, react_1.useState)(null);
|
|
130
|
+
(0, react_1.useEffect)(() => {
|
|
131
|
+
let cancelled = false;
|
|
132
|
+
let timerId = null;
|
|
133
|
+
const tryAcquire = () => {
|
|
134
|
+
if (cancelled)
|
|
135
|
+
return;
|
|
136
|
+
const p = react_native_vision_camera_1.VisionCameraProxy.initFrameProcessorPlugin('cv_flow_gate_process_frame', {});
|
|
137
|
+
if (p != null) {
|
|
138
|
+
setPlugin(p);
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
timerId = setTimeout(tryAcquire, 16);
|
|
142
|
+
};
|
|
143
|
+
tryAcquire();
|
|
144
|
+
return () => {
|
|
145
|
+
cancelled = true;
|
|
146
|
+
if (timerId != null)
|
|
147
|
+
clearTimeout(timerId);
|
|
148
|
+
};
|
|
149
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
150
|
+
}, []);
|
|
151
|
+
// ── Shared values (worklet ↔ JS thread) ─────────────────────────
|
|
152
|
+
const sharedYaw = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
153
|
+
const sharedPitch = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
154
|
+
const sharedRoll = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
155
|
+
const sharedFrameCounter = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
156
|
+
const sharedEvalEveryN = (0, react_native_worklets_core_1.useSharedValue)(Math.max(1, evalEveryNFrames));
|
|
157
|
+
const sharedFxNumerator = (0, react_native_worklets_core_1.useSharedValue)(1.0 / (2.0 * Math.tan((fovHorizDegrees * Math.PI / 180) / 2)));
|
|
158
|
+
const sharedFyNumerator = (0, react_native_worklets_core_1.useSharedValue)(1.0 / (2.0 * Math.tan((fovVertDegrees * Math.PI / 180) / 2)));
|
|
159
|
+
// Prop-derived shared values stay in sync via cheap effects.
|
|
160
|
+
(0, react_1.useEffect)(() => {
|
|
161
|
+
sharedEvalEveryN.value = Math.max(1, evalEveryNFrames);
|
|
162
|
+
}, [evalEveryNFrames, sharedEvalEveryN]);
|
|
163
|
+
(0, react_1.useEffect)(() => {
|
|
164
|
+
sharedFxNumerator.value =
|
|
165
|
+
1.0 / (2.0 * Math.tan((fovHorizDegrees * Math.PI / 180) / 2));
|
|
166
|
+
}, [fovHorizDegrees, sharedFxNumerator]);
|
|
167
|
+
(0, react_1.useEffect)(() => {
|
|
168
|
+
sharedFyNumerator.value =
|
|
169
|
+
1.0 / (2.0 * Math.tan((fovVertDegrees * Math.PI / 180) / 2));
|
|
170
|
+
}, [fovVertDegrees, sharedFyNumerator]);
|
|
171
|
+
// ── Gyro subscription (always-on while mounted) ─────────────────
|
|
172
|
+
//
|
|
173
|
+
// v0.11.0 — moved here from `useFrameProcessorDriver.start()`.
|
|
174
|
+
// The composition pattern needs gyro running whenever
|
|
175
|
+
// `useStitcherWorklet` is in use; gating the subscription on a
|
|
176
|
+
// separate start/stop pair would force every composed host to
|
|
177
|
+
// wire its own lifecycle. Cost is tiny: ≪1% CPU at 33ms
|
|
178
|
+
// sampling. See module header "Behaviour delta from pre-v0.11.0".
|
|
179
|
+
(0, react_1.useEffect)(() => {
|
|
180
|
+
let lastGyroAt = null;
|
|
181
|
+
(0, react_native_sensors_1.setUpdateIntervalForType)(react_native_sensors_1.SensorTypes.gyroscope, gyroIntervalMs);
|
|
182
|
+
const sub = react_native_sensors_1.gyroscope.subscribe({
|
|
183
|
+
next: ({ x, y, z }) => {
|
|
184
|
+
const now = Date.now();
|
|
185
|
+
if (lastGyroAt === null) {
|
|
186
|
+
lastGyroAt = now;
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
const dt = (now - lastGyroAt) / 1000.0;
|
|
190
|
+
lastGyroAt = now;
|
|
191
|
+
sharedYaw.value += y * dt;
|
|
192
|
+
sharedPitch.value += x * dt;
|
|
193
|
+
sharedRoll.value += z * dt;
|
|
194
|
+
},
|
|
195
|
+
error: (err) => {
|
|
196
|
+
// eslint-disable-next-line no-console
|
|
197
|
+
console.warn('[useStitcherWorklet] gyro error', err);
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
return () => {
|
|
201
|
+
sub.unsubscribe();
|
|
202
|
+
};
|
|
203
|
+
}, [gyroIntervalMs, sharedYaw, sharedPitch, sharedRoll]);
|
|
204
|
+
// ── Explicit reset (for per-capture pose zero-ing) ──────────────
|
|
205
|
+
const reset = (0, react_1.useCallback)(() => {
|
|
206
|
+
sharedYaw.value = 0;
|
|
207
|
+
sharedPitch.value = 0;
|
|
208
|
+
sharedRoll.value = 0;
|
|
209
|
+
sharedFrameCounter.value = 0;
|
|
210
|
+
}, [sharedYaw, sharedPitch, sharedRoll, sharedFrameCounter]);
|
|
211
|
+
// ── Worklet body ────────────────────────────────────────────────
|
|
212
|
+
//
|
|
213
|
+
// Returned as `handle.call`. Re-created when `plugin` changes
|
|
214
|
+
// (which happens at most once at acquire time); deps array on the
|
|
215
|
+
// useCallback ensures consumers' `useFrameProcessor([handle.call])`
|
|
216
|
+
// re-binds when the worklet identity changes.
|
|
217
|
+
//
|
|
218
|
+
// The `'worklet'` directive marks this function for the
|
|
219
|
+
// worklets-core transformer so it can be serialised into the
|
|
220
|
+
// producer-thread runtime; that's the contract that lets a host
|
|
221
|
+
// `useFrameProcessor` worklet body call it without a thread hop.
|
|
222
|
+
const call = (0, react_1.useCallback)((frame) => {
|
|
223
|
+
'worklet';
|
|
224
|
+
if (plugin == null)
|
|
225
|
+
return;
|
|
226
|
+
// Throttle (verbatim from useFrameProcessorDriver).
|
|
227
|
+
sharedFrameCounter.value += 1;
|
|
228
|
+
const N = sharedEvalEveryN.value;
|
|
229
|
+
if (N > 1 && (sharedFrameCounter.value % N) !== 0)
|
|
230
|
+
return;
|
|
231
|
+
// Pose synthesis (verbatim from useFrameProcessorDriver).
|
|
232
|
+
const halfYaw = sharedYaw.value / 2;
|
|
233
|
+
const halfPitch = sharedPitch.value / 2;
|
|
234
|
+
const halfRoll = sharedRoll.value / 2;
|
|
235
|
+
const cy_ = Math.cos(halfYaw);
|
|
236
|
+
const sy_ = Math.sin(halfYaw);
|
|
237
|
+
const cp = Math.cos(halfPitch);
|
|
238
|
+
const sp = Math.sin(halfPitch);
|
|
239
|
+
const cr = Math.cos(halfRoll);
|
|
240
|
+
const sr = Math.sin(halfRoll);
|
|
241
|
+
const qx = cy_ * sp * cr + sy_ * cp * sr;
|
|
242
|
+
const qy = sy_ * cp * cr - cy_ * sp * sr;
|
|
243
|
+
const qz = cy_ * cp * sr - sy_ * sp * cr;
|
|
244
|
+
const qw = cy_ * cp * cr + sy_ * sp * sr;
|
|
245
|
+
// Intrinsics from FoV + actual frame dims.
|
|
246
|
+
const w = frame.width;
|
|
247
|
+
const h = frame.height;
|
|
248
|
+
const fx = w * sharedFxNumerator.value;
|
|
249
|
+
const fy = h * sharedFyNumerator.value;
|
|
250
|
+
// vc's `plugin.call` is typed against vc's `Frame`. The worklet
|
|
251
|
+
// accepts the union (`Frame | StitcherFrame`); cast through
|
|
252
|
+
// `unknown` because the union doesn't satisfy vc's interface
|
|
253
|
+
// even though structurally both members do.
|
|
254
|
+
plugin.call(frame, {
|
|
255
|
+
tx: 0, ty: 0, tz: 0,
|
|
256
|
+
qx, qy, qz, qw,
|
|
257
|
+
fx, fy,
|
|
258
|
+
cx: w / 2, cy: h / 2,
|
|
259
|
+
imageWidth: w, imageHeight: h,
|
|
260
|
+
timestampMs: 0,
|
|
261
|
+
trackingStateRaw: 2, // RNSARTrackingState.tracking (no AR signal in non-AR mode)
|
|
262
|
+
});
|
|
263
|
+
}, [
|
|
264
|
+
plugin,
|
|
265
|
+
sharedFrameCounter,
|
|
266
|
+
sharedEvalEveryN,
|
|
267
|
+
sharedYaw,
|
|
268
|
+
sharedPitch,
|
|
269
|
+
sharedRoll,
|
|
270
|
+
sharedFxNumerator,
|
|
271
|
+
sharedFyNumerator,
|
|
272
|
+
]);
|
|
273
|
+
return { call, reset, isReady: plugin != null };
|
|
274
|
+
}
|
|
275
|
+
//# sourceMappingURL=useStitcherWorklet.js.map
|