react-native-image-stitcher 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +137 -0
- package/dist/camera/Camera.d.ts +30 -14
- package/dist/camera/Camera.js +18 -18
- package/dist/index.d.ts +2 -0
- package/dist/index.js +9 -1
- package/dist/stitching/useFrameProcessorDriver.d.ts +50 -95
- package/dist/stitching/useFrameProcessorDriver.js +76 -294
- package/dist/stitching/useStitcherWorklet.d.ts +185 -0
- package/dist/stitching/useStitcherWorklet.js +300 -0
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +58 -1
- package/package.json +2 -1
- package/src/camera/Camera.tsx +48 -32
- package/src/index.ts +13 -0
- package/src/stitching/__tests__/useStitcherWorklet.test.ts +202 -0
- package/src/stitching/useFrameProcessorDriver.ts +79 -320
- package/src/stitching/useStitcherWorklet.ts +415 -0
|
@@ -6,317 +6,99 @@
|
|
|
6
6
|
* from v0.6 onward (replaced the deprecated `useIncrementalJSDriver`
|
|
7
7
|
* hook, which was removed in v0.6).
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
* qw = cy*cp*cr + sy*sp*sr
|
|
58
|
-
*
|
|
59
|
-
* When roll=0 this collapses to the 2-axis form
|
|
60
|
-
* `(cy*sp, sy*cp, -sy*sp, cy*cp)` the legacy driver used, so
|
|
61
|
-
* captures held perfectly level produce identical poses to the
|
|
62
|
-
* pre-roll behaviour.
|
|
63
|
-
*
|
|
64
|
-
* Intrinsics are synthesised from the actual frame dimensions
|
|
65
|
-
* (`frame.width`, `frame.height`) plus the host-provided
|
|
66
|
-
* horizontal/vertical FoV defaults. The stitcher derives its FoV-
|
|
67
|
-
* overlap window from these, so the assumed FoV matters for the
|
|
68
|
-
* gate's overlap math but not for the panorama itself (the
|
|
69
|
-
* stitcher feature-matches + RANSACs the final alignment).
|
|
70
|
-
*
|
|
71
|
-
* Throttling
|
|
72
|
-
*
|
|
73
|
-
* `evalEveryNFrames` controls how often the worklet calls the
|
|
74
|
-
* plugin. Default 1 (every frame). Set higher to amortise the
|
|
75
|
-
* plugin call + consumeFrame's gate evaluation across multiple
|
|
76
|
-
* producer-thread frames on lower-end devices. Independent of —
|
|
77
|
-
* and stacks on top of — the stitcher's own internal
|
|
78
|
-
* `flowEvalEveryNFrames` (see `KeyframeGate.swift`); both
|
|
79
|
-
* throttles can be active simultaneously and the effective cadence
|
|
80
|
-
* is `evalEveryNFrames * flowEvalEveryNFrames`.
|
|
81
|
-
*
|
|
82
|
-
* Lifecycle
|
|
83
|
-
*
|
|
84
|
-
* `start()` subscribes to the gyro and resets pose accumulators.
|
|
85
|
-
* `stop()` unsubscribes and resets. The returned `frameProcessor`
|
|
86
|
-
* is meant to be passed to `<Camera frameProcessor={...} />` —
|
|
87
|
-
* it's stable as long as the plugin reference and the FoV props
|
|
88
|
-
* haven't changed. Returns `null` when the plugin isn't loaded
|
|
89
|
-
* yet; pass `null`-or-fallback to the Camera in that case.
|
|
90
|
-
*
|
|
91
|
-
* Pairing with `IncrementalStitcher.start({frameSourceMode})`
|
|
92
|
-
*
|
|
93
|
-
* The plugin's per-frame call into `consumeFrameFromPlugin` is
|
|
94
|
-
* gated by `IncrementalStitcher.frameProcessorIngestEnabled`,
|
|
95
|
-
* which is TRUE only when the stitcher was started with
|
|
96
|
-
* `frameSourceMode === 'frameProcessor'`. Hosts MUST call
|
|
97
|
-
* `incrementalStitcher.start({ frameSourceMode: 'frameProcessor',
|
|
98
|
-
* ... })` to actually get frames into the engine — otherwise the
|
|
99
|
-
* worklet runs to completion but the wrapper drops the call.
|
|
100
|
-
* `Camera.tsx` does this wiring automatically when the host opts
|
|
101
|
-
* into this driver.
|
|
9
|
+
* v0.11.0 — refactored to a thin wrapper around `useStitcherWorklet`.
|
|
10
|
+
* The plugin acquisition + shared-value declarations + gyro
|
|
11
|
+
* subscription + worklet body all live in `useStitcherWorklet` now;
|
|
12
|
+
* this hook just binds the returned worklet via vision-camera's
|
|
13
|
+
* `useFrameProcessor` and exposes the legacy `start` / `stop` /
|
|
14
|
+
* `isRunning` API for backwards compatibility with v0.10.x.
|
|
15
|
+
*
|
|
16
|
+
* ## Why the v0.11.0 split
|
|
17
|
+
*
|
|
18
|
+
* vision-camera v4 allows ONE frame processor per `<Camera>` mount.
|
|
19
|
+
* Pre-v0.11.0, hosts that wanted to compose their own worklet with
|
|
20
|
+
* the lib's first-party stitching couldn't — passing a host
|
|
21
|
+
* `frameProcessor` REPLACED the lib's processor. v0.11.0 closes
|
|
22
|
+
* this gap by exposing the worklet body via `useStitcherWorklet`
|
|
23
|
+
* so hosts can write:
|
|
24
|
+
*
|
|
25
|
+
* const stitcher = useStitcherWorklet();
|
|
26
|
+
* const fp = useFrameProcessor((frame) => {
|
|
27
|
+
* 'worklet';
|
|
28
|
+
* hostPreLogic(frame);
|
|
29
|
+
* stitcher.call(frame); // ← first-party stitching
|
|
30
|
+
* hostPostLogic(frame);
|
|
31
|
+
* }, [stitcher.call]);
|
|
32
|
+
*
|
|
33
|
+
* `useFrameProcessorDriver` keeps the legacy default-integration
|
|
34
|
+
* shape (start / stop / isRunning) for the `<Camera>` component's
|
|
35
|
+
* built-in non-AR path and for any host still using the v0.10.x API
|
|
36
|
+
* directly. No behavioural change for those callers.
|
|
37
|
+
*
|
|
38
|
+
* ## start / stop behaviour
|
|
39
|
+
*
|
|
40
|
+
* - `start()` calls `stitcher.reset()` to zero the accumulated
|
|
41
|
+
* pose (preserves the pre-v0.11.0 "each capture starts with
|
|
42
|
+
* pose = (0, 0, 0)" contract).
|
|
43
|
+
* - `stop()` also resets the pose (idempotent; matches the
|
|
44
|
+
* pre-v0.11.0 stop() side effect of zeroing yaw / pitch / roll).
|
|
45
|
+
* - The gyro subscription itself is owned by `useStitcherWorklet`
|
|
46
|
+
* and runs for the lifetime of the hook. In the default
|
|
47
|
+
* `<Camera>` integration this means gyro is on while the camera
|
|
48
|
+
* screen is mounted — same practical scope as pre-v0.11.0 in
|
|
49
|
+
* all observed host integrations (capture screens mount
|
|
50
|
+
* `<Camera>` for the duration of capture; idle screens don't).
|
|
51
|
+
*
|
|
52
|
+
* ## Pose synthesis / intrinsics / throttling
|
|
53
|
+
*
|
|
54
|
+
* Owned by `useStitcherWorklet`. See that file's header for the
|
|
55
|
+
* quaternion math, FoV-to-intrinsics derivation, throttle gate, and
|
|
56
|
+
* pairing-with-IncrementalStitcher.start docs.
|
|
102
57
|
*/
|
|
103
58
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
104
59
|
exports.useFrameProcessorDriver = useFrameProcessorDriver;
|
|
105
60
|
const react_1 = require("react");
|
|
106
|
-
const react_native_sensors_1 = require("react-native-sensors");
|
|
107
|
-
// Reanimated's `useSharedValue` is the documented vision-camera
|
|
108
|
-
// idiom, but it's a heavy peer dep. `react-native-worklets-core`
|
|
109
|
-
// (already a transitive dep via vision-camera v4 on RN 0.84) exposes
|
|
110
|
-
// the same API surface (a `value` getter/setter readable from
|
|
111
|
-
// worklets and the JS thread) and is sufficient for our use.
|
|
112
|
-
const react_native_worklets_core_1 = require("react-native-worklets-core");
|
|
113
61
|
const react_native_vision_camera_1 = require("react-native-vision-camera");
|
|
62
|
+
const useStitcherWorklet_1 = require("./useStitcherWorklet");
|
|
114
63
|
function useFrameProcessorDriver(options = {}) {
|
|
115
|
-
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
|
|
120
|
-
// (race observed in F8.1.a). We retry on a fixed timer instead of
|
|
121
|
-
// firing on every render — the earlier render-driven pattern
|
|
122
|
-
// (adversarial-review H3) re-invoked `initFrameProcessorPlugin`
|
|
123
|
-
// 60+ times per second during recording, and the vision-camera
|
|
124
|
-
// contract for repeated lookups is undocumented.
|
|
125
|
-
//
|
|
126
|
-
// Pattern: mount-once useEffect, try synchronously, fall back to a
|
|
127
|
-
// 16-ms retry timer until success or unmount.
|
|
128
|
-
const [plugin, setPlugin] = (0, react_1.useState)(null);
|
|
129
|
-
(0, react_1.useEffect)(() => {
|
|
130
|
-
let cancelled = false;
|
|
131
|
-
let timerId = null;
|
|
132
|
-
const tryAcquire = () => {
|
|
133
|
-
if (cancelled)
|
|
134
|
-
return;
|
|
135
|
-
const p = react_native_vision_camera_1.VisionCameraProxy.initFrameProcessorPlugin('cv_flow_gate_process_frame', {});
|
|
136
|
-
if (p != null) {
|
|
137
|
-
setPlugin(p);
|
|
138
|
-
return;
|
|
139
|
-
}
|
|
140
|
-
// ~one display-frame retry — matches the F8.1.a observation
|
|
141
|
-
// that the registry becomes ready by the next render tick.
|
|
142
|
-
timerId = setTimeout(tryAcquire, 16);
|
|
143
|
-
};
|
|
144
|
-
tryAcquire();
|
|
145
|
-
return () => {
|
|
146
|
-
cancelled = true;
|
|
147
|
-
if (timerId != null)
|
|
148
|
-
clearTimeout(timerId);
|
|
149
|
-
};
|
|
150
|
-
// Empty deps on purpose — runs ONCE on mount. Re-acquiring on
|
|
151
|
-
// re-render would race with worklet binding.
|
|
152
|
-
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
153
|
-
}, []);
|
|
154
|
-
// ── Shared values (worklet ↔ JS thread) ─────────────────────────
|
|
155
|
-
//
|
|
156
|
-
// Reanimated guarantees coherent reads from the producer thread.
|
|
157
|
-
// We write yaw/pitch on the JS thread (gyro callbacks); the worklet
|
|
158
|
-
// reads them every frame. No round-trip cost — these are mapped
|
|
159
|
-
// into the worklet's runtime by the Reanimated bridge.
|
|
160
|
-
//
|
|
161
|
-
// FoV-derived values (the "half-angle tangent reciprocal"
|
|
162
|
-
// f-numerators) are pre-computed on the JS thread + published via
|
|
163
|
-
// shared values so the worklet's dependency array shrinks to just
|
|
164
|
-
// `[plugin]`. Earlier draft baked `fovHorizDegrees` /
|
|
165
|
-
// `fovVertDegrees` into the closure → worklet re-serialised on
|
|
166
|
-
// every host re-render that changed the prop refs (adversarial-
|
|
167
|
-
// review M1).
|
|
168
|
-
const sharedYaw = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
169
|
-
const sharedPitch = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
170
|
-
// F8.3-followup-roll — integrate gyroscope Z (out-of-screen for a
|
|
171
|
-
// portrait device) to track wrist-twist roll. Field captures with
|
|
172
|
-
// casual hand-hold rarely stay perfectly level; without this the
|
|
173
|
-
// pose stream lies and the cv::Stitcher's intrinsic estimator may
|
|
174
|
-
// pick a worse projection mode.
|
|
175
|
-
const sharedRoll = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
176
|
-
const sharedFrameCounter = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
177
|
-
const sharedEvalEveryN = (0, react_native_worklets_core_1.useSharedValue)(Math.max(1, evalEveryNFrames));
|
|
178
|
-
const sharedFxNumerator = (0, react_native_worklets_core_1.useSharedValue)(1.0 / (2.0 * Math.tan((fovHorizDegrees * Math.PI / 180) / 2)));
|
|
179
|
-
const sharedFyNumerator = (0, react_native_worklets_core_1.useSharedValue)(1.0 / (2.0 * Math.tan((fovVertDegrees * Math.PI / 180) / 2)));
|
|
180
|
-
// Keep prop-derived shared values in sync. Cheap re-renders;
|
|
181
|
-
// these don't trigger worklet rebuild.
|
|
182
|
-
(0, react_1.useEffect)(() => {
|
|
183
|
-
sharedEvalEveryN.value = Math.max(1, evalEveryNFrames);
|
|
184
|
-
}, [evalEveryNFrames, sharedEvalEveryN]);
|
|
185
|
-
(0, react_1.useEffect)(() => {
|
|
186
|
-
sharedFxNumerator.value =
|
|
187
|
-
1.0 / (2.0 * Math.tan((fovHorizDegrees * Math.PI / 180) / 2));
|
|
188
|
-
}, [fovHorizDegrees, sharedFxNumerator]);
|
|
189
|
-
(0, react_1.useEffect)(() => {
|
|
190
|
-
sharedFyNumerator.value =
|
|
191
|
-
1.0 / (2.0 * Math.tan((fovVertDegrees * Math.PI / 180) / 2));
|
|
192
|
-
}, [fovVertDegrees, sharedFyNumerator]);
|
|
193
|
-
// ── Lifecycle state (JS thread only) ────────────────────────────
|
|
194
|
-
const gyroSubRef = (0, react_1.useRef)(null);
|
|
195
|
-
const lastGyroAtRef = (0, react_1.useRef)(null);
|
|
64
|
+
// v0.11.0 — delegate plugin / shared values / gyro / worklet body
|
|
65
|
+
// to `useStitcherWorklet`. This hook is now a thin wrapper that
|
|
66
|
+
// binds the returned worklet via `useFrameProcessor` and exposes
|
|
67
|
+
// the legacy lifecycle API.
|
|
68
|
+
const stitcher = (0, useStitcherWorklet_1.useStitcherWorklet)(options);
|
|
196
69
|
const isRunningRef = (0, react_1.useRef)(false);
|
|
197
|
-
const stop = (0, react_1.useCallback)(() => {
|
|
198
|
-
if (gyroSubRef.current) {
|
|
199
|
-
gyroSubRef.current.unsubscribe();
|
|
200
|
-
gyroSubRef.current = null;
|
|
201
|
-
}
|
|
202
|
-
isRunningRef.current = false;
|
|
203
|
-
sharedYaw.value = 0;
|
|
204
|
-
sharedPitch.value = 0;
|
|
205
|
-
sharedRoll.value = 0;
|
|
206
|
-
sharedFrameCounter.value = 0;
|
|
207
|
-
lastGyroAtRef.current = null;
|
|
208
|
-
}, [sharedYaw, sharedPitch, sharedRoll, sharedFrameCounter]);
|
|
209
70
|
const start = (0, react_1.useCallback)(() => {
|
|
210
71
|
if (isRunningRef.current)
|
|
211
72
|
return;
|
|
212
|
-
|
|
213
|
-
sharedPitch.value = 0;
|
|
214
|
-
sharedRoll.value = 0;
|
|
215
|
-
sharedFrameCounter.value = 0;
|
|
216
|
-
lastGyroAtRef.current = null;
|
|
73
|
+
stitcher.reset();
|
|
217
74
|
isRunningRef.current = true;
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
// inverted roll, flip the sign on `z * dt` below.
|
|
227
|
-
(0, react_native_sensors_1.setUpdateIntervalForType)(react_native_sensors_1.SensorTypes.gyroscope, gyroIntervalMs);
|
|
228
|
-
gyroSubRef.current = react_native_sensors_1.gyroscope.subscribe({
|
|
229
|
-
next: ({ x, y, z }) => {
|
|
230
|
-
const now = Date.now();
|
|
231
|
-
if (lastGyroAtRef.current === null) {
|
|
232
|
-
lastGyroAtRef.current = now;
|
|
233
|
-
return;
|
|
234
|
-
}
|
|
235
|
-
const dt = (now - lastGyroAtRef.current) / 1000.0;
|
|
236
|
-
lastGyroAtRef.current = now;
|
|
237
|
-
sharedYaw.value += y * dt;
|
|
238
|
-
sharedPitch.value += x * dt;
|
|
239
|
-
sharedRoll.value += z * dt;
|
|
240
|
-
},
|
|
241
|
-
error: (err) => {
|
|
242
|
-
// eslint-disable-next-line no-console
|
|
243
|
-
console.warn('[useFrameProcessorDriver] gyro error', err);
|
|
244
|
-
},
|
|
245
|
-
});
|
|
246
|
-
}, [gyroIntervalMs, sharedYaw, sharedPitch, sharedRoll, sharedFrameCounter]);
|
|
247
|
-
// ── Worklet ─────────────────────────────────────────────────────
|
|
75
|
+
}, [stitcher]);
|
|
76
|
+
const stop = (0, react_1.useCallback)(() => {
|
|
77
|
+
if (!isRunningRef.current)
|
|
78
|
+
return;
|
|
79
|
+
stitcher.reset();
|
|
80
|
+
isRunningRef.current = false;
|
|
81
|
+
}, [stitcher]);
|
|
82
|
+
// ── Worklet binding ─────────────────────────────────────────────
|
|
248
83
|
//
|
|
249
|
-
//
|
|
250
|
-
//
|
|
251
|
-
//
|
|
252
|
-
//
|
|
253
|
-
//
|
|
84
|
+
// `stitcher.call` is itself a worklet (see `useStitcherWorklet`),
|
|
85
|
+
// so we just forward each frame to it. Memoised on
|
|
86
|
+
// [stitcher.call] so the host's `<Camera>` doesn't see frame-
|
|
87
|
+
// processor identity churn on every render — only when the
|
|
88
|
+
// underlying plugin acquires (null → non-null).
|
|
254
89
|
const frameProcessor = (0, react_native_vision_camera_1.useFrameProcessor)((frame) => {
|
|
255
90
|
'worklet';
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
sharedFrameCounter.value += 1;
|
|
263
|
-
const N = sharedEvalEveryN.value;
|
|
264
|
-
if (N > 1 && (sharedFrameCounter.value % N) !== 0)
|
|
265
|
-
return;
|
|
266
|
-
// Synthesise quaternion from accumulated yaw + pitch + roll.
|
|
267
|
-
// YPR Tait-Bryan order: q = q_yaw * q_pitch * q_roll. When
|
|
268
|
-
// roll=0 this reduces to the legacy 2-axis form (cy*sp, sy*cp,
|
|
269
|
-
// -sy*sp, cy*cp), so captures held level produce identical
|
|
270
|
-
// poses to the pre-F8.3-followup-roll behaviour. See the
|
|
271
|
-
// expanded math in the file header doc-comment.
|
|
272
|
-
const halfYaw = sharedYaw.value / 2;
|
|
273
|
-
const halfPitch = sharedPitch.value / 2;
|
|
274
|
-
const halfRoll = sharedRoll.value / 2;
|
|
275
|
-
const cy_ = Math.cos(halfYaw);
|
|
276
|
-
const sy_ = Math.sin(halfYaw);
|
|
277
|
-
const cp = Math.cos(halfPitch);
|
|
278
|
-
const sp = Math.sin(halfPitch);
|
|
279
|
-
const cr = Math.cos(halfRoll);
|
|
280
|
-
const sr = Math.sin(halfRoll);
|
|
281
|
-
const qx = cy_ * sp * cr + sy_ * cp * sr;
|
|
282
|
-
const qy = sy_ * cp * cr - cy_ * sp * sr;
|
|
283
|
-
const qz = cy_ * cp * sr - sy_ * sp * cr;
|
|
284
|
-
const qw = cy_ * cp * cr + sy_ * sp * sr;
|
|
285
|
-
// Intrinsics from FoV + actual frame dims.
|
|
286
|
-
// fx = w * (1 / (2 * tan(fovH/2))) (the parenthesised half
|
|
287
|
-
// is the precomputed `sharedFxNumerator` — see M1 fix).
|
|
288
|
-
const w = frame.width;
|
|
289
|
-
const h = frame.height;
|
|
290
|
-
const fx = w * sharedFxNumerator.value;
|
|
291
|
-
const fy = h * sharedFyNumerator.value;
|
|
292
|
-
plugin.call(frame, {
|
|
293
|
-
tx: 0, ty: 0, tz: 0,
|
|
294
|
-
qx, qy, qz, qw,
|
|
295
|
-
fx, fy,
|
|
296
|
-
cx: w / 2, cy: h / 2,
|
|
297
|
-
imageWidth: w, imageHeight: h,
|
|
298
|
-
timestampMs: 0,
|
|
299
|
-
// 2 == RNSARTrackingState.tracking — we always claim "good
|
|
300
|
-
// tracking" because there's no ARKit signal to differentiate.
|
|
301
|
-
// (Same contract as the pre-v0.6 useIncrementalJSDriver.)
|
|
302
|
-
trackingStateRaw: 2,
|
|
303
|
-
});
|
|
304
|
-
// Deps array intentionally minimal: only `plugin` actually
|
|
305
|
-
// requires worklet rebuild. All FoV / pose / counter / cadence
|
|
306
|
-
// values flow through stable shared-value refs that Reanimated
|
|
307
|
-
// wires through the producer-thread runtime independently of
|
|
308
|
-
// React's render cycle. (Adversarial-review M1.)
|
|
309
|
-
}, [plugin]);
|
|
310
|
-
// ── Return handle ───────────────────────────────────────────────
|
|
311
|
-
//
|
|
312
|
-
// Returns a getter for `isRunning` so callers always see the live
|
|
313
|
-
// state (the hook itself doesn't re-render on start/stop — that's
|
|
314
|
-
// intentional, avoids stale-Camera-prop churn).
|
|
91
|
+
stitcher.call(frame);
|
|
92
|
+
}, [stitcher.call]);
|
|
93
|
+
// Match pre-v0.11.0 contract: return `null` for `frameProcessor`
|
|
94
|
+
// until the underlying JSI plugin has resolved. `<Camera>` falls
|
|
95
|
+
// back to `undefined` in the null window so vision-camera doesn't
|
|
96
|
+
// try to bind an unready worklet.
|
|
315
97
|
return (0, react_1.useMemo)(() => ({
|
|
316
98
|
start,
|
|
317
99
|
stop,
|
|
318
|
-
frameProcessor:
|
|
100
|
+
frameProcessor: stitcher.isReady ? frameProcessor : null,
|
|
319
101
|
get isRunning() { return isRunningRef.current; },
|
|
320
|
-
}), [start, stop,
|
|
102
|
+
}), [start, stop, frameProcessor, stitcher.isReady]);
|
|
321
103
|
}
|
|
322
104
|
//# sourceMappingURL=useFrameProcessorDriver.js.map
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* useStitcherWorklet — exposes the lib's first-party stitching as a
|
|
3
|
+
* callable worklet function for host-composed Frame Processors.
|
|
4
|
+
*
|
|
5
|
+
* v0.11.0 — closes the v0.8.0 Phase 5 either-or constraint by letting
|
|
6
|
+
* hosts COMPOSE: write ONE `useFrameProcessor` worklet body that calls
|
|
7
|
+
* BOTH your custom logic AND the lib's first-party stitching, instead
|
|
8
|
+
* of one displacing the other. See `docs/host-app-integration.md`
|
|
9
|
+
* § Tier 3 composition for the pattern.
|
|
10
|
+
*
|
|
11
|
+
* ## Why this is a separate hook
|
|
12
|
+
*
|
|
13
|
+
* vision-camera v4 lets a `<Camera>` mount accept exactly ONE frame
|
|
14
|
+
* processor. Pre-v0.11.0, hosts that passed a `frameProcessor` prop
|
|
15
|
+
* to the lib's `<Camera>` REPLACED the lib's first-party stitching
|
|
16
|
+
* processor in non-AR mode. Composing required hand-writing both
|
|
17
|
+
* worklet bodies in the host's processor. v0.11.0 extracts the
|
|
18
|
+
* lib's worklet body into this hook so hosts can compose with a
|
|
19
|
+
* single call:
|
|
20
|
+
*
|
|
21
|
+
* const stitcher = useStitcherWorklet();
|
|
22
|
+
* const fp = useFrameProcessor((frame) => {
|
|
23
|
+
* 'worklet';
|
|
24
|
+
* hostPreLogic(frame);
|
|
25
|
+
* stitcher.call(frame); // ← lib's first-party stitching
|
|
26
|
+
* hostPostLogic(frame);
|
|
27
|
+
* }, [stitcher.call]);
|
|
28
|
+
* return <Camera frameProcessor={fp} ... />;
|
|
29
|
+
*
|
|
30
|
+
* AR mode is unaffected — the AR-session dispatch path (v0.8.0 Phase
|
|
31
|
+
* 4b.i / 4b.iii) already composes natively.
|
|
32
|
+
*
|
|
33
|
+
* ## What this owns
|
|
34
|
+
*
|
|
35
|
+
* - vc Frame Processor plugin acquisition for
|
|
36
|
+
* `cv_flow_gate_process_frame` (the same plugin the legacy
|
|
37
|
+
* `useFrameProcessorDriver` used; reentrant by construction).
|
|
38
|
+
* - Shared values backing pose (yaw / pitch / roll), throttle
|
|
39
|
+
* counter, every-N gate, and FoV-derived intrinsics scalars.
|
|
40
|
+
* - Gyro subscription on the JS thread (always-on between mount
|
|
41
|
+
* and unmount; subscription cost is tiny).
|
|
42
|
+
* - The worklet body itself: throttle → pose synthesis →
|
|
43
|
+
* `plugin.call(frame, params)`.
|
|
44
|
+
*
|
|
45
|
+
* ## Lifecycle
|
|
46
|
+
*
|
|
47
|
+
* - Gyro auto-subscribes on mount, auto-unsubscribes on unmount.
|
|
48
|
+
* Composed hosts get pose tracking for free.
|
|
49
|
+
* - `reset()` zeros the accumulated yaw / pitch / roll between
|
|
50
|
+
* captures. `useFrameProcessorDriver` calls this on `start()` to
|
|
51
|
+
* preserve pre-v0.11.0 per-capture pose-reset behaviour;
|
|
52
|
+
* composed hosts should call it at the start of each capture too
|
|
53
|
+
* (otherwise pose drifts across captures).
|
|
54
|
+
*
|
|
55
|
+
* ## Behaviour delta from pre-v0.11.0
|
|
56
|
+
*
|
|
57
|
+
* Before: `useFrameProcessorDriver.start()` subscribed the gyro;
|
|
58
|
+
* `stop()` unsubscribed. The subscription was tied to the
|
|
59
|
+
* capture lifecycle.
|
|
60
|
+
*
|
|
61
|
+
* After: the gyro is subscribed for the lifetime of this hook
|
|
62
|
+
* (i.e., as long as the component using it is mounted). In the
|
|
63
|
+
* default `<Camera>` integration the hook mounts when the camera
|
|
64
|
+
* screen mounts, so the practical effect is the same; in
|
|
65
|
+
* custom-composed integrations the host controls mount/unmount
|
|
66
|
+
* by mounting/unmounting the component that calls
|
|
67
|
+
* `useStitcherWorklet`. The battery delta is small: gyroscope
|
|
68
|
+
* sampling at 33ms costs ≪1% CPU on every Android/iOS device
|
|
69
|
+
* the lib supports.
|
|
70
|
+
*
|
|
71
|
+
* `pose reset` semantics are preserved via the new explicit
|
|
72
|
+
* `reset()` method. Hosts that previously relied on `start()`
|
|
73
|
+
* to zero pose now call `stitcher.reset()` at the capture start.
|
|
74
|
+
*
|
|
75
|
+
* ## Pose synthesis (verbatim from `useFrameProcessorDriver`)
|
|
76
|
+
*
|
|
77
|
+
* Quaternion: q = q_yaw * q_pitch * q_roll (Tait-Bryan YPR, body
|
|
78
|
+
* frame). Expanded:
|
|
79
|
+
* qx = cy*sp*cr + sy*cp*sr
|
|
80
|
+
* qy = sy*cp*cr - cy*sp*sr
|
|
81
|
+
* qz = cy*cp*sr - sy*sp*cr
|
|
82
|
+
* qw = cy*cp*cr + sy*sp*sr
|
|
83
|
+
*
|
|
84
|
+
* When roll=0 this collapses to the legacy 2-axis form so captures
|
|
85
|
+
* held level produce bit-identical poses to the pre-v0.6 driver
|
|
86
|
+
* (and bit-identical to v0.10.x's `useFrameProcessorDriver`).
|
|
87
|
+
*
|
|
88
|
+
* ## Throttling (verbatim)
|
|
89
|
+
*
|
|
90
|
+
* `evalEveryNFrames` controls how often the worklet calls the
|
|
91
|
+
* plugin. Default 1. Independent of — and stacks on top of —
|
|
92
|
+
* the stitcher's own internal `flowEvalEveryNFrames` in
|
|
93
|
+
* `KeyframeGate.swift`; effective cadence is the product.
|
|
94
|
+
*
|
|
95
|
+
* ## Pairing with `IncrementalStitcher.start`
|
|
96
|
+
*
|
|
97
|
+
* The plugin's per-frame call into `consumeFrameFromPlugin` is
|
|
98
|
+
* gated by `IncrementalStitcher.frameProcessorIngestEnabled`,
|
|
99
|
+
* which is TRUE only when the stitcher was started with
|
|
100
|
+
* `frameSourceMode === 'frameProcessor'`. Hosts MUST call
|
|
101
|
+
* `incrementalStitcher.start({ frameSourceMode: 'frameProcessor',
|
|
102
|
+
* ... })` to actually get frames into the engine — otherwise the
|
|
103
|
+
* worklet runs to completion but the wrapper drops the call.
|
|
104
|
+
* `Camera.tsx` does this wiring automatically when the host opts
|
|
105
|
+
* into the lib's `useFrameProcessorDriver`. Hosts that compose
|
|
106
|
+
* their own worklet via this hook must do the wiring themselves.
|
|
107
|
+
*/
|
|
108
|
+
import type { Frame } from 'react-native-vision-camera';
|
|
109
|
+
import type { StitcherFrame } from './StitcherFrame';
|
|
110
|
+
/**
|
|
111
|
+
* Frames the lib's stitching worklet accepts. Accepting either a
|
|
112
|
+
* vc `Frame` (what the host's `useFrameProcessor` body sees) or the
|
|
113
|
+
* lib's `StitcherFrame` (what the lib's `useFrameProcessor` body
|
|
114
|
+
* sees) keeps the same `useStitcherWorklet` usable from both kinds
|
|
115
|
+
* of host worklet bodies without a cast on the call site. The
|
|
116
|
+
* worklet only reads `width` / `height`; the rest of the frame
|
|
117
|
+
* object is forwarded verbatim to the native plugin.
|
|
118
|
+
*/
|
|
119
|
+
export type StitcherWorkletInput = Frame | StitcherFrame;
|
|
120
|
+
export interface UseStitcherWorkletOptions {
|
|
121
|
+
/**
|
|
122
|
+
* Gyro sample interval in ms (~30 Hz default). Drives the JS-
|
|
123
|
+
* thread pose integration loop; not the producer-thread plugin
|
|
124
|
+
* call rate.
|
|
125
|
+
*/
|
|
126
|
+
gyroIntervalMs?: number;
|
|
127
|
+
/**
|
|
128
|
+
* Approximate horizontal FoV of the device camera, used to
|
|
129
|
+
* synthesise `fx` from frame width. Default 65° matches a typical
|
|
130
|
+
* mid-tier smartphone main camera.
|
|
131
|
+
*/
|
|
132
|
+
fovHorizDegrees?: number;
|
|
133
|
+
/**
|
|
134
|
+
* Approximate vertical FoV of the device camera, used to
|
|
135
|
+
* synthesise `fy` from frame height. Default 50° matches a typical
|
|
136
|
+
* 4:3 phone camera in landscape; for 16:9 portrait you probably
|
|
137
|
+
* want ~75°.
|
|
138
|
+
*/
|
|
139
|
+
fovVertDegrees?: number;
|
|
140
|
+
/**
|
|
141
|
+
* Evaluate the plugin every Nth producer-thread frame. Default 1
|
|
142
|
+
* (every frame).
|
|
143
|
+
*/
|
|
144
|
+
evalEveryNFrames?: number;
|
|
145
|
+
}
|
|
146
|
+
export interface StitcherWorkletHandle {
|
|
147
|
+
/**
|
|
148
|
+
* Worklet function: pass a `StitcherFrame` to perform one frame of
|
|
149
|
+
* the lib's first-party stitching (throttle + pose synthesis +
|
|
150
|
+
* native plugin call). Safe to call from inside another
|
|
151
|
+
* `'worklet'`-prefixed function (this is the canonical
|
|
152
|
+
* composition pattern).
|
|
153
|
+
*
|
|
154
|
+
* The returned function reference is stable across re-renders as
|
|
155
|
+
* long as the plugin reference doesn't change (which happens at
|
|
156
|
+
* most once — at the moment the JSI plugin finishes
|
|
157
|
+
* registering). Include `stitcher.call` in your `useFrameProcessor`
|
|
158
|
+
* deps so the host worklet rebuilds when the plugin acquires.
|
|
159
|
+
*
|
|
160
|
+
* Safe to invoke before the plugin is ready: the worklet
|
|
161
|
+
* internally short-circuits (the frame is silently skipped).
|
|
162
|
+
* Hosts that want to display a "stitcher initialising…" UI can
|
|
163
|
+
* read `isReady` to gate their own behaviour.
|
|
164
|
+
*/
|
|
165
|
+
call: (frame: StitcherWorkletInput) => void;
|
|
166
|
+
/**
|
|
167
|
+
* Zero accumulated yaw / pitch / roll. Call at the start of each
|
|
168
|
+
* capture so the pose stream starts from `(0, 0, 0)` instead of
|
|
169
|
+
* carrying drift from the previous capture or from idle time
|
|
170
|
+
* between captures. Idempotent; safe to call from JS.
|
|
171
|
+
*/
|
|
172
|
+
reset: () => void;
|
|
173
|
+
/**
|
|
174
|
+
* `true` once the JSI Frame Processor plugin
|
|
175
|
+
* (`cv_flow_gate_process_frame`) has resolved. Before this flips
|
|
176
|
+
* `true`, `call(frame)` is a no-op (the plugin reference is
|
|
177
|
+
* `null`). Hosts integrating via `useFrameProcessorDriver` use
|
|
178
|
+
* this to decide whether to render the frame-processor at all —
|
|
179
|
+
* the driver returns `null` for `frameProcessor` until ready, so
|
|
180
|
+
* `<Camera>` falls back gracefully.
|
|
181
|
+
*/
|
|
182
|
+
isReady: boolean;
|
|
183
|
+
}
|
|
184
|
+
export declare function useStitcherWorklet(options?: UseStitcherWorkletOptions): StitcherWorkletHandle;
|
|
185
|
+
//# sourceMappingURL=useStitcherWorklet.d.ts.map
|