react-native-image-stitcher 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +119 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +17 -3
- package/android/src/main/java/io/imagestitcher/rn/SaveFrameAsJpegPlugin.kt +162 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +20 -1
- package/dist/stitching/useFrameStream.d.ts +34 -0
- package/dist/stitching/useFrameStream.js +219 -0
- package/dist/stitching/useThrottledFrameProcessor.d.ts +33 -0
- package/dist/stitching/useThrottledFrameProcessor.js +132 -0
- package/dist/types.d.ts +87 -0
- package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +185 -0
- package/package.json +1 -1
- package/src/index.ts +19 -0
- package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +178 -0
- package/src/stitching/useFrameStream.ts +255 -0
- package/src/stitching/useThrottledFrameProcessor.ts +145 -0
- package/src/types.ts +95 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
//
|
|
4
|
+
// v0.9.0 Layer 2 — throttle gate over v0.8.0's `useFrameProcessor`.
|
|
5
|
+
//
|
|
6
|
+
// ## What this is
|
|
7
|
+
//
|
|
8
|
+
// A thin wrapper around `useFrameProcessor` that enforces a maximum
|
|
9
|
+
// invocation rate (`sampleHz`) at the worklet layer. The host's
|
|
10
|
+
// worklet fires up to `sampleHz` times per second; ticks too close
|
|
11
|
+
// together are dropped via a `useSharedValue<number>` monotonic-time
|
|
12
|
+
// gate inside the worklet body.
|
|
13
|
+
//
|
|
14
|
+
// ## When to use this (vs alternatives)
|
|
15
|
+
//
|
|
16
|
+
// - **`useFrameProcessor` directly** — every camera frame (~30-60 Hz).
|
|
17
|
+
// Use for true-realtime processing that wants to see every frame.
|
|
18
|
+
// - **`useThrottledFrameProcessor`** (this hook) — sub-frame-rate
|
|
19
|
+
// worklet-native processing. The worklet runtime has direct
|
|
20
|
+
// access to `frame.toArrayBuffer()`, `frame.arDepth`,
|
|
21
|
+
// `frame.arAnchors`, and can call other vc Frame Processor plugins
|
|
22
|
+
// (native OCR libraries, TFLite ML inference, etc.). Results
|
|
23
|
+
// bridged to JS via `runOnJS`.
|
|
24
|
+
// - **`useFrameStream`** (Layer 3, also in this directory) —
|
|
25
|
+
// sub-frame-rate JS-thread consumer. The lib JPEG-encodes each
|
|
26
|
+
// sample on the producer thread and delivers a `SampledFrame`
|
|
27
|
+
// (file path + pose + dims) to a JS-thread callback. Use for
|
|
28
|
+
// file-path OCR libraries (RN modules wrapping ML Kit etc.),
|
|
29
|
+
// cloud upload, thumbnail UI.
|
|
30
|
+
//
|
|
31
|
+
// ## Use-case mapping (canonical)
|
|
32
|
+
//
|
|
33
|
+
// | Use case | Layer | Why |
|
|
34
|
+
// |---------------------------------------|-------|----------------------------------|
|
|
35
|
+
// | OCR via Vision.framework / ML Kit | **2** | native libs, bbox in frame coords|
|
|
36
|
+
// | TFLite ML detection (via vc plugin) | **2** | same shape as OCR |
|
|
37
|
+
// | LiDAR depth → 3D reconstruction | **2** | depth too large to bridge |
|
|
38
|
+
// | Pose-only telemetry | **2** | tiny payload, no encoding needed |
|
|
39
|
+
// | File-path OCR (RN module) | 3 | host wants a JPEG, not pixels |
|
|
40
|
+
// | Cloud upload (sampled JPEG feed) | 3 | JPEG IS the payload |
|
|
41
|
+
// | Live thumbnail preview UI | 3 | `<Image source={{uri: ...}}>` |
|
|
42
|
+
//
|
|
43
|
+
// See `docs/host-app-integration.md` § "Tier 2 + 3" for recipes.
|
|
44
|
+
//
|
|
45
|
+
// ## Threading
|
|
46
|
+
//
|
|
47
|
+
// The wrapped worklet fires on whatever runtime `useFrameProcessor`
|
|
48
|
+
// dispatches on:
|
|
49
|
+
// - **Non-AR mode**: vision-camera's Frame Processor runtime
|
|
50
|
+
// (producer thread).
|
|
51
|
+
// - **AR mode**: the lib's `RNSARWorkletRuntime` (iOS) /
|
|
52
|
+
// worklets-core default context (Android) — fired by the AR
|
|
53
|
+
// session's per-frame dispatch. See v0.8.0 Phase 4b.i / 4b.iii.
|
|
54
|
+
//
|
|
55
|
+
// Either way, the worklet MUST NOT block — the next frame's
|
|
56
|
+
// processing is gated on this one returning. Long work belongs
|
|
57
|
+
// behind `runOnJS` / a separate worklet runtime.
|
|
58
|
+
//
|
|
59
|
+
// ## Behaviour at the throttle boundary
|
|
60
|
+
//
|
|
61
|
+
// The hook tracks a monotonic-time shared value of "last sample time".
|
|
62
|
+
// Each tick checks if `frame.timestamp - lastSampleMs.value >=
|
|
63
|
+
// (1000 / sampleHz)`. If yes, the worklet body runs and the value
|
|
64
|
+
// updates; if no, the worklet returns silently.
|
|
65
|
+
//
|
|
66
|
+
// Edge cases:
|
|
67
|
+
// - First-ever tick: `lastSampleMs.value` starts at 0; first frame's
|
|
68
|
+
// timestamp will be >> 0 → first tick always fires. Subsequent
|
|
69
|
+
// ticks throttle as expected.
|
|
70
|
+
// - vc v4 timestamp semantics: per the project's worklet-throttle
|
|
71
|
+
// gotcha note, `frame.timestamp` is NOT reliably nanoseconds in
|
|
72
|
+
// vc v4. The hook treats `frame.timestamp` as ALREADY in
|
|
73
|
+
// milliseconds (which is what vc v4 actually delivers; the
|
|
74
|
+
// v0.8.0 StitcherFrame contract documents this). If a future
|
|
75
|
+
// vc version changes the unit, the throttle math here needs
|
|
76
|
+
// re-checking.
|
|
77
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
78
|
+
exports.useThrottledFrameProcessor = useThrottledFrameProcessor;
|
|
79
|
+
const react_native_worklets_core_1 = require("react-native-worklets-core");
|
|
80
|
+
const useFrameProcessor_1 = require("./useFrameProcessor");
|
|
81
|
+
/**
|
|
82
|
+
* Throttled variant of `useFrameProcessor`. See the module
|
|
83
|
+
* docstring for the full use-case mapping; quick version:
|
|
84
|
+
*
|
|
85
|
+
* ```tsx
|
|
86
|
+
* const fp = useThrottledFrameProcessor(
|
|
87
|
+
* (frame) => {
|
|
88
|
+
* 'worklet';
|
|
89
|
+
* // worklet-native OCR / ML / depth processing here
|
|
90
|
+
* },
|
|
91
|
+
* { sampleHz: 2 },
|
|
92
|
+
* [],
|
|
93
|
+
* );
|
|
94
|
+
* return <Camera frameProcessor={fp} ... />;
|
|
95
|
+
* ```
|
|
96
|
+
*
|
|
97
|
+
* @param worklet Host's frame-processor worklet. Must be
|
|
98
|
+
* `'worklet'`-prefixed. Runs at most `sampleHz`
|
|
99
|
+
* times per second.
|
|
100
|
+
* @param options `{ sampleHz }` — clamped to `[0.5, 30]`.
|
|
101
|
+
* @param deps Standard React deps array. Treated the same as
|
|
102
|
+
* `useFrameProcessor`'s deps — when they change the
|
|
103
|
+
* inner worklet is re-bound.
|
|
104
|
+
*
|
|
105
|
+
* @returns A `useFrameProcessor`-shaped processor object, pass it
|
|
106
|
+
* to `<Camera frameProcessor={...}>`.
|
|
107
|
+
*/
|
|
108
|
+
function useThrottledFrameProcessor(worklet, options, deps) {
|
|
109
|
+
// Clamp + derive interval. Done outside the worklet so the
|
|
110
|
+
// useSharedValue / useFrameProcessor hooks see stable values.
|
|
111
|
+
const sampleHz = Math.max(0.5, Math.min(30, options.sampleHz));
|
|
112
|
+
const minIntervalMs = 1000 / sampleHz;
|
|
113
|
+
// Monotonic-time gate. Initialised to 0 → first tick always
|
|
114
|
+
// fires (frame.timestamp >> 0).
|
|
115
|
+
const lastSampleMs = (0, react_native_worklets_core_1.useSharedValue)(0);
|
|
116
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
117
|
+
return (0, useFrameProcessor_1.useFrameProcessor)((frame) => {
|
|
118
|
+
'worklet';
|
|
119
|
+
const now = frame.timestamp;
|
|
120
|
+
if (now - lastSampleMs.value < minIntervalMs) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
lastSampleMs.value = now;
|
|
124
|
+
worklet(frame);
|
|
125
|
+
},
|
|
126
|
+
// The throttle interval is captured in the worklet closure; if
|
|
127
|
+
// it changes we need to re-bind the worklet so the new
|
|
128
|
+
// `minIntervalMs` takes effect. Same for the host's worklet
|
|
129
|
+
// identity (so deps changes on the host side re-bind too).
|
|
130
|
+
[minIntervalMs, worklet, ...deps]);
|
|
131
|
+
}
|
|
132
|
+
//# sourceMappingURL=useThrottledFrameProcessor.js.map
|
package/dist/types.d.ts
CHANGED
|
@@ -35,6 +35,93 @@ export interface DeviceMetadata {
|
|
|
35
35
|
cameraId: string;
|
|
36
36
|
flashEnabled: boolean;
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* v0.9.0 Layer 3 — one sampled frame delivered by `useFrameStream`
|
|
40
|
+
* to the JS-thread handler.
|
|
41
|
+
*
|
|
42
|
+
* The JPEG file at `jpegPath` is the stream's own copy. Hosts that
|
|
43
|
+
* need long-term retention MUST copy the file synchronously inside
|
|
44
|
+
* the handler — the same path may be overwritten by a subsequent
|
|
45
|
+
* sample (slot reuse — see the hook's docstring for the rotation
|
|
46
|
+
* policy).
|
|
47
|
+
*/
|
|
48
|
+
export interface SampledFrame {
|
|
49
|
+
/** Absolute filesystem path to the JPEG. No `file://` prefix. */
|
|
50
|
+
jpegPath: string;
|
|
51
|
+
/**
|
|
52
|
+
* Pose at sample time. `translation` is `undefined` in non-AR
|
|
53
|
+
* mode (gyro provides rotation only; no spatial anchor).
|
|
54
|
+
*/
|
|
55
|
+
pose: {
|
|
56
|
+
rotation: [number, number, number, number];
|
|
57
|
+
translation?: [number, number, number];
|
|
58
|
+
};
|
|
59
|
+
/** Frame timestamp (ms; per the v0.8.0 StitcherFrame contract). */
|
|
60
|
+
timestamp: number;
|
|
61
|
+
/** JPEG width / height in pixels. */
|
|
62
|
+
width: number;
|
|
63
|
+
height: number;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* v0.9.0 Layer 3 — options for `useFrameStream`.
|
|
67
|
+
*
|
|
68
|
+
* For worklet-native processing without JPEG roundtrip (OCR via
|
|
69
|
+
* Vision/ML Kit, TFLite ML, LiDAR depth), use
|
|
70
|
+
* `useThrottledFrameProcessor` (Layer 2) instead.
|
|
71
|
+
*/
|
|
72
|
+
export interface FrameStreamOptions {
|
|
73
|
+
/**
|
|
74
|
+
* Target sampling rate in Hertz. Clamped to `[0.5, 10]`. The
|
|
75
|
+
* Layer 2 throttle gate enforces the rate inside the worklet;
|
|
76
|
+
* ticks too close together are dropped silently.
|
|
77
|
+
*
|
|
78
|
+
* Clamp upper bound (10 Hz) is intentionally lower than Layer 2's
|
|
79
|
+
* (30 Hz) — beyond 10 Hz the per-frame JPEG encode + JS-bridge
|
|
80
|
+
* cost dominates the wall-clock budget. Hosts that need higher
|
|
81
|
+
* rates should be on Layer 2 with their own JPEG encoder call
|
|
82
|
+
* (or no JPEG at all).
|
|
83
|
+
*/
|
|
84
|
+
sampleHz: number;
|
|
85
|
+
/**
|
|
86
|
+
* JPEG quality (0-100). Default 75. Clamped silently to
|
|
87
|
+
* `[1, 100]` by the underlying `save_frame_as_jpeg` native plugin.
|
|
88
|
+
*/
|
|
89
|
+
quality?: number;
|
|
90
|
+
/**
|
|
91
|
+
* Directory to write JPEG files into. Defaults to a per-app
|
|
92
|
+
* `<cache>/rnis-frame-stream/` subdirectory. The directory is
|
|
93
|
+
* `mkdir -p`'d on first use; hosts that supply an existing
|
|
94
|
+
* absolute path are responsible for its lifecycle.
|
|
95
|
+
*/
|
|
96
|
+
outputDir?: string;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* v0.9.0 Layer 2 — options for `useThrottledFrameProcessor`.
|
|
100
|
+
*
|
|
101
|
+
* Wraps v0.8.0's `useFrameProcessor` with a monotonic-time throttle
|
|
102
|
+
* gate so the supplied worklet fires at most `sampleHz` times per
|
|
103
|
+
* second. Use for sub-frame-rate worklet-native processing — native
|
|
104
|
+
* OCR (Vision.framework / ML Kit), TFLite ML detection, LiDAR depth
|
|
105
|
+
* processing — where the bbox / depth payloads are small enough to
|
|
106
|
+
* bridge to JS via `runOnJS`.
|
|
107
|
+
*
|
|
108
|
+
* For JS-thread JPEG consumers (file-path OCR libraries, cloud
|
|
109
|
+
* upload, thumbnail UI), use `useFrameStream` (Layer 3) instead.
|
|
110
|
+
*/
|
|
111
|
+
export interface ThrottledFrameProcessorOptions {
|
|
112
|
+
/**
|
|
113
|
+
* Target sampling rate in Hertz. Clamped to `[0.5, 30]`. Inside
|
|
114
|
+
* the worklet a monotonic-time gate enforces the rate; ticks too
|
|
115
|
+
* close together are silently dropped.
|
|
116
|
+
*
|
|
117
|
+
* The clamp upper bound (30 Hz) sits at typical AR rates on
|
|
118
|
+
* mid-range Android devices — beyond that, the host should just
|
|
119
|
+
* use `useFrameProcessor` directly (no throttle). The clamp
|
|
120
|
+
* lower bound (0.5 Hz) prevents accidentally-zero-divide values
|
|
121
|
+
* + matches `useFrameStream`'s convention.
|
|
122
|
+
*/
|
|
123
|
+
sampleHz: number;
|
|
124
|
+
}
|
|
38
125
|
export interface CaptureResult {
|
|
39
126
|
/** Unique device-generated UUID */
|
|
40
127
|
deviceUuid: string;
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// SaveFrameAsJpegPlugin.mm — v0.9.0 Layer 1: vc Frame Processor plugin
|
|
4
|
+
// that JPEG-encodes the supplied frame's pixel buffer to a host-
|
|
5
|
+
// supplied path. Worklet-callable; thin wrapper around the standard
|
|
6
|
+
// iOS CIImage → CGImage → UIImage → UIImageJPEGRepresentation path.
|
|
7
|
+
//
|
|
8
|
+
// JS-side usage (from a worklet — typically inside `useFrameStream`
|
|
9
|
+
// (Layer 3) or directly from a custom `useFrameProcessor` body):
|
|
10
|
+
//
|
|
11
|
+
// const plugin = VisionCameraProxy.initFrameProcessorPlugin(
|
|
12
|
+
// 'save_frame_as_jpeg', {},
|
|
13
|
+
// );
|
|
14
|
+
//
|
|
15
|
+
// const fp = useFrameProcessor((frame) => {
|
|
16
|
+
// 'worklet';
|
|
17
|
+
// if (plugin == null) return;
|
|
18
|
+
// const result = plugin.call(frame, {
|
|
19
|
+
// path: '/path/to/output.jpg',
|
|
20
|
+
// quality: 75, // 0-100; defaults to 75
|
|
21
|
+
// });
|
|
22
|
+
// // result: { ok: true, path, width, height } OR
|
|
23
|
+
// // { ok: false, error: "..." }
|
|
24
|
+
// }, [plugin]);
|
|
25
|
+
//
|
|
26
|
+
// ## Why a separate plugin (not folded into KeyframeGateFrameProcessor)
|
|
27
|
+
//
|
|
28
|
+
// `cv_flow_gate_process_frame` (the existing plugin) drives the lib's
|
|
29
|
+
// FIRST-PARTY stitching pipeline: it consumes the frame, evaluates
|
|
30
|
+
// the keyframe gate, dispatches into `IncrementalStitcher`. It owns
|
|
31
|
+
// state.
|
|
32
|
+
//
|
|
33
|
+
// `save_frame_as_jpeg` is STATELESS — a pure encode-and-write function.
|
|
34
|
+
// Mixing them would force every JS-side caller of either to pay both
|
|
35
|
+
// codepaths' arg-parsing costs (and would confuse the use-case
|
|
36
|
+
// boundary). Two plugins, one job each.
|
|
37
|
+
//
|
|
38
|
+
// ## CONDITIONAL COMPILATION
|
|
39
|
+
//
|
|
40
|
+
// Same `__has_include` guard as `KeyframeGateFrameProcessor.mm` — if
|
|
41
|
+
// vision-camera isn't on the host's classpath, this file is a no-op
|
|
42
|
+
// translation unit. See that file's header for the rationale.
|
|
43
|
+
|
|
44
|
+
#import <Foundation/Foundation.h>
|
|
45
|
+
|
|
46
|
+
#if __has_include(<VisionCamera/FrameProcessorPlugin.h>)
|
|
47
|
+
|
|
48
|
+
#import <VisionCamera/Frame.h>
|
|
49
|
+
#import <VisionCamera/FrameProcessorPlugin.h>
|
|
50
|
+
#import <VisionCamera/FrameProcessorPluginRegistry.h>
|
|
51
|
+
#import <VisionCamera/VisionCameraProxyHolder.h>
|
|
52
|
+
#import <CoreVideo/CoreVideo.h>
|
|
53
|
+
#import <CoreImage/CoreImage.h>
|
|
54
|
+
#import <UIKit/UIKit.h>
|
|
55
|
+
|
|
56
|
+
@interface SaveFrameAsJpegPlugin : FrameProcessorPlugin
|
|
57
|
+
@end
|
|
58
|
+
|
|
59
|
+
@implementation SaveFrameAsJpegPlugin
|
|
60
|
+
|
|
61
|
+
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy
|
|
62
|
+
withOptions:(NSDictionary* _Nullable)options {
|
|
63
|
+
return [super initWithProxy:proxy withOptions:options];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Helper: read a string arg with a fallback. Returns nil only when
|
|
67
|
+
// the arg is missing AND no fallback was supplied.
|
|
68
|
+
static NSString* sfj_argString(NSDictionary* args, NSString* key,
|
|
69
|
+
NSString* _Nullable fallback) {
|
|
70
|
+
id v = args[key];
|
|
71
|
+
if ([v isKindOfClass:[NSString class]]) return (NSString*)v;
|
|
72
|
+
return fallback;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Helper: read a numeric arg (NSNumber or NSString-parseable) with a
|
|
76
|
+
// fallback. Matches the pattern in KeyframeGateFrameProcessor.mm.
|
|
77
|
+
static double sfj_argDouble(NSDictionary* args, NSString* key,
|
|
78
|
+
double fallback) {
|
|
79
|
+
id v = args[key];
|
|
80
|
+
if ([v isKindOfClass:[NSNumber class]]) return [(NSNumber*)v doubleValue];
|
|
81
|
+
if ([v isKindOfClass:[NSString class]]) return [(NSString*)v doubleValue];
|
|
82
|
+
return fallback;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// The host-callable plugin entry point. vc dispatches each
|
|
86
|
+
// `plugin.call(frame, args)` from a worklet here.
|
|
87
|
+
//
|
|
88
|
+
// ## Arguments
|
|
89
|
+
//
|
|
90
|
+
// - `path` (string, REQUIRED): absolute filesystem path to write
|
|
91
|
+
// the JPEG to. Parent directory must exist (we don't `mkdir -p`).
|
|
92
|
+
// Existing file is overwritten atomically.
|
|
93
|
+
// - `quality` (number, optional): 0-100 JPEG quality. Default 75
|
|
94
|
+
// (matches `KeyframeGate.onAccept`'s encoder). Clamped silently
|
|
95
|
+
// to `[1, 100]`.
|
|
96
|
+
//
|
|
97
|
+
// ## Returns
|
|
98
|
+
//
|
|
99
|
+
// - On success: `{ ok: YES, path: <path>, width: <px>, height: <px> }`
|
|
100
|
+
// - On failure: `{ ok: NO, error: "<reason>" }`
|
|
101
|
+
//
|
|
102
|
+
// Errors are surfaced via the result dict, NOT thrown as `JSError` —
|
|
103
|
+
// host worklets that want to react to encoder failures (e.g., to
|
|
104
|
+
// rotate slot paths, or to back off) can branch on `result.ok`
|
|
105
|
+
// without try/catch boilerplate. Throwing would break the
|
|
106
|
+
// Layer 3 `useFrameStream` flow which only sees the result.
|
|
107
|
+
- (id)callback:(Frame*)frame withArguments:(NSDictionary*)arguments {
|
|
108
|
+
NSString* path = sfj_argString(arguments, @"path", nil);
|
|
109
|
+
if (path == nil) {
|
|
110
|
+
return @{@"ok": @NO, @"error": @"missing required `path` argument"};
|
|
111
|
+
}
|
|
112
|
+
double q = sfj_argDouble(arguments, @"quality", 75.0);
|
|
113
|
+
if (q < 1.0) q = 1.0;
|
|
114
|
+
if (q > 100.0) q = 100.0;
|
|
115
|
+
|
|
116
|
+
CMSampleBufferRef sampleBuffer = frame.buffer;
|
|
117
|
+
if (sampleBuffer == NULL) {
|
|
118
|
+
return @{@"ok": @NO, @"error": @"frame.buffer was NULL"};
|
|
119
|
+
}
|
|
120
|
+
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
|
|
121
|
+
if (pixelBuffer == NULL) {
|
|
122
|
+
return @{@"ok": @NO, @"error": @"CMSampleBufferGetImageBuffer returned NULL"};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// CIImage → CGImage → UIImage → JPEG. Standard iOS path; the
|
|
126
|
+
// CIContext + colorSpace are cheap to construct per-call (CoreImage
|
|
127
|
+
// caches GPU resources internally). If profiling shows this in
|
|
128
|
+
// the hot path, lift the context to a static; for v0.9.0 baseline,
|
|
129
|
+
// per-call construction is fine.
|
|
130
|
+
CIImage* ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
|
|
131
|
+
if (ciImage == nil) {
|
|
132
|
+
return @{@"ok": @NO, @"error": @"CIImage imageWithCVPixelBuffer returned nil"};
|
|
133
|
+
}
|
|
134
|
+
CIContext* ctx = [CIContext context];
|
|
135
|
+
CGImageRef cgImage = [ctx createCGImage:ciImage fromRect:ciImage.extent];
|
|
136
|
+
if (cgImage == NULL) {
|
|
137
|
+
return @{@"ok": @NO, @"error": @"CIContext createCGImage failed"};
|
|
138
|
+
}
|
|
139
|
+
UIImage* uiImage = [UIImage imageWithCGImage:cgImage];
|
|
140
|
+
size_t width = CGImageGetWidth(cgImage);
|
|
141
|
+
size_t height = CGImageGetHeight(cgImage);
|
|
142
|
+
CGImageRelease(cgImage);
|
|
143
|
+
|
|
144
|
+
NSData* jpegData = UIImageJPEGRepresentation(uiImage, (CGFloat)(q / 100.0));
|
|
145
|
+
if (jpegData == nil) {
|
|
146
|
+
return @{@"ok": @NO, @"error": @"UIImageJPEGRepresentation returned nil"};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Atomic write — under the hood NSData writes to a temp file then
|
|
150
|
+
// renames. Avoids torn writes if a reader tries to open the path
|
|
151
|
+
// mid-write (would otherwise see a partial JPEG and choke).
|
|
152
|
+
NSError* err = nil;
|
|
153
|
+
BOOL ok = [jpegData writeToFile:path
|
|
154
|
+
options:NSDataWritingAtomic
|
|
155
|
+
error:&err];
|
|
156
|
+
if (!ok) {
|
|
157
|
+
NSString* msg = err.localizedDescription ?: @"NSData writeToFile returned NO";
|
|
158
|
+
return @{@"ok": @NO, @"error": msg};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return @{
|
|
162
|
+
@"ok": @YES,
|
|
163
|
+
@"path": path,
|
|
164
|
+
@"width": @(width),
|
|
165
|
+
@"height": @(height),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// Auto-register the plugin at class-load time. Name must match what
|
|
170
|
+
// JS passes to `VisionCameraProxy.initFrameProcessorPlugin('save_frame_as_jpeg')`.
|
|
171
|
+
// Same pattern as KeyframeGateFrameProcessor's +load.
|
|
172
|
+
+ (void)load {
|
|
173
|
+
[FrameProcessorPluginRegistry
|
|
174
|
+
addFrameProcessorPlugin:@"save_frame_as_jpeg"
|
|
175
|
+
withInitializer:^FrameProcessorPlugin* _Nonnull(
|
|
176
|
+
VisionCameraProxyHolder* proxy,
|
|
177
|
+
NSDictionary* _Nullable options) {
|
|
178
|
+
return [[SaveFrameAsJpegPlugin alloc]
|
|
179
|
+
initWithProxy:proxy withOptions:options];
|
|
180
|
+
}];
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
@end
|
|
184
|
+
|
|
185
|
+
#endif // __has_include(<VisionCamera/FrameProcessorPlugin.h>)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "react-native-image-stitcher",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
package/src/index.ts
CHANGED
|
@@ -198,6 +198,25 @@ export type {
|
|
|
198
198
|
// cross-runtime handoff (the AR runtime iterating the registry).
|
|
199
199
|
// See the hook's docstring + StitcherFrame.ts for the contract.
|
|
200
200
|
export { useFrameProcessor } from './stitching/useFrameProcessor';
|
|
201
|
+
// v0.9.0 Layer 2 — `useThrottledFrameProcessor`. Throttle gate over
|
|
202
|
+
// `useFrameProcessor` for sub-frame-rate worklet-native processing
|
|
203
|
+
// (native OCR via Vision.framework / ML Kit, TFLite ML detection,
|
|
204
|
+
// LiDAR depth). The worklet runtime has direct access to
|
|
205
|
+
// `frame.toArrayBuffer()` / `frame.arDepth`; bridge small payloads
|
|
206
|
+
// (bboxes, depth-derived metrics) to JS via `runOnJS`. For JS-thread
|
|
207
|
+
// JPEG consumers (file-path OCR libs, cloud upload, thumbnail UI),
|
|
208
|
+
// prefer `useFrameStream` (Layer 3, ships in the same release).
|
|
209
|
+
export { useThrottledFrameProcessor } from './stitching/useThrottledFrameProcessor';
|
|
210
|
+
export type { ThrottledFrameProcessorOptions } from './types';
|
|
211
|
+
// v0.9.0 Layer 3 — `useFrameStream`. JS-thread sampled-frame
|
|
212
|
+
// stream over Layer 1 (`save_frame_as_jpeg` vc plugin) + Layer 2
|
|
213
|
+
// (`useThrottledFrameProcessor`). Use for JS-thread consumers:
|
|
214
|
+
// file-path OCR libs (RN modules), cloud upload, thumbnail UI.
|
|
215
|
+
// For worklet-native processing (Vision/ML Kit as vc plugins,
|
|
216
|
+
// TFLite ML, LiDAR depth), prefer `useThrottledFrameProcessor`
|
|
217
|
+
// (Layer 2) — lower latency, no JPEG roundtrip.
|
|
218
|
+
export { useFrameStream } from './stitching/useFrameStream';
|
|
219
|
+
export type { FrameStreamOptions, SampledFrame } from './types';
|
|
201
220
|
// vision-camera Frame Processor driver for non-AR captures. As
|
|
202
221
|
// of v0.6 the only non-AR driver exported (the legacy
|
|
203
222
|
// `useIncrementalJSDriver` was removed; was deprecated in v0.5).
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
/**
|
|
3
|
+
* Unit tests for the v0.9.0 Layer 2 `useThrottledFrameProcessor` hook.
|
|
4
|
+
*
|
|
5
|
+
* The worklet runtime can't run in jest (no JSI, no worklets-core).
|
|
6
|
+
* What we CAN test:
|
|
7
|
+
*
|
|
8
|
+
* - The `sampleHz` clamping (`[0.5, 30]`)
|
|
9
|
+
* - `minIntervalMs` math (1000 / sampleHz)
|
|
10
|
+
* - The deps propagation (host's deps → useFrameProcessor's deps)
|
|
11
|
+
* - The throttle gate logic (extracted as a pure function for
|
|
12
|
+
* isolated verification — see `_throttleGateForTests`).
|
|
13
|
+
*
|
|
14
|
+
* The hook itself is tested via a thin React-renderer-free harness:
|
|
15
|
+
* we mock `useFrameProcessor` + `useSharedValue` so we can verify
|
|
16
|
+
* the call shape without booting the worklet runtime.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { useThrottledFrameProcessor } from '../useThrottledFrameProcessor';
|
|
20
|
+
|
|
21
|
+
// ─── Mock vision-camera + worklets-core ─────────────────────────────
|
|
22
|
+
// These are minimal-shim mocks — enough surface for the hook to call
|
|
23
|
+
// `useFrameProcessor(workletBody, deps)` and `useSharedValue(0)`.
|
|
24
|
+
|
|
25
|
+
const useFrameProcessorMock = jest.fn();
|
|
26
|
+
const useSharedValueMock = jest.fn();
|
|
27
|
+
|
|
28
|
+
jest.mock('../useFrameProcessor', () => ({
|
|
29
|
+
useFrameProcessor: (...args: unknown[]) => useFrameProcessorMock(...args),
|
|
30
|
+
}));
|
|
31
|
+
|
|
32
|
+
jest.mock('react-native-worklets-core', () => ({
|
|
33
|
+
useSharedValue: (initial: number) => useSharedValueMock(initial),
|
|
34
|
+
}));
|
|
35
|
+
|
|
36
|
+
describe('useThrottledFrameProcessor', () => {
|
|
37
|
+
beforeEach(() => {
|
|
38
|
+
useFrameProcessorMock.mockReset();
|
|
39
|
+
useSharedValueMock.mockReset();
|
|
40
|
+
// Default behaviour for useSharedValue: return an object with a
|
|
41
|
+
// mutable `.value` field (mirrors worklets-core's API).
|
|
42
|
+
useSharedValueMock.mockImplementation((initial: number) => ({
|
|
43
|
+
value: initial,
|
|
44
|
+
}));
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
describe('sampleHz clamping', () => {
|
|
48
|
+
it('clamps below 0.5 to 0.5', () => {
|
|
49
|
+
const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
|
|
50
|
+
useThrottledFrameProcessor(noop, { sampleHz: 0.1 }, []);
|
|
51
|
+
// useFrameProcessor receives the wrapped worklet; the deps
|
|
52
|
+
// array's first entry is `minIntervalMs`. For sampleHz=0.5,
|
|
53
|
+
// minIntervalMs = 2000.
|
|
54
|
+
const [, deps] = useFrameProcessorMock.mock.calls[0]!;
|
|
55
|
+
expect(deps[0]).toBeCloseTo(2000);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('clamps above 30 to 30', () => {
|
|
59
|
+
const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
|
|
60
|
+
useThrottledFrameProcessor(noop, { sampleHz: 999 }, []);
|
|
61
|
+
const [, deps] = useFrameProcessorMock.mock.calls[0]!;
|
|
62
|
+
// sampleHz=30 → minIntervalMs = 33.333...
|
|
63
|
+
expect(deps[0]).toBeCloseTo(1000 / 30);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('passes through in-range sampleHz unchanged', () => {
|
|
67
|
+
const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
|
|
68
|
+
useThrottledFrameProcessor(noop, { sampleHz: 2 }, []);
|
|
69
|
+
const [, deps] = useFrameProcessorMock.mock.calls[0]!;
|
|
70
|
+
expect(deps[0]).toBeCloseTo(500);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it('accepts boundary values exactly', () => {
|
|
74
|
+
const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
|
|
75
|
+
useThrottledFrameProcessor(noop, { sampleHz: 0.5 }, []);
|
|
76
|
+
let deps = useFrameProcessorMock.mock.calls[0]![1];
|
|
77
|
+
expect(deps[0]).toBeCloseTo(2000);
|
|
78
|
+
|
|
79
|
+
useFrameProcessorMock.mockClear();
|
|
80
|
+
useThrottledFrameProcessor(noop, { sampleHz: 30 }, []);
|
|
81
|
+
deps = useFrameProcessorMock.mock.calls[0]![1];
|
|
82
|
+
expect(deps[0]).toBeCloseTo(1000 / 30);
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
describe('deps propagation', () => {
|
|
87
|
+
it('appends host deps after the internal interval + worklet deps', () => {
|
|
88
|
+
const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
|
|
89
|
+
const hostDep1 = { id: 'a' };
|
|
90
|
+
const hostDep2 = 42;
|
|
91
|
+
useThrottledFrameProcessor(noop, { sampleHz: 2 }, [hostDep1, hostDep2]);
|
|
92
|
+
const [, deps] = useFrameProcessorMock.mock.calls[0]!;
|
|
93
|
+
// Expected shape: [minIntervalMs, worklet, ...hostDeps]
|
|
94
|
+
expect(deps).toHaveLength(4);
|
|
95
|
+
expect(deps[0]).toBeCloseTo(500);
|
|
96
|
+
expect(deps[1]).toBe(noop);
|
|
97
|
+
expect(deps[2]).toBe(hostDep1);
|
|
98
|
+
expect(deps[3]).toBe(hostDep2);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('with empty host deps: deps = [minIntervalMs, worklet]', () => {
|
|
102
|
+
const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
|
|
103
|
+
useThrottledFrameProcessor(noop, { sampleHz: 2 }, []);
|
|
104
|
+
const [, deps] = useFrameProcessorMock.mock.calls[0]!;
|
|
105
|
+
expect(deps).toHaveLength(2);
|
|
106
|
+
expect(deps[0]).toBeCloseTo(500);
|
|
107
|
+
expect(deps[1]).toBe(noop);
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
describe('throttle gate', () => {
|
|
112
|
+
// The throttle logic lives INSIDE the wrapped worklet body, which
|
|
113
|
+
// jest can't execute directly (it's a `'worklet'`-prefixed
|
|
114
|
+
// function). But the wrapped function IS just a plain JS
|
|
115
|
+
// function until the worklets-core babel plugin transforms it,
|
|
116
|
+
// so we can call it manually with mock frames + a mock
|
|
117
|
+
// shared-value gate.
|
|
118
|
+
//
|
|
119
|
+
// The body's logic:
|
|
120
|
+
// if (frame.timestamp - lastSampleMs.value < minIntervalMs) return;
|
|
121
|
+
// lastSampleMs.value = frame.timestamp;
|
|
122
|
+
// worklet(frame);
|
|
123
|
+
|
|
124
|
+
it('fires the worklet on the first frame regardless of timestamp', () => {
|
|
125
|
+
const hostWorklet = jest.fn();
|
|
126
|
+
useThrottledFrameProcessor(hostWorklet, { sampleHz: 2 }, []);
|
|
127
|
+
const [wrappedBody] = useFrameProcessorMock.mock.calls[0]!;
|
|
128
|
+
|
|
129
|
+
const frame = { timestamp: 12345 } as Parameters<typeof hostWorklet>[0];
|
|
130
|
+
wrappedBody(frame);
|
|
131
|
+
|
|
132
|
+
expect(hostWorklet).toHaveBeenCalledTimes(1);
|
|
133
|
+
expect(hostWorklet).toHaveBeenCalledWith(frame);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('skips a frame too close to the previous sample', () => {
|
|
137
|
+
const hostWorklet = jest.fn();
|
|
138
|
+
useThrottledFrameProcessor(hostWorklet, { sampleHz: 2 }, []); // 500ms interval
|
|
139
|
+
const [wrappedBody] = useFrameProcessorMock.mock.calls[0]!;
|
|
140
|
+
|
|
141
|
+
wrappedBody({ timestamp: 1000 } as never);
|
|
142
|
+
wrappedBody({ timestamp: 1100 } as never); // 100ms after — too soon
|
|
143
|
+
wrappedBody({ timestamp: 1200 } as never); // 200ms after — too soon
|
|
144
|
+
|
|
145
|
+
expect(hostWorklet).toHaveBeenCalledTimes(1);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it('fires again exactly at the interval boundary', () => {
|
|
149
|
+
const hostWorklet = jest.fn();
|
|
150
|
+
useThrottledFrameProcessor(hostWorklet, { sampleHz: 2 }, []); // 500ms
|
|
151
|
+
const [wrappedBody] = useFrameProcessorMock.mock.calls[0]!;
|
|
152
|
+
|
|
153
|
+
wrappedBody({ timestamp: 1000 } as never);
|
|
154
|
+
wrappedBody({ timestamp: 1500 } as never); // exactly at boundary
|
|
155
|
+
|
|
156
|
+
expect(hostWorklet).toHaveBeenCalledTimes(2);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('fires again past the interval boundary', () => {
|
|
160
|
+
const hostWorklet = jest.fn();
|
|
161
|
+
useThrottledFrameProcessor(hostWorklet, { sampleHz: 2 }, []); // 500ms
|
|
162
|
+
const [wrappedBody] = useFrameProcessorMock.mock.calls[0]!;
|
|
163
|
+
|
|
164
|
+
wrappedBody({ timestamp: 1000 } as never);
|
|
165
|
+
wrappedBody({ timestamp: 1600 } as never); // 600ms after
|
|
166
|
+
|
|
167
|
+
expect(hostWorklet).toHaveBeenCalledTimes(2);
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
describe('shared value lifecycle', () => {
|
|
172
|
+
it('initializes lastSampleMs to 0', () => {
|
|
173
|
+
const noop = (() => {}) as unknown as Parameters<typeof useThrottledFrameProcessor>[0];
|
|
174
|
+
useThrottledFrameProcessor(noop, { sampleHz: 2 }, []);
|
|
175
|
+
expect(useSharedValueMock).toHaveBeenCalledWith(0);
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
});
|