react-native-image-stitcher 0.14.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +160 -0
- package/README.md +35 -0
- package/RNImageStitcher.podspec +8 -1
- package/android/build.gradle +0 -16
- package/android/src/main/cpp/CMakeLists.txt +2 -63
- package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
- package/cpp/keyframe_gate.cpp +82 -23
- package/cpp/keyframe_gate.hpp +31 -2
- package/cpp/stitcher.cpp +208 -28
- package/cpp/tests/CMakeLists.txt +18 -12
- package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
- package/cpp/tests/warp_guard_test.cpp +48 -0
- package/cpp/warp_guard.hpp +41 -0
- package/dist/ar/useARSession.d.ts +9 -0
- package/dist/ar/useARSession.js +24 -2
- package/dist/camera/Camera.d.ts +31 -16
- package/dist/camera/Camera.js +27 -4
- package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
- package/dist/camera/CaptureStitchStatsToast.js +27 -7
- package/dist/camera/PanoramaSettings.d.ts +10 -223
- package/dist/camera/PanoramaSettings.js +6 -28
- package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
- package/dist/camera/PanoramaSettingsBridge.js +3 -102
- package/dist/camera/PanoramaSettingsModal.js +7 -1
- package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
- package/dist/camera/buildPanoramaInitialSettings.js +4 -0
- package/dist/camera/cameraErrorMessages.d.ts +32 -0
- package/dist/camera/cameraErrorMessages.js +53 -0
- package/dist/camera/selectCaptureDevice.d.ts +5 -1
- package/dist/camera/selectCaptureDevice.js +22 -2
- package/dist/camera/useCapture.js +38 -0
- package/dist/index.d.ts +5 -8
- package/dist/index.js +11 -34
- package/dist/stitching/incremental.d.ts +1 -117
- package/dist/stitching/stitchVideo.d.ts +0 -35
- package/dist/types.d.ts +0 -87
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
- package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
- package/package.json +3 -2
- package/src/ar/useARSession.ts +35 -5
- package/src/camera/Camera.tsx +63 -24
- package/src/camera/CaptureStitchStatsToast.tsx +58 -14
- package/src/camera/PanoramaSettings.ts +16 -289
- package/src/camera/PanoramaSettingsBridge.ts +3 -114
- package/src/camera/PanoramaSettingsModal.tsx +14 -1
- package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
- package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
- package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
- package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
- package/src/camera/buildPanoramaInitialSettings.ts +17 -0
- package/src/camera/cameraErrorMessages.ts +84 -0
- package/src/camera/selectCaptureDevice.ts +28 -3
- package/src/camera/useCapture.ts +44 -1
- package/src/index.ts +11 -40
- package/src/stitching/incremental.ts +3 -140
- package/src/stitching/stitchVideo.ts +0 -26
- package/src/types.ts +0 -95
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
- package/cpp/stitcher_frame_jsi.cpp +0 -214
- package/cpp/stitcher_frame_jsi.hpp +0 -108
- package/cpp/stitcher_proxy_jsi.cpp +0 -109
- package/cpp/stitcher_proxy_jsi.hpp +0 -46
- package/cpp/stitcher_worklet_dispatch.cpp +0 -103
- package/cpp/stitcher_worklet_dispatch.hpp +0 -71
- package/cpp/stitcher_worklet_registry.cpp +0 -91
- package/cpp/stitcher_worklet_registry.hpp +0 -146
- package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
- package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
- package/dist/stitching/IncrementalStitcherView.js +0 -157
- package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
- package/dist/stitching/StitcherWorkletRegistry.js +0 -78
- package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
- package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
- package/dist/stitching/useFrameProcessor.d.ts +0 -119
- package/dist/stitching/useFrameProcessor.js +0 -196
- package/dist/stitching/useFrameStream.d.ts +0 -34
- package/dist/stitching/useFrameStream.js +0 -234
- package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
- package/dist/stitching/useThrottledFrameProcessor.js +0 -132
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
- package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
- package/src/stitching/IncrementalStitcherView.tsx +0 -198
- package/src/stitching/StitcherWorkletRegistry.ts +0 -156
- package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
- package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
- package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
- package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
- package/src/stitching/useFrameProcessor.ts +0 -226
- package/src/stitching/useFrameStream.ts +0 -271
- package/src/stitching/useThrottledFrameProcessor.ts +0 -145
package/src/index.ts
CHANGED
|
@@ -39,6 +39,12 @@ export type {
|
|
|
39
39
|
FramesDroppedInfo,
|
|
40
40
|
} from './camera/Camera';
|
|
41
41
|
|
|
42
|
+
// Recoverable-stitch-failure → friendly Alert copy. Hosts call this in
|
|
43
|
+
// their onError handler to surface actionable guidance ("pan more slowly",
|
|
44
|
+
// "pivot in place") instead of the raw cv::Stitcher diagnostic.
|
|
45
|
+
export { userFacingStitchError } from './camera/cameraErrorMessages';
|
|
46
|
+
export type { UserFacingStitchError } from './camera/cameraErrorMessages';
|
|
47
|
+
|
|
42
48
|
// ─────────────────────────────────────────────────────────────────────
|
|
43
49
|
// AR foundation (public since 0.1.0)
|
|
44
50
|
// ─────────────────────────────────────────────────────────────────────
|
|
@@ -128,8 +134,6 @@ export type { PanoramaSettingsModalProps } from './camera/PanoramaSettingsModal'
|
|
|
128
134
|
export {
|
|
129
135
|
DEFAULT_PANORAMA_SETTINGS,
|
|
130
136
|
DEFAULT_FLOW_GATE_SETTINGS,
|
|
131
|
-
DEFAULT_SLITSCAN_SETTINGS,
|
|
132
|
-
DEFAULT_HYBRID_SETTINGS,
|
|
133
137
|
} from './camera/PanoramaSettings';
|
|
134
138
|
export type {
|
|
135
139
|
CaptureBaseSettings,
|
|
@@ -137,14 +141,6 @@ export type {
|
|
|
137
141
|
BatchStitcherSettings,
|
|
138
142
|
FrameSelectionSettings,
|
|
139
143
|
FlowGateSettings,
|
|
140
|
-
SlitscanSettings,
|
|
141
|
-
SlitscanPaintingSettings,
|
|
142
|
-
SlitscanRegistrationSettings,
|
|
143
|
-
SlitscanAdvancedSettings,
|
|
144
|
-
Ncc1dSettings,
|
|
145
|
-
Ncc2dSettings,
|
|
146
|
-
PlaneProjectionSettings,
|
|
147
|
-
HybridSettings,
|
|
148
144
|
} from './camera/PanoramaSettings';
|
|
149
145
|
|
|
150
146
|
// Settings → native config adapters. Layer 2 hosts building their
|
|
@@ -153,8 +149,6 @@ export type {
|
|
|
153
149
|
// the single source of truth for the JS↔native wire format.
|
|
154
150
|
export {
|
|
155
151
|
panoramaSettingsToNativeConfig,
|
|
156
|
-
slitscanSettingsToNativeConfig,
|
|
157
|
-
hybridSettingsToNativeConfig,
|
|
158
152
|
} from './camera/PanoramaSettingsBridge';
|
|
159
153
|
export type { NativeConfigDict } from './camera/PanoramaSettingsBridge';
|
|
160
154
|
export { ViewportCropOverlay } from './camera/ViewportCropOverlay';
|
|
@@ -206,34 +200,11 @@ export type {
|
|
|
206
200
|
StitcherFrameProcessor,
|
|
207
201
|
ARAnchor,
|
|
208
202
|
} from './stitching/StitcherFrame';
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
//
|
|
212
|
-
//
|
|
213
|
-
//
|
|
214
|
-
// API-stable but registration-only until Phase 4b lands the
|
|
215
|
-
// cross-runtime handoff (the AR runtime iterating the registry).
|
|
216
|
-
// See the hook's docstring + StitcherFrame.ts for the contract.
|
|
217
|
-
export { useFrameProcessor } from './stitching/useFrameProcessor';
|
|
218
|
-
// v0.9.0 Layer 2 — `useThrottledFrameProcessor`. Throttle gate over
|
|
219
|
-
// `useFrameProcessor` for sub-frame-rate worklet-native processing
|
|
220
|
-
// (native OCR via Vision.framework / ML Kit, TFLite ML detection,
|
|
221
|
-
// LiDAR depth). The worklet runtime has direct access to
|
|
222
|
-
// `frame.toArrayBuffer()` / `frame.arDepth`; bridge small payloads
|
|
223
|
-
// (bboxes, depth-derived metrics) to JS via `runOnJS`. For JS-thread
|
|
224
|
-
// JPEG consumers (file-path OCR libs, cloud upload, thumbnail UI),
|
|
225
|
-
// prefer `useFrameStream` (Layer 3, ships in the same release).
|
|
226
|
-
export { useThrottledFrameProcessor } from './stitching/useThrottledFrameProcessor';
|
|
227
|
-
export type { ThrottledFrameProcessorOptions } from './types';
|
|
228
|
-
// v0.9.0 Layer 3 — `useFrameStream`. JS-thread sampled-frame
|
|
229
|
-
// stream over Layer 1 (`save_frame_as_jpeg` vc plugin) + Layer 2
|
|
230
|
-
// (`useThrottledFrameProcessor`). Use for JS-thread consumers:
|
|
231
|
-
// file-path OCR libs (RN modules), cloud upload, thumbnail UI.
|
|
232
|
-
// For worklet-native processing (Vision/ML Kit as vc plugins,
|
|
233
|
-
// TFLite ML, LiDAR depth), prefer `useThrottledFrameProcessor`
|
|
234
|
-
// (Layer 2) — lower latency, no JPEG roundtrip.
|
|
235
|
-
export { useFrameStream } from './stitching/useFrameStream';
|
|
236
|
-
export type { FrameStreamOptions, SampledFrame } from './types';
|
|
203
|
+
// NOTE: the host-worklet / frame-stream hooks `useFrameProcessor`,
|
|
204
|
+
// `useThrottledFrameProcessor` and `useFrameStream` (v0.8–v0.9) were
|
|
205
|
+
// archived in the batch-keyframe cleanup — they drove the third-party
|
|
206
|
+
// `__stitcherProxy` observer API, not batch-keyframe capture. Source is
|
|
207
|
+
// preserved under archive/src/stitching/ to build on later.
|
|
237
208
|
// vision-camera Frame Processor driver for non-AR captures. As
|
|
238
209
|
// of v0.6 the only non-AR driver exported (the legacy
|
|
239
210
|
// `useIncrementalJSDriver` was removed; was deprecated in v0.5).
|
|
@@ -422,9 +422,9 @@ export interface IncrementalStartOptions {
|
|
|
422
422
|
* 'slitscan' fall back to 'slitscan-both' with a deprecation warning
|
|
423
423
|
* in the native log.
|
|
424
424
|
*/
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
425
|
+
// Only 'batch-keyframe' remains; the live engines were archived in the
|
|
426
|
+
// batch-keyframe cleanup (see archive/).
|
|
427
|
+
engine?: 'batch-keyframe';
|
|
428
428
|
/**
|
|
429
429
|
* V15 — per-stage correction config overrides. Mode-driven defaults
|
|
430
430
|
* are applied first (see RLISStitcherConfig +configForMode:); fields
|
|
@@ -441,143 +441,6 @@ export interface IncrementalStartOptions {
|
|
|
441
441
|
* fields optional (omit to accept the engine-mode default).
|
|
442
442
|
*/
|
|
443
443
|
export interface StitcherConfig {
|
|
444
|
-
// Slit shaping (slit-scan engine only)
|
|
445
|
-
/** Fraction of pan-axis the rectilinear slit retains per frame.
|
|
446
|
-
* Range 0.10 – 0.70, default 0.30 in V15 slit-scan modes. */
|
|
447
|
-
kPanAxisFractionRect: number;
|
|
448
|
-
/** Minimum pan-axis advance (px) before a frame is accepted.
|
|
449
|
-
* 0 = accept on every consumeFrame (Apple-dense slit-scan, V15
|
|
450
|
-
* default). 50 = V13.0g default. */
|
|
451
|
-
kMinAcceptDeltaPx: number;
|
|
452
|
-
|
|
453
|
-
// Per-stage correction toggles
|
|
454
|
-
/** V13.0e+ ORB triangulation + median-Z parallax correction. */
|
|
455
|
-
enableTriangulation: boolean;
|
|
456
|
-
/** V13.0g per-accept incremental Δt accumulator on top of triangulation. */
|
|
457
|
-
enableTriAccumulator: boolean;
|
|
458
|
-
/** V15 1D NCC perpendicular-axis wobble correction (slitscan-rotate
|
|
459
|
-
* default). Independent of the other correction stages. */
|
|
460
|
-
enable1dNcc: boolean;
|
|
461
|
-
/** 1D NCC search radius in pixels (5 – 60). */
|
|
462
|
-
nccSearchRadius1d: number;
|
|
463
|
-
/** V13.0g 2D NCC fine-alignment after triangulation. */
|
|
464
|
-
enable2dNcc: boolean;
|
|
465
|
-
/** V14.0a RANSAC homography per slit + cv::warpPerspective. When
|
|
466
|
-
* enabled and successful, supersedes the rectangular paste path. */
|
|
467
|
-
enableRansacHomography: boolean;
|
|
468
|
-
|
|
469
|
-
// Paint mode (slit-scan engine only)
|
|
470
|
-
/** 'FirstPaintedWins' protects already-painted pixels (V13.0e+
|
|
471
|
-
* default). 'FeatherBlend' alpha-blends new content into already-
|
|
472
|
-
* painted overlap pixels (V13.0d-style; V15 slitscan-both default). */
|
|
473
|
-
paintMode: 'FirstPaintedWins' | 'FeatherBlend';
|
|
474
|
-
|
|
475
|
-
// Hybrid engine
|
|
476
|
-
/** 'Cylindrical' (V12.x – V14.0a behaviour) or 'Planar' (V15 default;
|
|
477
|
-
* cv::detail::PlaneWarper). Planar is well-behaved for pans <60°. */
|
|
478
|
-
hybridProjection: 'Cylindrical' | 'Planar';
|
|
479
|
-
|
|
480
|
-
/** V15.0c — where on the camera frame the per-accept sliver is taken.
|
|
481
|
-
* 'Center' (V13.x default), 'Bottom' (leading edge for top-to-bottom
|
|
482
|
-
* pan), or 'Top' (leading edge for bottom-to-top pan). */
|
|
483
|
-
sliverPosition: 'Center' | 'Bottom' | 'Top';
|
|
484
|
-
|
|
485
|
-
/** V15.0c — when true, the FIRST accepted frame paints the entire
|
|
486
|
-
* camera frame at canvas (0, 0); subsequent frames still use the
|
|
487
|
-
* configured sliver clip. Default false; set true when sliverPosition
|
|
488
|
-
* is Bottom/Top so the canvas is anchored with full-frame content. */
|
|
489
|
-
firstFrameFullFrame: boolean;
|
|
490
|
-
|
|
491
|
-
/** **DEPRECATED in V15.0d** — use `planeSource` instead.
|
|
492
|
-
*
|
|
493
|
-
* V15.0b boolean toggle for the plane-projected stitch path.
|
|
494
|
-
* Kept for backward compat: when `planeSource` is left at its
|
|
495
|
-
* default (Disabled), `useDetectedPlane = true` upgrades it to
|
|
496
|
-
* ARKitDetected. New callers should set `planeSource` directly. */
|
|
497
|
-
useDetectedPlane: boolean;
|
|
498
|
-
|
|
499
|
-
/** V15.0d — source of the plane used by the V15.0b plane-projected
|
|
500
|
-
* stitch path.
|
|
501
|
-
*
|
|
502
|
-
* - 'Disabled' (default): no plane projection; slit-scan path runs.
|
|
503
|
-
* - 'ARKitDetected': use ARKit's first vertical plane that aligns
|
|
504
|
-
* with the camera's view direction (filter threshold:
|
|
505
|
-
* `arkitPlaneAlignmentThreshold`). Falls back to slit-scan
|
|
506
|
-
* silently when no aligned plane is found.
|
|
507
|
-
* - 'Virtual': synthesize a plane at first frame: origin =
|
|
508
|
-
* camera_pos + `virtualPlaneDepthMeters` × camera_forward;
|
|
509
|
-
* normal = -camera_forward. Always works; no ARKit dependency.
|
|
510
|
-
*
|
|
511
|
-
* Field testing showed ARKit plane detection often picks the WRONG
|
|
512
|
-
* surface (side wall, doorframe) — Virtual mode is the safer
|
|
513
|
-
* default for arbitrary scenes. ARKitDetected wins when ARKit
|
|
514
|
-
* finds the correct fixture face. */
|
|
515
|
-
planeSource: 'Disabled' | 'ARKitDetected' | 'Virtual';
|
|
516
|
-
|
|
517
|
-
/** V15.0d — depth (metres) at which the synthetic plane is placed
|
|
518
|
-
* in front of the camera when `planeSource = Virtual`. Set to
|
|
519
|
-
* the user's typical scan distance. Range 0.3 – 5.0 m. Default
|
|
520
|
-
* 1.5 m. */
|
|
521
|
-
virtualPlaneDepthMeters: number;
|
|
522
|
-
|
|
523
|
-
/** V15.0d — minimum dot product between an ARKit-detected plane's
|
|
524
|
-
* surface normal and the camera's facing direction for the plane
|
|
525
|
-
* to be accepted (when `planeSource = ARKitDetected`). 1.0 =
|
|
526
|
-
* plane perfectly facing camera; 0.0 = plane edge-on; negative
|
|
527
|
-
* = facing away. Range 0.0 – 1.0. Default 0.6 (≈53° max angle
|
|
528
|
-
* off-camera). */
|
|
529
|
-
arkitPlaneAlignmentThreshold: number;
|
|
530
|
-
|
|
531
|
-
/** V15.0g — how the plane-projection helper renders each frame onto
|
|
532
|
-
* the canvas. Affects ARKitDetected and Virtual modes; ignored
|
|
533
|
-
* when planeSource = Disabled.
|
|
534
|
-
*
|
|
535
|
-
* - 'Trapezoidal' (V15.0b legacy): geometrically-correct 3D
|
|
536
|
-
* raycast. Each camera pixel maps to its plane intersection.
|
|
537
|
-
* Result is a trapezoid that grows distorted with tilt
|
|
538
|
-
* (cooler-bottom-2.3×-wider-than-top problem).
|
|
539
|
-
* - 'Rectified' (V15.0g default): camera frame pasted as a clean
|
|
540
|
-
* rectangle around its plane-projected anchor. Eliminates the
|
|
541
|
-
* tilt-induced trapezoidal distortion at the cost of strict 3D-
|
|
542
|
-
* correctness — the camera's per-pixel perspective stays inside
|
|
543
|
-
* the rectangle but doesn't reconcile across tilts. */
|
|
544
|
-
planeProjectionStyle: 'Trapezoidal' | 'Rectified';
|
|
545
|
-
|
|
546
|
-
/** V15.0d — 2D NCC search half-window in pixels. Was hardcoded
|
|
547
|
-
* ±12 in V15.0c.4. Smaller = less wandering on repetitive
|
|
548
|
-
* textures (peg holes, slatted panels), but easier to miss the
|
|
549
|
-
* true overlap when pose noise is high. Range 4 – 30. Default
|
|
550
|
-
* 12. */
|
|
551
|
-
nccSearchMargin2d: number;
|
|
552
|
-
|
|
553
|
-
/** V15.0d — 2D NCC confidence threshold below which the correction
|
|
554
|
-
* is rejected. Was hardcoded 0.75 in V15.0c.4. Higher = stricter,
|
|
555
|
-
* fewer false matches on repetitive textures, but more frames
|
|
556
|
-
* where NCC silently doesn't fire. Range 0.30 – 0.99. Default
|
|
557
|
-
* 0.75. */
|
|
558
|
-
nccConfidenceThreshold2d: number;
|
|
559
|
-
|
|
560
|
-
/** V15.0d (1B) — exponential-moving-average smoothing on 2D NCC
|
|
561
|
-
* corrections. When enabled, the applied correction is
|
|
562
|
-
* `α × current + (1−α) × prev` instead of just `current`. Damps
|
|
563
|
-
* single-frame snaps to spurious peaks. Default false. */
|
|
564
|
-
enableNcc2dEmaSmoothing: boolean;
|
|
565
|
-
|
|
566
|
-
/** V15.0d — EMA weight on the CURRENT-frame NCC correction
|
|
567
|
-
* (1 − α weight on the previous correction). Range 0.05 – 0.95.
|
|
568
|
-
* Default 0.4 (60% prev / 40% current — heavy damping). */
|
|
569
|
-
ncc2dEmaAlpha: number;
|
|
570
|
-
|
|
571
|
-
/** V15.0d (1C) — pan-axis-aware 2D NCC. When enabled, the cross-
|
|
572
|
-
* axis (perpendicular to pan) NCC correction is clamped tighter
|
|
573
|
-
* than the pan-axis (since 1D NCC + pose already handle cross-
|
|
574
|
-
* axis wobble). Default false. */
|
|
575
|
-
enableNcc2dPanAxisLock: boolean;
|
|
576
|
-
|
|
577
|
-
/** V15.0d — cross-axis clamp (pixels) for the pan-axis-aware mode.
|
|
578
|
-
* Range 0 – 30. Default 5. */
|
|
579
|
-
ncc2dCrossAxisLockPx: number;
|
|
580
|
-
|
|
581
444
|
// Frame selection (V16)
|
|
582
445
|
|
|
583
446
|
/** V16 — how the engine decides which ARFrames to ingest.
|
|
@@ -76,32 +76,6 @@ export interface StitchVideoOptions {
|
|
|
76
76
|
* Right choice on low-RAM devices or with `feather`.
|
|
77
77
|
*/
|
|
78
78
|
seamFinderType?: 'graphcut' | 'skip';
|
|
79
|
-
/**
|
|
80
|
-
* Phase 5: pose-driven stitching. When present and non-empty,
|
|
81
|
-
* the native stitcher skips features → matching → BundleAdjuster
|
|
82
|
-
* and builds cv::detail::CameraParams directly from each pose's
|
|
83
|
-
* intrinsics + quaternion. Each entry has the shape returned
|
|
84
|
-
* by `NativeModules.RNSARSession.snapshotPoseLog()`:
|
|
85
|
-
*
|
|
86
|
-
* { tx, ty, tz, qx, qy, qz, qw,
|
|
87
|
-
* fx, fy, cx, cy,
|
|
88
|
-
* imageWidth, imageHeight,
|
|
89
|
-
* timestampMs, trackingState }
|
|
90
|
-
*
|
|
91
|
-
* Frames whose closest pose is beyond a 100 ms tolerance are
|
|
92
|
-
* dropped before stitching; if fewer than 2 remain the call
|
|
93
|
-
* rejects with `opencv-failed-1032` so the host can fall back
|
|
94
|
-
* to the feature-matched path (re-call `stitchVideo` without
|
|
95
|
-
* `poses`).
|
|
96
|
-
*/
|
|
97
|
-
poses?: Array<{
|
|
98
|
-
tx: number; ty: number; tz: number;
|
|
99
|
-
qx: number; qy: number; qz: number; qw: number;
|
|
100
|
-
fx: number; fy: number; cx: number; cy: number;
|
|
101
|
-
imageWidth: number; imageHeight: number;
|
|
102
|
-
timestampMs: number;
|
|
103
|
-
trackingState: number;
|
|
104
|
-
}>;
|
|
105
79
|
}
|
|
106
80
|
|
|
107
81
|
|
package/src/types.ts
CHANGED
|
@@ -56,101 +56,6 @@ export interface DeviceMetadata {
|
|
|
56
56
|
// `Camera.tsx` adapts this into the public `CameraCaptureResult` (a
|
|
57
57
|
// discriminated union of photo + panorama) before emitting `onCapture`.
|
|
58
58
|
|
|
59
|
-
/**
|
|
60
|
-
* v0.9.0 Layer 3 — one sampled frame delivered by `useFrameStream`
|
|
61
|
-
* to the JS-thread handler.
|
|
62
|
-
*
|
|
63
|
-
* The JPEG file at `jpegPath` is the stream's own copy. Hosts that
|
|
64
|
-
* need long-term retention MUST copy the file synchronously inside
|
|
65
|
-
* the handler — the same path may be overwritten by a subsequent
|
|
66
|
-
* sample (slot reuse — see the hook's docstring for the rotation
|
|
67
|
-
* policy).
|
|
68
|
-
*/
|
|
69
|
-
export interface SampledFrame {
|
|
70
|
-
/** Absolute filesystem path to the JPEG. No `file://` prefix. */
|
|
71
|
-
jpegPath: string;
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* Pose at sample time. `translation` is `undefined` in non-AR
|
|
75
|
-
* mode (gyro provides rotation only; no spatial anchor).
|
|
76
|
-
*/
|
|
77
|
-
pose: {
|
|
78
|
-
rotation: [number, number, number, number];
|
|
79
|
-
translation?: [number, number, number];
|
|
80
|
-
};
|
|
81
|
-
|
|
82
|
-
/** Frame timestamp (ms; per the v0.8.0 StitcherFrame contract). */
|
|
83
|
-
timestamp: number;
|
|
84
|
-
|
|
85
|
-
/** JPEG width / height in pixels. */
|
|
86
|
-
width: number;
|
|
87
|
-
height: number;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* v0.9.0 Layer 3 — options for `useFrameStream`.
|
|
92
|
-
*
|
|
93
|
-
* For worklet-native processing without JPEG roundtrip (OCR via
|
|
94
|
-
* Vision/ML Kit, TFLite ML, LiDAR depth), use
|
|
95
|
-
* `useThrottledFrameProcessor` (Layer 2) instead.
|
|
96
|
-
*/
|
|
97
|
-
export interface FrameStreamOptions {
|
|
98
|
-
/**
|
|
99
|
-
* Target sampling rate in Hertz. Clamped to `[0.5, 10]`. The
|
|
100
|
-
* Layer 2 throttle gate enforces the rate inside the worklet;
|
|
101
|
-
* ticks too close together are dropped silently.
|
|
102
|
-
*
|
|
103
|
-
* Clamp upper bound (10 Hz) is intentionally lower than Layer 2's
|
|
104
|
-
* (30 Hz) — beyond 10 Hz the per-frame JPEG encode + JS-bridge
|
|
105
|
-
* cost dominates the wall-clock budget. Hosts that need higher
|
|
106
|
-
* rates should be on Layer 2 with their own JPEG encoder call
|
|
107
|
-
* (or no JPEG at all).
|
|
108
|
-
*/
|
|
109
|
-
sampleHz: number;
|
|
110
|
-
|
|
111
|
-
/**
|
|
112
|
-
* JPEG quality (0-100). Default 75. Clamped silently to
|
|
113
|
-
* `[1, 100]` by the underlying `save_frame_as_jpeg` native plugin.
|
|
114
|
-
*/
|
|
115
|
-
quality?: number;
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* Directory to write JPEG files into. Defaults to a per-app
|
|
119
|
-
* `<cache>/rnis-frame-stream/` subdirectory. The directory is
|
|
120
|
-
* `mkdir -p`'d on first use; hosts that supply an existing
|
|
121
|
-
* absolute path are responsible for its lifecycle.
|
|
122
|
-
*/
|
|
123
|
-
outputDir?: string;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
/**
|
|
127
|
-
* v0.9.0 Layer 2 — options for `useThrottledFrameProcessor`.
|
|
128
|
-
*
|
|
129
|
-
* Wraps v0.8.0's `useFrameProcessor` with a monotonic-time throttle
|
|
130
|
-
* gate so the supplied worklet fires at most `sampleHz` times per
|
|
131
|
-
* second. Use for sub-frame-rate worklet-native processing — native
|
|
132
|
-
* OCR (Vision.framework / ML Kit), TFLite ML detection, LiDAR depth
|
|
133
|
-
* processing — where the bbox / depth payloads are small enough to
|
|
134
|
-
* bridge to JS via `runOnJS`.
|
|
135
|
-
*
|
|
136
|
-
* For JS-thread JPEG consumers (file-path OCR libraries, cloud
|
|
137
|
-
* upload, thumbnail UI), use `useFrameStream` (Layer 3) instead.
|
|
138
|
-
*/
|
|
139
|
-
export interface ThrottledFrameProcessorOptions {
|
|
140
|
-
/**
|
|
141
|
-
* Target sampling rate in Hertz. Clamped to `[0.5, 30]`. Inside
|
|
142
|
-
* the worklet a monotonic-time gate enforces the rate; ticks too
|
|
143
|
-
* close together are silently dropped.
|
|
144
|
-
*
|
|
145
|
-
* The clamp upper bound (30 Hz) sits at typical AR rates on
|
|
146
|
-
* mid-range Android devices — beyond that, the host should just
|
|
147
|
-
* use `useFrameProcessor` directly (no throttle). The clamp
|
|
148
|
-
* lower bound (0.5 Hz) prevents accidentally-zero-divide values
|
|
149
|
-
* + matches `useFrameStream`'s convention.
|
|
150
|
-
*/
|
|
151
|
-
sampleHz: number;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
59
|
export interface CaptureResult {
|
|
155
60
|
/** Unique device-generated UUID */
|
|
156
61
|
deviceUuid: string;
|
|
@@ -1,227 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
-
//
|
|
3
|
-
// stitcher_jsi_install_jni.cpp — JNI binding for the Android-side
|
|
4
|
-
// JSI install (v0.8.0 Phase 4b.ii).
|
|
5
|
-
//
|
|
6
|
-
// Kotlin's `StitcherJsiInstallerModule.nativeInstall(jsiRuntimeRef)`
|
|
7
|
-
// calls into this file. We unbox the `jsi::Runtime*` from the
|
|
8
|
-
// Java `long` and hand it to the shared
|
|
9
|
-
// `retailens::installStitcherProxy(runtime)` function which sets
|
|
10
|
-
// `globalThis.__stitcherProxy`. Same destination as iOS — the
|
|
11
|
-
// host object class lives in `cpp/stitcher_proxy_jsi.{hpp,cpp}`.
|
|
12
|
-
//
|
|
13
|
-
// ## Why a `long` ref, not a JSI handle wrapper class
|
|
14
|
-
//
|
|
15
|
-
// `ReactApplicationContext.getJavaScriptContextHolder()` returns a
|
|
16
|
-
// `JavaScriptContextHolder` whose `.get()` returns a Java `long`
|
|
17
|
-
// that's the raw pointer to the C++ `jsi::Runtime*`. Same
|
|
18
|
-
// contract as worklets-core's `WorkletsModule.nativeInstall`
|
|
19
|
-
// (verified at the same call site). Caller is responsible for
|
|
20
|
-
// ensuring the runtime outlives this call — in practice, the
|
|
21
|
-
// runtime IS the JS thread's runtime which lives the whole
|
|
22
|
-
// process lifetime, so this is structurally always safe in our
|
|
23
|
-
// usage.
|
|
24
|
-
//
|
|
25
|
-
// ## Threading
|
|
26
|
-
//
|
|
27
|
-
// Kotlin invokes this from a `@ReactMethod(isBlockingSynchronousMethod
|
|
28
|
-
// = true)` so we're already on the JS thread. Synchronous JSI
|
|
29
|
-
// access is safe.
|
|
30
|
-
|
|
31
|
-
#include "stitcher_proxy_jsi.hpp"
|
|
32
|
-
|
|
33
|
-
// v0.8.0 Phase 4b.iii — per-frame fan-out support. The shared
|
|
34
|
-
// `dispatchToHostWorklets` posts to worklets-core's default context;
|
|
35
|
-
// this JNI file's `nativeDispatchToHostWorklets` constructs the
|
|
36
|
-
// `StitcherFrameData` from raw bytes + pose + dims and forwards it.
|
|
37
|
-
#include "stitcher_frame_data.hpp"
|
|
38
|
-
#include "stitcher_worklet_dispatch.hpp"
|
|
39
|
-
#include "stitcher_worklet_registry.hpp"
|
|
40
|
-
|
|
41
|
-
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
|
|
42
|
-
|
|
43
|
-
#include <jni.h>
|
|
44
|
-
#include <jsi/jsi.h>
|
|
45
|
-
|
|
46
|
-
#include <android/log.h>
|
|
47
|
-
|
|
48
|
-
#include <cstdint>
|
|
49
|
-
#include <cstring>
|
|
50
|
-
#include <memory>
|
|
51
|
-
#include <utility>
|
|
52
|
-
#include <vector>
|
|
53
|
-
|
|
54
|
-
#define LOG_TAG "StitcherJsiInstaller"
|
|
55
|
-
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
|
|
56
|
-
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
|
|
57
|
-
|
|
58
|
-
extern "C" JNIEXPORT jboolean JNICALL
|
|
59
|
-
Java_io_imagestitcher_rn_StitcherJsiInstallerModule_nativeInstall(
|
|
60
|
-
JNIEnv* /*env*/, jobject /*thiz*/, jlong jsiRuntimeRef) {
|
|
61
|
-
if (jsiRuntimeRef == 0) {
|
|
62
|
-
// ReactApplicationContext.getJavaScriptContextHolder().get()
|
|
63
|
-
// returns 0 when the runtime isn't ready (rare — JS would have
|
|
64
|
-
// had to call us before its own runtime was up; impossible in
|
|
65
|
-
// practice). Defensive.
|
|
66
|
-
return JNI_FALSE;
|
|
67
|
-
}
|
|
68
|
-
auto* runtime = reinterpret_cast<facebook::jsi::Runtime*>(jsiRuntimeRef);
|
|
69
|
-
retailens::installStitcherProxy(*runtime);
|
|
70
|
-
LOGI("installed globalThis.__stitcherProxy on main JS runtime.");
|
|
71
|
-
return JNI_TRUE;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// ─── v0.8.0 Phase 4b.iii — Android NV21 PixelBufferReader ──────────
|
|
75
|
-
//
|
|
76
|
-
// Owns a heap-allocated `std::vector<uint8_t>` of pre-copied NV21
|
|
77
|
-
// bytes. Constructed by `nativeDispatchToHostWorklets` after one
|
|
78
|
-
// JNI byte-array copy from Kotlin; outlives the AR render thread
|
|
79
|
-
// scope via `StitcherFrameData::pixelReader`'s `shared_ptr` —
|
|
80
|
-
// dropped when the host object is invalidated.
|
|
81
|
-
|
|
82
|
-
namespace {
|
|
83
|
-
|
|
84
|
-
class AndroidNV21BufferReader : public retailens::PixelBufferReader {
|
|
85
|
-
public:
|
|
86
|
-
explicit AndroidNV21BufferReader(std::vector<uint8_t>&& bytes)
|
|
87
|
-
: _bytes(std::move(bytes)) {}
|
|
88
|
-
|
|
89
|
-
std::size_t byteSize() const override { return _bytes.size(); }
|
|
90
|
-
|
|
91
|
-
std::size_t copyTo(uint8_t* dst, std::size_t maxBytes) override {
|
|
92
|
-
if (dst == nullptr) return 0;
|
|
93
|
-
std::size_t n = std::min(maxBytes, _bytes.size());
|
|
94
|
-
if (n > 0) {
|
|
95
|
-
std::memcpy(dst, _bytes.data(), n);
|
|
96
|
-
}
|
|
97
|
-
return n;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
private:
|
|
101
|
-
std::vector<uint8_t> _bytes;
|
|
102
|
-
};
|
|
103
|
-
|
|
104
|
-
} // namespace
|
|
105
|
-
|
|
106
|
-
// ─── v0.8.0 Phase 4b.iii — registry count accessor ─────────────────
|
|
107
|
-
//
|
|
108
|
-
// Cheap (microsecond) accessor for the per-frame gate in
|
|
109
|
-
// `RNSARCameraView.onDrawFrame`. Avoids the NV21 byte-pack cost
|
|
110
|
-
// when no host worklets are registered AND no capture is active.
|
|
111
|
-
// Same atomic-read the JSI host object's `count()` host function
|
|
112
|
-
// goes through.
|
|
113
|
-
extern "C" JNIEXPORT jint JNICALL
|
|
114
|
-
Java_io_imagestitcher_rn_StitcherWorkletRuntime_nativeRegistryCount(
|
|
115
|
-
JNIEnv* /*env*/, jobject /*thiz*/) {
|
|
116
|
-
return static_cast<jint>(
|
|
117
|
-
retailens::StitcherWorkletRegistry::shared().count());
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
// ─── v0.8.0 Phase 4b.iii — per-frame dispatch JNI binding ──────────
|
|
121
|
-
//
|
|
122
|
-
// Called from Kotlin's `StitcherWorkletRuntime.dispatchToHostWorklets`
|
|
123
|
-
// after the first-party stitching block has returned (the AR-frame
|
|
124
|
-
// data is still in scope on the Kotlin side because
|
|
125
|
-
// `RNSARCameraView.onDrawFrame` reads the ARCore Frame, builds the
|
|
126
|
-
// NV21 byte[], invokes first-party via `runFirstParty { ... }`,
|
|
127
|
-
// THEN calls into here).
|
|
128
|
-
//
|
|
129
|
-
// The byte[] is COPIED into our owned vector — ARCore's pixel data
|
|
130
|
-
// becomes inaccessible shortly after `onDrawFrame` returns, and our
|
|
131
|
-
// async dispatch must outlive that scope. Cost: one ~3MB memcpy
|
|
132
|
-
// per frame at 1080p NV21 (~90 MB/s at 30 fps; <5 ms on a mid-range
|
|
133
|
-
// Android device). Fast-path early-exit when the registry is empty
|
|
134
|
-
// skips the copy entirely.
|
|
135
|
-
//
|
|
136
|
-
// trackingState: Kotlin passes one of "" / "notAvailable" / "limited"
|
|
137
|
-
// / "normal" (empty string = field unset → JS sees undefined).
|
|
138
|
-
extern "C" JNIEXPORT void JNICALL
|
|
139
|
-
Java_io_imagestitcher_rn_StitcherWorkletRuntime_nativeDispatchToHostWorklets(
|
|
140
|
-
JNIEnv* env, jobject /*thiz*/,
|
|
141
|
-
jbyteArray nv21Bytes,
|
|
142
|
-
jint width, jint height,
|
|
143
|
-
jdouble qx, jdouble qy, jdouble qz, jdouble qw,
|
|
144
|
-
jdouble tx, jdouble ty, jdouble tz,
|
|
145
|
-
jdouble timestampNs,
|
|
146
|
-
jstring trackingState) {
|
|
147
|
-
// Fast-path early-exit BEFORE the JNI byte-array copy. Saves the
|
|
148
|
-
// ~3MB memcpy + JSI host object alloc on every frame in the
|
|
149
|
-
// common first-party-only case.
|
|
150
|
-
if (retailens::StitcherWorkletRegistry::shared().count() == 0) {
|
|
151
|
-
return;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
if (nv21Bytes == nullptr) {
|
|
155
|
-
LOGE("nativeDispatchToHostWorklets: nv21Bytes is null");
|
|
156
|
-
return;
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
const jsize byteLen = env->GetArrayLength(nv21Bytes);
|
|
160
|
-
if (byteLen <= 0) {
|
|
161
|
-
LOGE("nativeDispatchToHostWorklets: nv21Bytes is empty");
|
|
162
|
-
return;
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// Copy into our owned vector. `GetByteArrayRegion` is the
|
|
166
|
-
// canonical "copy" path — `GetByteArrayElements + Release` MAY
|
|
167
|
-
// pin the JVM array (zero-copy) but the contract isn't
|
|
168
|
-
// guaranteed; we need our own buffer for the async dispatch
|
|
169
|
-
// anyway, so the explicit copy is cleaner.
|
|
170
|
-
std::vector<uint8_t> bytes(static_cast<std::size_t>(byteLen));
|
|
171
|
-
env->GetByteArrayRegion(
|
|
172
|
-
nv21Bytes, 0, byteLen,
|
|
173
|
-
reinterpret_cast<jbyte*>(bytes.data()));
|
|
174
|
-
|
|
175
|
-
// Extract trackingState string (may be null on the Kotlin side
|
|
176
|
-
// for non-AR or pre-tracking frames — guard accordingly).
|
|
177
|
-
std::string trackingStateStr;
|
|
178
|
-
if (trackingState != nullptr) {
|
|
179
|
-
const char* cs = env->GetStringUTFChars(trackingState, nullptr);
|
|
180
|
-
if (cs != nullptr) {
|
|
181
|
-
trackingStateStr = cs;
|
|
182
|
-
env->ReleaseStringUTFChars(trackingState, cs);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
// Build StitcherFrameData. Field semantics match the iOS
|
|
187
|
-
// `StitcherFrameHostObject::fromARFrame:pose:` factory; this is
|
|
188
|
-
// the Android equivalent path.
|
|
189
|
-
retailens::StitcherFrameData data;
|
|
190
|
-
data.source = "ar";
|
|
191
|
-
data.width = static_cast<int32_t>(width);
|
|
192
|
-
data.height = static_cast<int32_t>(height);
|
|
193
|
-
// ARCore's camera image is YUV_420_888 on Android, mapped to NV21
|
|
194
|
-
// by the existing `YuvImageConverter.packNV21` path — the byte[]
|
|
195
|
-
// we receive is interleaved Y then VU. Worklets gate on this
|
|
196
|
-
// string identifier (`'yuv'` vs `'unknown'`); v0.8.0 always
|
|
197
|
-
// emits `'yuv'` for AR mode on Android (NV21).
|
|
198
|
-
data.pixelFormat = "yuv";
|
|
199
|
-
// Android AR-mode camera image is always landscape-natural; the
|
|
200
|
-
// mapping matches iOS' coarse two-value set. Hosts that need
|
|
201
|
-
// exact display orientation read it from the device-orientation
|
|
202
|
-
// sensors (see `useDeviceOrientation` hook).
|
|
203
|
-
data.orientation = (width >= height) ? "landscape-right" : "portrait";
|
|
204
|
-
data.timestampNs = timestampNs;
|
|
205
|
-
data.qx = qx;
|
|
206
|
-
data.qy = qy;
|
|
207
|
-
data.qz = qz;
|
|
208
|
-
data.qw = qw;
|
|
209
|
-
data.tx = tx;
|
|
210
|
-
data.ty = ty;
|
|
211
|
-
data.tz = tz;
|
|
212
|
-
data.hasTranslation = true; // AR mode always has translation
|
|
213
|
-
data.arTrackingState = trackingStateStr;
|
|
214
|
-
data.pixelReader =
|
|
215
|
-
std::make_shared<AndroidNV21BufferReader>(std::move(bytes));
|
|
216
|
-
|
|
217
|
-
// Dispatch on worklets-core's default context. That context is
|
|
218
|
-
// initialised by JS' `Worklets.install()` (which runs at lib
|
|
219
|
-
// bootstrap when worklets-core's module is imported); by the
|
|
220
|
-
// time host worklets are registered, the default context is up.
|
|
221
|
-
// The shared dispatch helper handles the registry snapshot,
|
|
222
|
-
// host-object construction (inside the worklet thread), per-
|
|
223
|
-
// worklet failure isolation, and invalidation.
|
|
224
|
-
retailens::dispatchToHostWorklets(
|
|
225
|
-
RNWorklet::JsiWorkletContext::getDefaultInstance(),
|
|
226
|
-
std::move(data));
|
|
227
|
-
}
|