react-native-image-stitcher 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +199 -1
- package/android/src/main/java/io/imagestitcher/rn/CvFlowGateFrameProcessor.kt +2 -2
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +2 -30
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +90 -368
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +6 -3
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +17 -30
- package/dist/camera/Camera.d.ts +11 -27
- package/dist/camera/Camera.js +46 -78
- package/dist/index.d.ts +2 -3
- package/dist/index.js +10 -6
- package/dist/stitching/incremental.d.ts +79 -11
- package/dist/stitching/useFrameProcessorDriver.d.ts +7 -6
- package/dist/stitching/useFrameProcessorDriver.js +12 -11
- package/dist/stitching/useKeyframeStream.d.ts +69 -0
- package/dist/stitching/useKeyframeStream.js +120 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +48 -208
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +0 -8
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +6 -126
- package/ios/Sources/RNImageStitcher/KeyframeGateFrameProcessor.mm +6 -6
- package/package.json +1 -1
- package/src/camera/Camera.tsx +57 -106
- package/src/index.ts +9 -9
- package/src/stitching/incremental.ts +84 -11
- package/src/stitching/useFrameProcessorDriver.ts +12 -11
- package/src/stitching/useKeyframeStream.ts +127 -0
- package/dist/stitching/useIncrementalJSDriver.d.ts +0 -74
- package/dist/stitching/useIncrementalJSDriver.js +0 -220
- package/src/stitching/useIncrementalJSDriver.ts +0 -297
|
@@ -18,7 +18,6 @@ import com.google.ar.core.exceptions.CameraNotAvailableException
|
|
|
18
18
|
import com.google.ar.core.exceptions.SessionPausedException
|
|
19
19
|
import io.imagestitcher.rn.ar.BackgroundRenderer
|
|
20
20
|
import io.imagestitcher.rn.ar.YuvImageConverter
|
|
21
|
-
import java.io.File
|
|
22
21
|
import java.util.concurrent.atomic.AtomicReference
|
|
23
22
|
import javax.microedition.khronos.egl.EGLConfig
|
|
24
23
|
import javax.microedition.khronos.opengles.GL10
|
|
@@ -76,13 +75,6 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
76
75
|
private var surfaceWidth: Int = 0
|
|
77
76
|
private var surfaceHeight: Int = 0
|
|
78
77
|
|
|
79
|
-
/// Tmp directory for the per-frame JPEG file we hand to the
|
|
80
|
-
/// incremental engine. Created lazily and reused across frames
|
|
81
|
-
/// — no per-frame allocation.
|
|
82
|
-
private val tmpJpegFile: File by lazy {
|
|
83
|
-
File(context.cacheDir, "rlis-arframe.jpg")
|
|
84
|
-
}
|
|
85
|
-
|
|
86
78
|
/// Whether to feed the AR session's frames into the incremental
|
|
87
79
|
/// engine. Toggled by IncrementalStitcher.start/stop
|
|
88
80
|
/// via setIncrementalIngestionActive() below.
|
|
@@ -510,28 +502,18 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
510
502
|
val rotationForEncode = if (lastDisplayRotation >= 0)
|
|
511
503
|
lastDisplayRotation else android.view.Surface.ROTATION_0
|
|
512
504
|
|
|
513
|
-
//
|
|
514
|
-
//
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
//
|
|
518
|
-
//
|
|
519
|
-
//
|
|
505
|
+
// F8.6 (v0.6) — the eager JPEG encode for live-engine mode
|
|
506
|
+
// is gone. Pass the already-packed NV21 directly via
|
|
507
|
+
// `nv21PixelData`; the engine's new `addFramePixelData`
|
|
508
|
+
// path builds the BGR cv::Mat in-process via cvtColor,
|
|
509
|
+
// skipping the JPEG decode round-trip downstream. In
|
|
510
|
+
// batch-keyframe mode the engine ignores `nv21PixelData`
|
|
511
|
+
// (it uses `grayData` + `onAccept` lazily); no behaviour
|
|
512
|
+
// change there.
|
|
520
513
|
//
|
|
521
|
-
//
|
|
522
|
-
//
|
|
523
|
-
//
|
|
524
|
-
// hold time.
|
|
525
|
-
val legacyJpegPath: String? = if (module.isBatchKeyframeMode) {
|
|
526
|
-
null
|
|
527
|
-
} else {
|
|
528
|
-
YuvImageConverter.encodeJpegFromNV21(
|
|
529
|
-
packed,
|
|
530
|
-
tmpJpegFile.absolutePath,
|
|
531
|
-
jpegQuality = 70,
|
|
532
|
-
displayRotation = rotationForEncode,
|
|
533
|
-
)
|
|
534
|
-
}
|
|
514
|
+
// (Was: eager JPEG encode for non-batch-keyframe modes,
|
|
515
|
+
// written to `tmpJpegFile`, passed as `legacyJpegPath`.
|
|
516
|
+
// See the v0.3 / F8.6 entries in CHANGELOG.md.)
|
|
535
517
|
module.ingestFromARCameraView(
|
|
536
518
|
tx = tArr[0].toDouble(),
|
|
537
519
|
ty = tArr[1].toDouble(),
|
|
@@ -553,7 +535,12 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
553
535
|
grayWidth = packed.width,
|
|
554
536
|
grayHeight = packed.height,
|
|
555
537
|
grayStride = packed.width,
|
|
556
|
-
legacyJpegPath =
|
|
538
|
+
legacyJpegPath = null,
|
|
539
|
+
// F8.6 — pixel-data path for live engines. Batch-
|
|
540
|
+
// keyframe mode ignores these (bails earlier).
|
|
541
|
+
nv21PixelData = packed.nv21,
|
|
542
|
+
nv21PixelWidth = packed.width,
|
|
543
|
+
nv21PixelHeight = packed.height,
|
|
557
544
|
onAccept = { targetPath ->
|
|
558
545
|
// Lazy JPEG encode. Runs ONLY if the C++ KeyframeGate
|
|
559
546
|
// accepted the frame. Encodes from the pre-packed
|
package/dist/camera/Camera.d.ts
CHANGED
|
@@ -200,36 +200,20 @@ export interface CameraProps {
|
|
|
200
200
|
* Introduced for F8 (FrameProcessor port) — see
|
|
201
201
|
* `docs/f8-frame-processor-plan.md`.
|
|
202
202
|
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
*
|
|
208
|
-
* one-time console.warn — supplying your own worklet would race
|
|
209
|
-
* with the SDK's pixel-buffer feed.
|
|
203
|
+
* The SDK installs its own frame processor via
|
|
204
|
+
* `useFrameProcessorDriver`. Setting this prop is ignored with
|
|
205
|
+
* a one-time `console.warn` — supplying a host worklet would
|
|
206
|
+
* race with the SDK's pixel-buffer feed. Either remove the prop
|
|
207
|
+
* or fork the SDK if you genuinely need a custom worklet.
|
|
210
208
|
*
|
|
211
|
-
*
|
|
212
|
-
* * Default (modern non-AR): SDK owns the worklet, this prop
|
|
213
|
-
* is ignored.
|
|
214
|
-
* * `legacyDriver={true}`: SDK uses the old `useIncrementalJSDriver`
|
|
215
|
-
* (takeSnapshot path). Honoured for diagnostics or as an
|
|
216
|
-
* escape hatch.
|
|
217
|
-
* * AR mode: vision-camera Camera isn't mounted, this prop is
|
|
218
|
-
* irrelevant.
|
|
219
|
-
*/
|
|
220
|
-
frameProcessor?: ReadonlyFrameProcessor | DrawableFrameProcessor;
|
|
221
|
-
/**
|
|
222
|
-
* Opt back into the legacy `useIncrementalJSDriver` for non-AR
|
|
223
|
-
* captures (the v0.4 path: `takeSnapshot` → JPEG → cache file →
|
|
224
|
-
* `IncrementalStitcher.processFrameAtPath`).
|
|
209
|
+
* AR mode is irrelevant: vision-camera's Camera isn't mounted.
|
|
225
210
|
*
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
* specific issue with the new driver and need to ship a fix.
|
|
211
|
+
* (v0.5 had a `legacyDriver` escape hatch that routed back to
|
|
212
|
+
* `useIncrementalJSDriver`. That hook + prop were removed in
|
|
213
|
+
* v0.6 per the deprecation timeline announced in the v0.5.0
|
|
214
|
+
* CHANGELOG.)
|
|
231
215
|
*/
|
|
232
|
-
|
|
216
|
+
frameProcessor?: ReadonlyFrameProcessor | DrawableFrameProcessor;
|
|
233
217
|
}
|
|
234
218
|
/**
|
|
235
219
|
* The public `<Camera>` component.
|
package/dist/camera/Camera.js
CHANGED
|
@@ -97,7 +97,6 @@ const lowMemDevice_1 = require("./lowMemDevice");
|
|
|
97
97
|
const useCapture_1 = require("./useCapture");
|
|
98
98
|
const useDeviceOrientation_1 = require("./useDeviceOrientation");
|
|
99
99
|
const incremental_1 = require("../stitching/incremental");
|
|
100
|
-
const useIncrementalJSDriver_1 = require("../stitching/useIncrementalJSDriver");
|
|
101
100
|
const useFrameProcessorDriver_1 = require("../stitching/useFrameProcessorDriver");
|
|
102
101
|
const useIncrementalStitcher_1 = require("../stitching/useIncrementalStitcher");
|
|
103
102
|
const useIMUTranslationGate_1 = require("../sensors/useIMUTranslationGate");
|
|
@@ -271,7 +270,7 @@ function extractPanoramaOverrides(props) {
|
|
|
271
270
|
* The public `<Camera>` component.
|
|
272
271
|
*/
|
|
273
272
|
function Camera(props) {
|
|
274
|
-
const { defaultCaptureSource = 'ar', defaultLens = '1x', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, frameProcessor: hostFrameProcessor,
|
|
273
|
+
const { defaultCaptureSource = 'ar', defaultLens = '1x', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, frameProcessor: hostFrameProcessor, engine = 'batch-keyframe', } = props;
|
|
275
274
|
const insets = (0, react_native_safe_area_context_1.useSafeAreaInsets)();
|
|
276
275
|
// ── State ───────────────────────────────────────────────────────
|
|
277
276
|
const [arPreference, setArPreference] = (0, react_1.useState)(defaultCaptureSource === 'ar');
|
|
@@ -417,58 +416,44 @@ function Camera(props) {
|
|
|
417
416
|
mod?.markNextFrameAsLastKeyframe?.().catch(() => undefined);
|
|
418
417
|
},
|
|
419
418
|
});
|
|
420
|
-
//
|
|
421
|
-
// engine consumes frames from the ARSession stream
|
|
422
|
-
// hook stays idle.
|
|
419
|
+
// Frame Processor driver for non-AR captures (iOS + Android).
|
|
420
|
+
// In AR mode the engine consumes frames from the ARSession stream
|
|
421
|
+
// natively, so this hook stays idle.
|
|
423
422
|
//
|
|
424
423
|
// IMPORTANT: start()/stop() are called imperatively from the hold
|
|
425
424
|
// handlers below — NOT from a useEffect driven by statusPhase. The
|
|
426
425
|
// hook returns a fresh object identity on every render, and during
|
|
427
426
|
// a recording the engine emits IncrementalStateUpdate events that
|
|
428
|
-
// cause re-renders multiple times per second. An effect with
|
|
429
|
-
//
|
|
430
|
-
//
|
|
431
|
-
//
|
|
432
|
-
//
|
|
433
|
-
//
|
|
434
|
-
//
|
|
435
|
-
// imperative pattern (start on hold-start, stop on hold-end) avoids
|
|
436
|
-
// the re-render churn entirely.
|
|
437
|
-
const jsDriver = (0, useIncrementalJSDriver_1.useIncrementalJSDriver)();
|
|
438
|
-
// F8.3 — vision-camera Frame Processor variant. Always
|
|
439
|
-
// instantiated so we don't have conditional hook calls; only one
|
|
440
|
-
// of the two drivers actually .start()s per capture. Stop() on
|
|
441
|
-
// an idle driver is a no-op.
|
|
427
|
+
// cause re-renders multiple times per second. An effect with the
|
|
428
|
+
// driver in its deps would teardown + restart on every event,
|
|
429
|
+
// resetting the gyro accumulator (yaw/pitch) to zero each cycle.
|
|
430
|
+
// User-visible symptom: "only the first keyframe is accepted, every
|
|
431
|
+
// subsequent ingest sees pose=(0,0) and is rejected as a duplicate".
|
|
432
|
+
// The imperative pattern (start on hold-start, stop on hold-end)
|
|
433
|
+
// avoids the re-render churn entirely.
|
|
442
434
|
const fpDriver = (0, useFrameProcessorDriver_1.useFrameProcessorDriver)();
|
|
443
|
-
// Safety:
|
|
444
|
-
// mid-recording. Empty deps so this only fires on unmount.
|
|
435
|
+
// Safety: stop the driver if the component unmounts mid-recording.
|
|
445
436
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
446
|
-
(0, react_1.useEffect)(() => () => {
|
|
447
|
-
//
|
|
448
|
-
//
|
|
449
|
-
//
|
|
450
|
-
//
|
|
451
|
-
//
|
|
452
|
-
//
|
|
453
|
-
//
|
|
437
|
+
(0, react_1.useEffect)(() => () => { fpDriver.stop(); }, []);
|
|
438
|
+
// One-shot deprecation warning when the host supplies their own
|
|
439
|
+
// `frameProcessor` prop. Two worklets racing on the same
|
|
440
|
+
// producer thread would corrupt the engine's workQueue ordering,
|
|
441
|
+
// so the SDK's own worklet wins and the host's is silently
|
|
442
|
+
// ignored. (v0.5 had a `legacyDriver` opt-out for hosts that
|
|
443
|
+
// wanted to route around the SDK driver; that was removed in
|
|
444
|
+
// v0.6 along with `useIncrementalJSDriver`.)
|
|
454
445
|
const hostFrameProcessorIgnoredWarnedRef = (0, react_1.useRef)(false);
|
|
455
446
|
if (hostFrameProcessor != null
|
|
456
|
-
&& !legacyDriver
|
|
457
447
|
&& !hostFrameProcessorIgnoredWarnedRef.current) {
|
|
458
448
|
hostFrameProcessorIgnoredWarnedRef.current = true;
|
|
459
449
|
// eslint-disable-next-line no-console
|
|
460
450
|
console.warn('[react-native-image-stitcher] The `frameProcessor` prop on '
|
|
461
|
-
+ '<Camera> is ignored
|
|
462
|
-
+ '
|
|
463
|
-
+ '
|
|
451
|
+
+ '<Camera> is ignored — the SDK installs its own worklet '
|
|
452
|
+
+ 'via useFrameProcessorDriver. Remove the prop, or fork '
|
|
453
|
+
+ 'the SDK if you genuinely need a custom worklet.');
|
|
464
454
|
}
|
|
465
|
-
// The Frame Processor worklet
|
|
466
|
-
|
|
467
|
-
// 1. Legacy mode: honor the host's prop (or null).
|
|
468
|
-
// 2. Modern mode: SDK driver's worklet, regardless of host's prop.
|
|
469
|
-
const effectiveFrameProcessor = legacyDriver
|
|
470
|
-
? (hostFrameProcessor ?? null)
|
|
471
|
-
: fpDriver.frameProcessor;
|
|
455
|
+
// The Frame Processor worklet bound to vision-camera's Camera.
|
|
456
|
+
const effectiveFrameProcessor = fpDriver.frameProcessor;
|
|
472
457
|
// ── Subscribe to engine state for live keyframe thumbs ──────────
|
|
473
458
|
(0, react_1.useEffect)(() => {
|
|
474
459
|
const sub = (0, incremental_1.subscribeIncrementalState)((state) => {
|
|
@@ -524,17 +509,17 @@ function Camera(props) {
|
|
|
524
509
|
const accepted = incrementalState?.acceptedCount ?? 0;
|
|
525
510
|
if (accepted > lastAcceptedCountRef.current) {
|
|
526
511
|
lastAcceptedCountRef.current = accepted;
|
|
527
|
-
// F8.3 review-of-review (M3 revert):
|
|
528
|
-
// `legacyDriver` because the Frame
|
|
529
|
-
// consult `imuGate` for its own pose
|
|
530
|
-
// load-bearing side effect:
|
|
531
|
-
// IIR-integrator drift
|
|
532
|
-
// `imuGate.getTotalAbsMetres()` is read
|
|
533
|
-
//
|
|
512
|
+
// F8.3 review-of-review (M3 revert): an earlier draft gated
|
|
513
|
+
// this on the pre-v0.6 `legacyDriver` prop because the Frame
|
|
514
|
+
// Processor driver doesn't consult `imuGate` for its own pose
|
|
515
|
+
// synthesis. That ignored a load-bearing side effect:
|
|
516
|
+
// `imuGate.resetAnchor()` bounds the IIR-integrator drift
|
|
517
|
+
// window per-accept, and `imuGate.getTotalAbsMetres()` is read
|
|
518
|
+
// at finalize time as `imuTranslationMetres` into the native
|
|
534
519
|
// stitchMode auto-resolver (PANORAMA vs SCANS). Without the
|
|
535
520
|
// per-accept reset, long FP-driver captures let IIR drift
|
|
536
|
-
// compound → inflated metres → biased toward SCANS.
|
|
537
|
-
//
|
|
521
|
+
// compound → inflated metres → biased toward SCANS. Now fires
|
|
522
|
+
// for ALL non-AR captures (the only non-AR driver post-v0.6).
|
|
538
523
|
if (isNonAR) {
|
|
539
524
|
imuGate.resetAnchor();
|
|
540
525
|
}
|
|
@@ -651,13 +636,11 @@ function Camera(props) {
|
|
|
651
636
|
snapshotEveryNAccepts: 1,
|
|
652
637
|
frameRotationDegrees: orientationRotation,
|
|
653
638
|
captureOrientation: deviceOrientation,
|
|
654
|
-
//
|
|
655
|
-
//
|
|
656
|
-
//
|
|
657
|
-
// ARSession-driven path.
|
|
658
|
-
frameSourceMode: isNonAR
|
|
659
|
-
? (legacyDriver ? 'jsDriver' : 'frameProcessor')
|
|
660
|
-
: 'arSession',
|
|
639
|
+
// Non-AR captures use the Frame Processor driver
|
|
640
|
+
// (vision-camera producer-thread worklet → cv_flow_gate
|
|
641
|
+
// plugin → IncrementalStitcher.consumeFrame). AR captures
|
|
642
|
+
// use the ARSession-driven path.
|
|
643
|
+
frameSourceMode: isNonAR ? 'frameProcessor' : 'arSession',
|
|
661
644
|
composeWidth: 1920,
|
|
662
645
|
composeHeight: 1080,
|
|
663
646
|
canvasWidth: 5000,
|
|
@@ -673,22 +656,13 @@ function Camera(props) {
|
|
|
673
656
|
// matching comment on the per-accept reset useEffect above).
|
|
674
657
|
// Keep firing it on every capture start, not just legacy mode.
|
|
675
658
|
imuGate.resetAnchor();
|
|
676
|
-
// Start the non-AR
|
|
677
|
-
// ARSession so
|
|
678
|
-
//
|
|
679
|
-
//
|
|
680
|
-
//
|
|
681
|
-
// * Legacy: JS driver — `takeSnapshot` + `processFrameAtPath`
|
|
682
|
-
// via the cameraRef.
|
|
683
|
-
// Imperative-pattern rationale: see the useIncrementalJSDriver
|
|
684
|
-
// comment above re. why this isn't a useEffect.
|
|
659
|
+
// Start the Frame Processor driver for non-AR captures. AR
|
|
660
|
+
// mode feeds natively from ARSession so the driver stays idle.
|
|
661
|
+
// Imperative pattern (vs useEffect) because the driver's start
|
|
662
|
+
// resets pose accumulators that should only fire at the
|
|
663
|
+
// hold-start moment, not on every re-render.
|
|
685
664
|
if (isNonAR) {
|
|
686
|
-
|
|
687
|
-
jsDriver.start(visionCameraRef);
|
|
688
|
-
}
|
|
689
|
-
else {
|
|
690
|
-
fpDriver.start();
|
|
691
|
-
}
|
|
665
|
+
fpDriver.start();
|
|
692
666
|
}
|
|
693
667
|
}
|
|
694
668
|
catch (err) {
|
|
@@ -703,9 +677,7 @@ function Camera(props) {
|
|
|
703
677
|
settings,
|
|
704
678
|
effectiveCaptureSource,
|
|
705
679
|
imuGate,
|
|
706
|
-
jsDriver,
|
|
707
680
|
fpDriver,
|
|
708
|
-
legacyDriver,
|
|
709
681
|
engine,
|
|
710
682
|
onError,
|
|
711
683
|
]);
|
|
@@ -715,10 +687,7 @@ function Camera(props) {
|
|
|
715
687
|
setStatusPhase('stitching');
|
|
716
688
|
// Stop pumping new frames before finalizing so the engine isn't
|
|
717
689
|
// racing the final cv::Stitcher pass against late-arriving
|
|
718
|
-
// keyframes.
|
|
719
|
-
// corresponding driver wasn't started (AR mode, or the inactive
|
|
720
|
-
// driver in non-AR mode).
|
|
721
|
-
jsDriver.stop();
|
|
690
|
+
// keyframes. No-op in AR mode (the driver was never started).
|
|
722
691
|
fpDriver.stop();
|
|
723
692
|
try {
|
|
724
693
|
// Compose the panorama output path: host-controlled if
|
|
@@ -788,7 +757,6 @@ function Camera(props) {
|
|
|
788
757
|
onFramesDropped,
|
|
789
758
|
onError,
|
|
790
759
|
recordingStartedAt,
|
|
791
|
-
jsDriver,
|
|
792
760
|
fpDriver,
|
|
793
761
|
// F10 Phase 2 review N1 — these four were missing pre-fix. The
|
|
794
762
|
// callback reads `settings.debug` (to gate the stitchToast),
|
package/dist/index.d.ts
CHANGED
|
@@ -62,10 +62,9 @@ export type { TakePhotoCallOptions } from './camera/useCapture';
|
|
|
62
62
|
export { useVideoCapture } from './camera/useVideoCapture';
|
|
63
63
|
export { useDeviceOrientation } from './camera/useDeviceOrientation';
|
|
64
64
|
export { IncrementalOutcome, incrementalStitcherIsAvailable, subscribeIncrementalState, getIncrementalNativeModule, cleanupOldKeyframes, } from './stitching/incremental';
|
|
65
|
-
export type { IncrementalState } from './stitching/incremental';
|
|
65
|
+
export type { IncrementalState, AcceptedKeyframe } from './stitching/incremental';
|
|
66
66
|
export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
|
|
67
|
-
export {
|
|
68
|
-
export type { UseIncrementalJSDriverOptions, IncrementalJSDriverHandle, } from './stitching/useIncrementalJSDriver';
|
|
67
|
+
export { useKeyframeStream } from './stitching/useKeyframeStream';
|
|
69
68
|
export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
|
|
70
69
|
export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
|
|
71
70
|
export { stitchVideo } from './stitching/stitchVideo';
|
package/dist/index.js
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* adds RetaiLens-specific features on top.
|
|
23
23
|
*/
|
|
24
24
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
-
exports.stitchVideo = exports.useFrameProcessorDriver = exports.
|
|
25
|
+
exports.stitchVideo = exports.useFrameProcessorDriver = exports.useKeyframeStream = exports.useIncrementalStitcher = exports.cleanupOldKeyframes = exports.getIncrementalNativeModule = exports.subscribeIncrementalState = exports.incrementalStitcherIsAvailable = exports.IncrementalOutcome = exports.useDeviceOrientation = exports.useVideoCapture = exports.useCapture = exports.ViewportCropOverlay = exports.hybridSettingsToNativeConfig = exports.slitscanSettingsToNativeConfig = exports.panoramaSettingsToNativeConfig = exports.DEFAULT_HYBRID_SETTINGS = exports.DEFAULT_SLITSCAN_SETTINGS = exports.DEFAULT_FLOW_GATE_SETTINGS = exports.DEFAULT_PANORAMA_SETTINGS = exports.PanoramaSettingsModal = exports.PanoramaGuidance = exports.PanoramaBandOverlay = exports.IncrementalPanGuide = exports.CaptureThumbnailStrip = exports.useStitchStatsToast = exports.CaptureStitchStatsToast = exports.CaptureOrientationPill = exports.CaptureKeyframePill = exports.CaptureMemoryPill = exports.CaptureDebugOverlay = exports.CaptureStatusOverlay = exports.CapturePreview = exports.CaptureControlsBar = exports.CaptureHeader = exports.CameraView = exports.ARCameraView = exports.useIMUTranslationGate = exports.ARTrackingState = exports.useARSession = exports.CameraError = exports.Camera = void 0;
|
|
26
26
|
// ─────────────────────────────────────────────────────────────────────
|
|
27
27
|
// Layer 1 — the high-level <Camera> component
|
|
28
28
|
// ─────────────────────────────────────────────────────────────────────
|
|
@@ -139,11 +139,15 @@ Object.defineProperty(exports, "getIncrementalNativeModule", { enumerable: true,
|
|
|
139
139
|
Object.defineProperty(exports, "cleanupOldKeyframes", { enumerable: true, get: function () { return incremental_1.cleanupOldKeyframes; } });
|
|
140
140
|
var useIncrementalStitcher_1 = require("./stitching/useIncrementalStitcher");
|
|
141
141
|
Object.defineProperty(exports, "useIncrementalStitcher", { enumerable: true, get: function () { return useIncrementalStitcher_1.useIncrementalStitcher; } });
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
//
|
|
145
|
-
//
|
|
146
|
-
|
|
142
|
+
// v0.7.0 — Tier 1 subscriber API. Fires on each accepted keyframe
|
|
143
|
+
// in batch-keyframe captures (see hook's docstring for engine-mode
|
|
144
|
+
// caveat). Foundation for plugin-pattern host features (OCR per
|
|
145
|
+
// keyframe, packet detection, server-side analysis, etc.).
|
|
146
|
+
var useKeyframeStream_1 = require("./stitching/useKeyframeStream");
|
|
147
|
+
Object.defineProperty(exports, "useKeyframeStream", { enumerable: true, get: function () { return useKeyframeStream_1.useKeyframeStream; } });
|
|
148
|
+
// vision-camera Frame Processor driver for non-AR captures. As
|
|
149
|
+
// of v0.6 the only non-AR driver exported (the legacy
|
|
150
|
+
// `useIncrementalJSDriver` was removed; was deprecated in v0.5).
|
|
147
151
|
var useFrameProcessorDriver_1 = require("./stitching/useFrameProcessorDriver");
|
|
148
152
|
Object.defineProperty(exports, "useFrameProcessorDriver", { enumerable: true, get: function () { return useFrameProcessorDriver_1.useFrameProcessorDriver; } });
|
|
149
153
|
// ── Batch stitching ───────────────────────────────────────────────────
|
|
@@ -54,6 +54,48 @@ export declare enum IncrementalOutcome {
|
|
|
54
54
|
*/
|
|
55
55
|
SkippedKeyframeMaxReached = 9
|
|
56
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* v0.7.0 (Tier 1) — public payload type for an accepted keyframe.
|
|
59
|
+
* Delivered to subscribers of the `useKeyframeStream` hook.
|
|
60
|
+
*
|
|
61
|
+
* Emits once per keyframe accepted by the stitching engine — typically
|
|
62
|
+
* 4-6 times per panorama, not per camera frame. Use for low-frequency
|
|
63
|
+
* per-keyframe host work (OCR on the saved JPEG, packet detection,
|
|
64
|
+
* server-side analysis, analytics, etc.).
|
|
65
|
+
*
|
|
66
|
+
* Caveat: only the `batch-keyframe` engine emits these events as of
|
|
67
|
+
* v0.7.0. Live engines (`firstwins-rectilinear`, `hybrid`,
|
|
68
|
+
* `slitscan-*`) paint into a live canvas instead of saving per-accept
|
|
69
|
+
* JPEGs and do not currently surface accept events through this
|
|
70
|
+
* channel; the hook silently does not fire there. A v0.7.1 follow-up
|
|
71
|
+
* may add live-engine accept emit if a real consumer needs it.
|
|
72
|
+
*
|
|
73
|
+
* The JPEG at `jpegPath` is the engine's own copy under the active
|
|
74
|
+
* capture's session directory. It persists for the lifetime of the
|
|
75
|
+
* panorama and is cleaned up automatically when the panorama finalises
|
|
76
|
+
* or is abandoned (or via explicit `cleanupKeyframes`). Host code
|
|
77
|
+
* wanting to retain it long-term must copy synchronously inside the
|
|
78
|
+
* handler.
|
|
79
|
+
*/
|
|
80
|
+
export interface AcceptedKeyframe {
|
|
81
|
+
/** Absolute filesystem path to the keyframe JPEG. No `file://` prefix. */
|
|
82
|
+
jpegPath: string;
|
|
83
|
+
/**
|
|
84
|
+
* Pose snapshot at the moment of acceptance. Quaternion
|
|
85
|
+
* convention: `(x, y, z, w)`; lib uses
|
|
86
|
+
* `q = q_yaw * q_pitch * q_roll`. Translation in metres (world
|
|
87
|
+
* coords) is present in AR mode and undefined in non-AR mode (no
|
|
88
|
+
* spatial anchor — only gyro-derived rotation is available).
|
|
89
|
+
*/
|
|
90
|
+
pose: {
|
|
91
|
+
rotation: [number, number, number, number];
|
|
92
|
+
translation?: [number, number, number];
|
|
93
|
+
};
|
|
94
|
+
/** Milliseconds since the Unix epoch when the engine accepted this keyframe. */
|
|
95
|
+
timestamp: number;
|
|
96
|
+
/** Zero-based index of this keyframe within the in-progress panorama. */
|
|
97
|
+
index: number;
|
|
98
|
+
}
|
|
57
99
|
export interface IncrementalState {
|
|
58
100
|
/**
|
|
59
101
|
* Path to the latest panorama snapshot JPEG (file path, no
|
|
@@ -145,6 +187,33 @@ export interface IncrementalState {
|
|
|
145
187
|
* for the thumbnail strip.
|
|
146
188
|
*/
|
|
147
189
|
batchKeyframeIndex?: number;
|
|
190
|
+
/**
|
|
191
|
+
* v0.7.0 (Tier 1) — pose snapshot at the moment the engine
|
|
192
|
+
* accepted this keyframe. Populated alongside
|
|
193
|
+
* `batchKeyframeThumbnailPath` + `batchKeyframeIndex` on the
|
|
194
|
+
* keyframe-accepted state emit from the `batch-keyframe` engine.
|
|
195
|
+
* Undefined for other engines and for non-accept events.
|
|
196
|
+
*
|
|
197
|
+
* Quaternion convention: `(x, y, z, w)`; lib uses
|
|
198
|
+
* `q = q_yaw * q_pitch * q_roll`. AR mode populates `translation`
|
|
199
|
+
* from the AR camera transform (metres, world coords). Non-AR
|
|
200
|
+
* mode omits `translation` (no spatial anchor — only gyro-derived
|
|
201
|
+
* rotation is available).
|
|
202
|
+
*
|
|
203
|
+
* Foundation for the `useKeyframeStream` Tier 1 host hook.
|
|
204
|
+
*/
|
|
205
|
+
batchKeyframePose?: {
|
|
206
|
+
rotation: [number, number, number, number];
|
|
207
|
+
translation?: [number, number, number];
|
|
208
|
+
};
|
|
209
|
+
/**
|
|
210
|
+
* v0.7.0 (Tier 1) — monotonic timestamp (milliseconds since the
|
|
211
|
+
* Unix epoch) when the engine accepted this keyframe. Populated
|
|
212
|
+
* alongside the other `batchKeyframe*` fields on the
|
|
213
|
+
* keyframe-accepted emit. Undefined for other engines and for
|
|
214
|
+
* non-accept events.
|
|
215
|
+
*/
|
|
216
|
+
batchKeyframeAcceptedAtMs?: number;
|
|
148
217
|
/**
|
|
149
218
|
* 2026-05-16 — realtime+batch fusion (Option A "Replace on
|
|
150
219
|
* completion"). True between the moment a hybrid-engine
|
|
@@ -191,23 +260,22 @@ export interface IncrementalStartOptions {
|
|
|
191
260
|
* bridge.start() requires `RNSARSession.start()` to
|
|
192
261
|
* have already been called.
|
|
193
262
|
*
|
|
194
|
-
* - 'jsDriver' — engine skips AR-session registration; JS
|
|
195
|
-
* feeds frames via `processFrameAtPath`. Use in iOS non-AR
|
|
196
|
-
* captures (vision-camera + gyro). No AR session required.
|
|
197
|
-
* LEGACY; deprecated in v0.5, removed in v0.6.
|
|
198
|
-
*
|
|
199
263
|
* - 'frameProcessor' (F8.3 iOS / F8.4 Android, v0.5+) — engine
|
|
200
264
|
* flips on `frameProcessorIngestEnabled` so the vision-camera
|
|
201
265
|
* Frame Processor plugin (`cv_flow_gate_process_frame`) can
|
|
202
266
|
* feed pixel data directly into the engine's gate path. iOS
|
|
203
267
|
* passes the `CVPixelBuffer` straight to `consumeFrame`;
|
|
204
|
-
* Android extracts the Y plane to a ByteArray and
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
* captures driven by `useFrameProcessorDriver`. Pairs
|
|
208
|
-
* `Camera`'s default driver mode.
|
|
268
|
+
* Android extracts the Y plane to a ByteArray and (since
|
|
269
|
+
* F8.6, v0.5.1) routes live-engine ingest through
|
|
270
|
+
* `addFramePixelData` without a JPEG round-trip. Use in
|
|
271
|
+
* non-AR captures driven by `useFrameProcessorDriver`. Pairs
|
|
272
|
+
* with `Camera`'s default driver mode.
|
|
273
|
+
*
|
|
274
|
+
* `'jsDriver'` was removed in v0.6 (deprecated in v0.5). Hosts
|
|
275
|
+
* that used it should switch to `useFrameProcessorDriver` (or
|
|
276
|
+
* just let `<Camera>` use its default).
|
|
209
277
|
*/
|
|
210
|
-
frameSourceMode?: 'arSession' | '
|
|
278
|
+
frameSourceMode?: 'arSession' | 'frameProcessor';
|
|
211
279
|
/** Compose-resolution width in pixels (default 720 for portrait, 960 for landscape). */
|
|
212
280
|
composeWidth?: number;
|
|
213
281
|
/** Compose-resolution height in pixels (default 960 for portrait, 720 for landscape). */
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* useFrameProcessorDriver — vision-camera Frame Processor + gyro
|
|
3
|
-
* driver for the incremental panorama engine.
|
|
4
|
-
* `useIncrementalJSDriver`
|
|
3
|
+
* driver for the incremental panorama engine. Sole non-AR driver
|
|
4
|
+
* from v0.6 onward (replaced the deprecated `useIncrementalJSDriver`
|
|
5
|
+
* hook, which was removed in v0.6).
|
|
5
6
|
*
|
|
6
|
-
* Why this exists (vs the JS-driver predecessor)
|
|
7
|
+
* Why this exists (vs the pre-v0.6 JS-driver predecessor)
|
|
7
8
|
*
|
|
8
|
-
* The JS driver
|
|
9
|
-
* path to `IncrementalStitcher.processFrameAtPath
|
|
10
|
-
*
|
|
9
|
+
* The old JS driver took a JPEG snapshot every ~250 ms and fed the
|
|
10
|
+
* path to `IncrementalStitcher.processFrameAtPath` (both removed in
|
|
11
|
+
* v0.6). That path had three costs:
|
|
11
12
|
*
|
|
12
13
|
* 1. JPEG encode (`takeSnapshot` ≈ 30–80 ms on iPhone 16 Pro)
|
|
13
14
|
* 2. Disk write of the JPEG
|
|
@@ -2,14 +2,15 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
/**
|
|
4
4
|
* useFrameProcessorDriver — vision-camera Frame Processor + gyro
|
|
5
|
-
* driver for the incremental panorama engine.
|
|
6
|
-
* `useIncrementalJSDriver`
|
|
5
|
+
* driver for the incremental panorama engine. Sole non-AR driver
|
|
6
|
+
* from v0.6 onward (replaced the deprecated `useIncrementalJSDriver`
|
|
7
|
+
* hook, which was removed in v0.6).
|
|
7
8
|
*
|
|
8
|
-
* Why this exists (vs the JS-driver predecessor)
|
|
9
|
+
* Why this exists (vs the pre-v0.6 JS-driver predecessor)
|
|
9
10
|
*
|
|
10
|
-
* The JS driver
|
|
11
|
-
* path to `IncrementalStitcher.processFrameAtPath
|
|
12
|
-
*
|
|
11
|
+
* The old JS driver took a JPEG snapshot every ~250 ms and fed the
|
|
12
|
+
* path to `IncrementalStitcher.processFrameAtPath` (both removed in
|
|
13
|
+
* v0.6). That path had three costs:
|
|
13
14
|
*
|
|
14
15
|
* 1. JPEG encode (`takeSnapshot` ≈ 30–80 ms on iPhone 16 Pro)
|
|
15
16
|
* 2. Disk write of the JPEG
|
|
@@ -220,9 +221,9 @@ function useFrameProcessorDriver(options = {}) {
|
|
|
220
221
|
// y = horizontal pan (yaw, about world-Y)
|
|
221
222
|
// x = vertical tilt (pitch, about world-X)
|
|
222
223
|
// z = wrist-twist roll (about world-Z, normal to the screen)
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
//
|
|
224
|
+
// Right-hand-rule convention throughout — same signs the pre-v0.6
|
|
225
|
+
// `useIncrementalJSDriver` produced. If field captures show
|
|
226
|
+
// inverted roll, flip the sign on `z * dt` below.
|
|
226
227
|
(0, react_native_sensors_1.setUpdateIntervalForType)(react_native_sensors_1.SensorTypes.gyroscope, gyroIntervalMs);
|
|
227
228
|
gyroSubRef.current = react_native_sensors_1.gyroscope.subscribe({
|
|
228
229
|
next: ({ x, y, z }) => {
|
|
@@ -296,8 +297,8 @@ function useFrameProcessorDriver(options = {}) {
|
|
|
296
297
|
imageWidth: w, imageHeight: h,
|
|
297
298
|
timestampMs: 0,
|
|
298
299
|
// 2 == RNSARTrackingState.tracking — we always claim "good
|
|
299
|
-
// tracking" because there's no ARKit signal to differentiate
|
|
300
|
-
// (
|
|
300
|
+
// tracking" because there's no ARKit signal to differentiate.
|
|
301
|
+
// (Same contract as the pre-v0.6 useIncrementalJSDriver.)
|
|
301
302
|
trackingStateRaw: 2,
|
|
302
303
|
});
|
|
303
304
|
// Deps array intentionally minimal: only `plugin` actually
|