react-native-image-stitcher 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +200 -8
- package/android/src/main/java/io/imagestitcher/rn/CvFlowGateFrameProcessor.kt +2 -2
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +120 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +266 -385
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +6 -3
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +17 -30
- package/dist/camera/Camera.d.ts +29 -27
- package/dist/camera/Camera.js +48 -79
- package/dist/index.d.ts +0 -2
- package/dist/index.js +4 -6
- package/dist/stitching/incremental.d.ts +10 -11
- package/dist/stitching/useFrameProcessorDriver.d.ts +7 -6
- package/dist/stitching/useFrameProcessorDriver.js +12 -11
- package/ios/Package.swift +35 -21
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +85 -206
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +0 -8
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +6 -126
- package/ios/Sources/RNImageStitcher/KeyframeGateFrameProcessor.mm +6 -6
- package/package.json +1 -1
- package/src/camera/Camera.tsx +83 -107
- package/src/index.ts +3 -8
- package/src/stitching/incremental.ts +10 -11
- package/src/stitching/useFrameProcessorDriver.ts +12 -11
- package/dist/stitching/useIncrementalJSDriver.d.ts +0 -74
- package/dist/stitching/useIncrementalJSDriver.js +0 -220
- package/src/stitching/useIncrementalJSDriver.ts +0 -297
|
@@ -247,9 +247,12 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
247
247
|
* the Y plane from the ARCore camera image (YUV_420_888) and
|
|
248
248
|
* hands it through. Zero-copy on the way in (the byte[] is
|
|
249
249
|
* pinned via GetPrimitiveArrayCritical in the JNI).
|
|
250
|
-
* - Non-AR mode (`
|
|
251
|
-
*
|
|
252
|
-
*
|
|
250
|
+
* - Non-AR mode (`CvFlowGateFrameProcessor` via
|
|
251
|
+
* `IncrementalStitcher.consumeFrameFromPlugin`): extracts the
|
|
252
|
+
* Y plane from the vision-camera Frame's YUV_420_888 image on
|
|
253
|
+
* the producer thread and hands it through. (Pre-v0.6 a
|
|
254
|
+
* JS-driver `processFrameAtPath` path also called this with
|
|
255
|
+
* JPEG-decoded grayscale; both were removed in v0.6.)
|
|
253
256
|
*
|
|
254
257
|
* @param grayData The grayscale plane bytes. Length must be
|
|
255
258
|
* at least `grayStride * grayHeight`.
|
|
@@ -18,7 +18,6 @@ import com.google.ar.core.exceptions.CameraNotAvailableException
|
|
|
18
18
|
import com.google.ar.core.exceptions.SessionPausedException
|
|
19
19
|
import io.imagestitcher.rn.ar.BackgroundRenderer
|
|
20
20
|
import io.imagestitcher.rn.ar.YuvImageConverter
|
|
21
|
-
import java.io.File
|
|
22
21
|
import java.util.concurrent.atomic.AtomicReference
|
|
23
22
|
import javax.microedition.khronos.egl.EGLConfig
|
|
24
23
|
import javax.microedition.khronos.opengles.GL10
|
|
@@ -76,13 +75,6 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
76
75
|
private var surfaceWidth: Int = 0
|
|
77
76
|
private var surfaceHeight: Int = 0
|
|
78
77
|
|
|
79
|
-
/// Tmp directory for the per-frame JPEG file we hand to the
|
|
80
|
-
/// incremental engine. Created lazily and reused across frames
|
|
81
|
-
/// — no per-frame allocation.
|
|
82
|
-
private val tmpJpegFile: File by lazy {
|
|
83
|
-
File(context.cacheDir, "rlis-arframe.jpg")
|
|
84
|
-
}
|
|
85
|
-
|
|
86
78
|
/// Whether to feed the AR session's frames into the incremental
|
|
87
79
|
/// engine. Toggled by IncrementalStitcher.start/stop
|
|
88
80
|
/// via setIncrementalIngestionActive() below.
|
|
@@ -510,28 +502,18 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
510
502
|
val rotationForEncode = if (lastDisplayRotation >= 0)
|
|
511
503
|
lastDisplayRotation else android.view.Surface.ROTATION_0
|
|
512
504
|
|
|
513
|
-
//
|
|
514
|
-
//
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
//
|
|
518
|
-
//
|
|
519
|
-
//
|
|
505
|
+
// F8.6 (v0.6) — the eager JPEG encode for live-engine mode
|
|
506
|
+
// is gone. Pass the already-packed NV21 directly via
|
|
507
|
+
// `nv21PixelData`; the engine's new `addFramePixelData`
|
|
508
|
+
// path builds the BGR cv::Mat in-process via cvtColor,
|
|
509
|
+
// skipping the JPEG decode round-trip downstream. In
|
|
510
|
+
// batch-keyframe mode the engine ignores `nv21PixelData`
|
|
511
|
+
// (it uses `grayData` + `onAccept` lazily); no behaviour
|
|
512
|
+
// change there.
|
|
520
513
|
//
|
|
521
|
-
//
|
|
522
|
-
//
|
|
523
|
-
//
|
|
524
|
-
// hold time.
|
|
525
|
-
val legacyJpegPath: String? = if (module.isBatchKeyframeMode) {
|
|
526
|
-
null
|
|
527
|
-
} else {
|
|
528
|
-
YuvImageConverter.encodeJpegFromNV21(
|
|
529
|
-
packed,
|
|
530
|
-
tmpJpegFile.absolutePath,
|
|
531
|
-
jpegQuality = 70,
|
|
532
|
-
displayRotation = rotationForEncode,
|
|
533
|
-
)
|
|
534
|
-
}
|
|
514
|
+
// (Was: eager JPEG encode for non-batch-keyframe modes,
|
|
515
|
+
// written to `tmpJpegFile`, passed as `legacyJpegPath`.
|
|
516
|
+
// See the v0.3 / F8.6 entries in CHANGELOG.md.)
|
|
535
517
|
module.ingestFromARCameraView(
|
|
536
518
|
tx = tArr[0].toDouble(),
|
|
537
519
|
ty = tArr[1].toDouble(),
|
|
@@ -553,7 +535,12 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
553
535
|
grayWidth = packed.width,
|
|
554
536
|
grayHeight = packed.height,
|
|
555
537
|
grayStride = packed.width,
|
|
556
|
-
legacyJpegPath =
|
|
538
|
+
legacyJpegPath = null,
|
|
539
|
+
// F8.6 — pixel-data path for live engines. Batch-
|
|
540
|
+
// keyframe mode ignores these (bails earlier).
|
|
541
|
+
nv21PixelData = packed.nv21,
|
|
542
|
+
nv21PixelWidth = packed.width,
|
|
543
|
+
nv21PixelHeight = packed.height,
|
|
557
544
|
onAccept = { targetPath ->
|
|
558
545
|
// Lazy JPEG encode. Runs ONLY if the C++ KeyframeGate
|
|
559
546
|
// accepted the frame. Encodes from the pre-packed
|
package/dist/camera/Camera.d.ts
CHANGED
|
@@ -140,6 +140,24 @@ export interface CameraProps {
|
|
|
140
140
|
enablePanoramaMode?: boolean;
|
|
141
141
|
showSettingsButton?: boolean;
|
|
142
142
|
style?: StyleProp<ViewStyle>;
|
|
143
|
+
/**
|
|
144
|
+
* Which incremental stitcher engine to drive. Default
|
|
145
|
+
* `'batch-keyframe'` — collects accepted JPEGs and runs
|
|
146
|
+
* `cv::Stitcher` once at finalize time. This is the v0.4+
|
|
147
|
+
* production default and what the v0.5 Frame Processor migration
|
|
148
|
+
* exercises.
|
|
149
|
+
*
|
|
150
|
+
* Switch to a live engine (`'firstwins-rectilinear'` or
|
|
151
|
+
* `'hybrid'`) for low-latency in-flight stitching. Live engines
|
|
152
|
+
* exercise the F8.6 pixel-buffer ingest path (skipping the JPEG
|
|
153
|
+
* encode/decode round-trip; ~30–50 ms saved per accept) when the
|
|
154
|
+
* Frame Processor driver is active.
|
|
155
|
+
*
|
|
156
|
+
* See `docs/f8-frame-processor-plan.md` and the v0.5.0
|
|
157
|
+
* CHANGELOG for the trade-offs between batch-keyframe and live
|
|
158
|
+
* engines.
|
|
159
|
+
*/
|
|
160
|
+
engine?: 'batch-keyframe' | 'hybrid' | 'slitscan-rotate' | 'slitscan-both' | 'firstwins' | 'firstwins-zoomed' | 'firstwins-rectilinear' | 'slitscan';
|
|
143
161
|
/**
|
|
144
162
|
* Optional destination directory for captures. When set, the lib
|
|
145
163
|
* lands tap-photos at `${outputDir}/photo-${ts}.jpg` and panoramas
|
|
@@ -182,36 +200,20 @@ export interface CameraProps {
|
|
|
182
200
|
* Introduced for F8 (FrameProcessor port) — see
|
|
183
201
|
* `docs/f8-frame-processor-plan.md`.
|
|
184
202
|
*
|
|
185
|
-
*
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
*
|
|
190
|
-
* one-time console.warn — supplying your own worklet would race
|
|
191
|
-
* with the SDK's pixel-buffer feed.
|
|
203
|
+
* The SDK installs its own frame processor via
|
|
204
|
+
* `useFrameProcessorDriver`. Setting this prop is ignored with
|
|
205
|
+
* a one-time `console.warn` — supplying a host worklet would
|
|
206
|
+
* race with the SDK's pixel-buffer feed. Either remove the prop
|
|
207
|
+
* or fork the SDK if you genuinely need a custom worklet.
|
|
192
208
|
*
|
|
193
|
-
*
|
|
194
|
-
* * Default (modern non-AR): SDK owns the worklet, this prop
|
|
195
|
-
* is ignored.
|
|
196
|
-
* * `legacyDriver={true}`: SDK uses the old `useIncrementalJSDriver`
|
|
197
|
-
* (takeSnapshot path). Honoured for diagnostics or as an
|
|
198
|
-
* escape hatch.
|
|
199
|
-
* * AR mode: vision-camera Camera isn't mounted, this prop is
|
|
200
|
-
* irrelevant.
|
|
201
|
-
*/
|
|
202
|
-
frameProcessor?: ReadonlyFrameProcessor | DrawableFrameProcessor;
|
|
203
|
-
/**
|
|
204
|
-
* Opt back into the legacy `useIncrementalJSDriver` for non-AR
|
|
205
|
-
* captures (the v0.4 path: `takeSnapshot` → JPEG → cache file →
|
|
206
|
-
* `IncrementalStitcher.processFrameAtPath`).
|
|
209
|
+
* AR mode is irrelevant: vision-camera's Camera isn't mounted.
|
|
207
210
|
*
|
|
208
|
-
*
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
*
|
|
212
|
-
* specific issue with the new driver and need to ship a fix.
|
|
211
|
+
* (v0.5 had a `legacyDriver` escape hatch that routed back to
|
|
212
|
+
* `useIncrementalJSDriver`. That hook + prop were removed in
|
|
213
|
+
* v0.6 per the deprecation timeline announced in the v0.5.0
|
|
214
|
+
* CHANGELOG.)
|
|
213
215
|
*/
|
|
214
|
-
|
|
216
|
+
frameProcessor?: ReadonlyFrameProcessor | DrawableFrameProcessor;
|
|
215
217
|
}
|
|
216
218
|
/**
|
|
217
219
|
* The public `<Camera>` component.
|
package/dist/camera/Camera.js
CHANGED
|
@@ -97,7 +97,6 @@ const lowMemDevice_1 = require("./lowMemDevice");
|
|
|
97
97
|
const useCapture_1 = require("./useCapture");
|
|
98
98
|
const useDeviceOrientation_1 = require("./useDeviceOrientation");
|
|
99
99
|
const incremental_1 = require("../stitching/incremental");
|
|
100
|
-
const useIncrementalJSDriver_1 = require("../stitching/useIncrementalJSDriver");
|
|
101
100
|
const useFrameProcessorDriver_1 = require("../stitching/useFrameProcessorDriver");
|
|
102
101
|
const useIncrementalStitcher_1 = require("../stitching/useIncrementalStitcher");
|
|
103
102
|
const useIMUTranslationGate_1 = require("../sensors/useIMUTranslationGate");
|
|
@@ -271,7 +270,7 @@ function extractPanoramaOverrides(props) {
|
|
|
271
270
|
* The public `<Camera>` component.
|
|
272
271
|
*/
|
|
273
272
|
function Camera(props) {
|
|
274
|
-
const { defaultCaptureSource = 'ar', defaultLens = '1x', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, frameProcessor: hostFrameProcessor,
|
|
273
|
+
const { defaultCaptureSource = 'ar', defaultLens = '1x', enablePhotoMode = true, enablePanoramaMode = true, showSettingsButton = false, style, outputDir, onCapture, onCaptureSourceChange, onLensChange, onFramesDropped, onError, frameProcessor: hostFrameProcessor, engine = 'batch-keyframe', } = props;
|
|
275
274
|
const insets = (0, react_native_safe_area_context_1.useSafeAreaInsets)();
|
|
276
275
|
// ── State ───────────────────────────────────────────────────────
|
|
277
276
|
const [arPreference, setArPreference] = (0, react_1.useState)(defaultCaptureSource === 'ar');
|
|
@@ -417,58 +416,44 @@ function Camera(props) {
|
|
|
417
416
|
mod?.markNextFrameAsLastKeyframe?.().catch(() => undefined);
|
|
418
417
|
},
|
|
419
418
|
});
|
|
420
|
-
//
|
|
421
|
-
// engine consumes frames from the ARSession stream
|
|
422
|
-
// hook stays idle.
|
|
419
|
+
// Frame Processor driver for non-AR captures (iOS + Android).
|
|
420
|
+
// In AR mode the engine consumes frames from the ARSession stream
|
|
421
|
+
// natively, so this hook stays idle.
|
|
423
422
|
//
|
|
424
423
|
// IMPORTANT: start()/stop() are called imperatively from the hold
|
|
425
424
|
// handlers below — NOT from a useEffect driven by statusPhase. The
|
|
426
425
|
// hook returns a fresh object identity on every render, and during
|
|
427
426
|
// a recording the engine emits IncrementalStateUpdate events that
|
|
428
|
-
// cause re-renders multiple times per second. An effect with
|
|
429
|
-
//
|
|
430
|
-
//
|
|
431
|
-
//
|
|
432
|
-
//
|
|
433
|
-
//
|
|
434
|
-
//
|
|
435
|
-
// imperative pattern (start on hold-start, stop on hold-end) avoids
|
|
436
|
-
// the re-render churn entirely.
|
|
437
|
-
const jsDriver = (0, useIncrementalJSDriver_1.useIncrementalJSDriver)();
|
|
438
|
-
// F8.3 — vision-camera Frame Processor variant. Always
|
|
439
|
-
// instantiated so we don't have conditional hook calls; only one
|
|
440
|
-
// of the two drivers actually .start()s per capture. Stop() on
|
|
441
|
-
// an idle driver is a no-op.
|
|
427
|
+
// cause re-renders multiple times per second. An effect with the
|
|
428
|
+
// driver in its deps would teardown + restart on every event,
|
|
429
|
+
// resetting the gyro accumulator (yaw/pitch) to zero each cycle.
|
|
430
|
+
// User-visible symptom: "only the first keyframe is accepted, every
|
|
431
|
+
// subsequent ingest sees pose=(0,0) and is rejected as a duplicate".
|
|
432
|
+
// The imperative pattern (start on hold-start, stop on hold-end)
|
|
433
|
+
// avoids the re-render churn entirely.
|
|
442
434
|
const fpDriver = (0, useFrameProcessorDriver_1.useFrameProcessorDriver)();
|
|
443
|
-
// Safety:
|
|
444
|
-
// mid-recording. Empty deps so this only fires on unmount.
|
|
435
|
+
// Safety: stop the driver if the component unmounts mid-recording.
|
|
445
436
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
446
|
-
(0, react_1.useEffect)(() => () => {
|
|
447
|
-
//
|
|
448
|
-
//
|
|
449
|
-
//
|
|
450
|
-
//
|
|
451
|
-
//
|
|
452
|
-
//
|
|
453
|
-
//
|
|
437
|
+
(0, react_1.useEffect)(() => () => { fpDriver.stop(); }, []);
|
|
438
|
+
// One-shot deprecation warning when the host supplies their own
|
|
439
|
+
// `frameProcessor` prop. Two worklets racing on the same
|
|
440
|
+
// producer thread would corrupt the engine's workQueue ordering,
|
|
441
|
+
// so the SDK's own worklet wins and the host's is silently
|
|
442
|
+
// ignored. (v0.5 had a `legacyDriver` opt-out for hosts that
|
|
443
|
+
// wanted to route around the SDK driver; that was removed in
|
|
444
|
+
// v0.6 along with `useIncrementalJSDriver`.)
|
|
454
445
|
const hostFrameProcessorIgnoredWarnedRef = (0, react_1.useRef)(false);
|
|
455
446
|
if (hostFrameProcessor != null
|
|
456
|
-
&& !legacyDriver
|
|
457
447
|
&& !hostFrameProcessorIgnoredWarnedRef.current) {
|
|
458
448
|
hostFrameProcessorIgnoredWarnedRef.current = true;
|
|
459
449
|
// eslint-disable-next-line no-console
|
|
460
450
|
console.warn('[react-native-image-stitcher] The `frameProcessor` prop on '
|
|
461
|
-
+ '<Camera> is ignored
|
|
462
|
-
+ '
|
|
463
|
-
+ '
|
|
451
|
+
+ '<Camera> is ignored — the SDK installs its own worklet '
|
|
452
|
+
+ 'via useFrameProcessorDriver. Remove the prop, or fork '
|
|
453
|
+
+ 'the SDK if you genuinely need a custom worklet.');
|
|
464
454
|
}
|
|
465
|
-
// The Frame Processor worklet
|
|
466
|
-
|
|
467
|
-
// 1. Legacy mode: honor the host's prop (or null).
|
|
468
|
-
// 2. Modern mode: SDK driver's worklet, regardless of host's prop.
|
|
469
|
-
const effectiveFrameProcessor = legacyDriver
|
|
470
|
-
? (hostFrameProcessor ?? null)
|
|
471
|
-
: fpDriver.frameProcessor;
|
|
455
|
+
// The Frame Processor worklet bound to vision-camera's Camera.
|
|
456
|
+
const effectiveFrameProcessor = fpDriver.frameProcessor;
|
|
472
457
|
// ── Subscribe to engine state for live keyframe thumbs ──────────
|
|
473
458
|
(0, react_1.useEffect)(() => {
|
|
474
459
|
const sub = (0, incremental_1.subscribeIncrementalState)((state) => {
|
|
@@ -524,17 +509,17 @@ function Camera(props) {
|
|
|
524
509
|
const accepted = incrementalState?.acceptedCount ?? 0;
|
|
525
510
|
if (accepted > lastAcceptedCountRef.current) {
|
|
526
511
|
lastAcceptedCountRef.current = accepted;
|
|
527
|
-
// F8.3 review-of-review (M3 revert):
|
|
528
|
-
// `legacyDriver` because the Frame
|
|
529
|
-
// consult `imuGate` for its own pose
|
|
530
|
-
// load-bearing side effect:
|
|
531
|
-
// IIR-integrator drift
|
|
532
|
-
// `imuGate.getTotalAbsMetres()` is read
|
|
533
|
-
//
|
|
512
|
+
// F8.3 review-of-review (M3 revert): an earlier draft gated
|
|
513
|
+
// this on the pre-v0.6 `legacyDriver` prop because the Frame
|
|
514
|
+
// Processor driver doesn't consult `imuGate` for its own pose
|
|
515
|
+
// synthesis. That ignored a load-bearing side effect:
|
|
516
|
+
// `imuGate.resetAnchor()` bounds the IIR-integrator drift
|
|
517
|
+
// window per-accept, and `imuGate.getTotalAbsMetres()` is read
|
|
518
|
+
// at finalize time as `imuTranslationMetres` into the native
|
|
534
519
|
// stitchMode auto-resolver (PANORAMA vs SCANS). Without the
|
|
535
520
|
// per-accept reset, long FP-driver captures let IIR drift
|
|
536
|
-
// compound → inflated metres → biased toward SCANS.
|
|
537
|
-
//
|
|
521
|
+
// compound → inflated metres → biased toward SCANS. Now fires
|
|
522
|
+
// for ALL non-AR captures (the only non-AR driver post-v0.6).
|
|
538
523
|
if (isNonAR) {
|
|
539
524
|
imuGate.resetAnchor();
|
|
540
525
|
}
|
|
@@ -651,18 +636,16 @@ function Camera(props) {
|
|
|
651
636
|
snapshotEveryNAccepts: 1,
|
|
652
637
|
frameRotationDegrees: orientationRotation,
|
|
653
638
|
captureOrientation: deviceOrientation,
|
|
654
|
-
//
|
|
655
|
-
//
|
|
656
|
-
//
|
|
657
|
-
// ARSession-driven path.
|
|
658
|
-
frameSourceMode: isNonAR
|
|
659
|
-
? (legacyDriver ? 'jsDriver' : 'frameProcessor')
|
|
660
|
-
: 'arSession',
|
|
639
|
+
// Non-AR captures use the Frame Processor driver
|
|
640
|
+
// (vision-camera producer-thread worklet → cv_flow_gate
|
|
641
|
+
// plugin → IncrementalStitcher.consumeFrame). AR captures
|
|
642
|
+
// use the ARSession-driven path.
|
|
643
|
+
frameSourceMode: isNonAR ? 'frameProcessor' : 'arSession',
|
|
661
644
|
composeWidth: 1920,
|
|
662
645
|
composeHeight: 1080,
|
|
663
646
|
canvasWidth: 5000,
|
|
664
647
|
canvasHeight: 5000,
|
|
665
|
-
engine
|
|
648
|
+
engine,
|
|
666
649
|
config: (0, PanoramaSettingsBridge_1.panoramaSettingsToNativeConfig)({
|
|
667
650
|
...settings,
|
|
668
651
|
captureSource: effectiveCaptureSource,
|
|
@@ -673,22 +656,13 @@ function Camera(props) {
|
|
|
673
656
|
// matching comment on the per-accept reset useEffect above).
|
|
674
657
|
// Keep firing it on every capture start, not just legacy mode.
|
|
675
658
|
imuGate.resetAnchor();
|
|
676
|
-
// Start the non-AR
|
|
677
|
-
// ARSession so
|
|
678
|
-
//
|
|
679
|
-
//
|
|
680
|
-
//
|
|
681
|
-
// * Legacy: JS driver — `takeSnapshot` + `processFrameAtPath`
|
|
682
|
-
// via the cameraRef.
|
|
683
|
-
// Imperative-pattern rationale: see the useIncrementalJSDriver
|
|
684
|
-
// comment above re. why this isn't a useEffect.
|
|
659
|
+
// Start the Frame Processor driver for non-AR captures. AR
|
|
660
|
+
// mode feeds natively from ARSession so the driver stays idle.
|
|
661
|
+
// Imperative pattern (vs useEffect) because the driver's start
|
|
662
|
+
// resets pose accumulators that should only fire at the
|
|
663
|
+
// hold-start moment, not on every re-render.
|
|
685
664
|
if (isNonAR) {
|
|
686
|
-
|
|
687
|
-
jsDriver.start(visionCameraRef);
|
|
688
|
-
}
|
|
689
|
-
else {
|
|
690
|
-
fpDriver.start();
|
|
691
|
-
}
|
|
665
|
+
fpDriver.start();
|
|
692
666
|
}
|
|
693
667
|
}
|
|
694
668
|
catch (err) {
|
|
@@ -703,9 +677,8 @@ function Camera(props) {
|
|
|
703
677
|
settings,
|
|
704
678
|
effectiveCaptureSource,
|
|
705
679
|
imuGate,
|
|
706
|
-
jsDriver,
|
|
707
680
|
fpDriver,
|
|
708
|
-
|
|
681
|
+
engine,
|
|
709
682
|
onError,
|
|
710
683
|
]);
|
|
711
684
|
const handleHoldEnd = (0, react_1.useCallback)(async () => {
|
|
@@ -714,10 +687,7 @@ function Camera(props) {
|
|
|
714
687
|
setStatusPhase('stitching');
|
|
715
688
|
// Stop pumping new frames before finalizing so the engine isn't
|
|
716
689
|
// racing the final cv::Stitcher pass against late-arriving
|
|
717
|
-
// keyframes.
|
|
718
|
-
// corresponding driver wasn't started (AR mode, or the inactive
|
|
719
|
-
// driver in non-AR mode).
|
|
720
|
-
jsDriver.stop();
|
|
690
|
+
// keyframes. No-op in AR mode (the driver was never started).
|
|
721
691
|
fpDriver.stop();
|
|
722
692
|
try {
|
|
723
693
|
// Compose the panorama output path: host-controlled if
|
|
@@ -787,7 +757,6 @@ function Camera(props) {
|
|
|
787
757
|
onFramesDropped,
|
|
788
758
|
onError,
|
|
789
759
|
recordingStartedAt,
|
|
790
|
-
jsDriver,
|
|
791
760
|
fpDriver,
|
|
792
761
|
// F10 Phase 2 review N1 — these four were missing pre-fix. The
|
|
793
762
|
// callback reads `settings.debug` (to gate the stitchToast),
|
package/dist/index.d.ts
CHANGED
|
@@ -64,8 +64,6 @@ export { useDeviceOrientation } from './camera/useDeviceOrientation';
|
|
|
64
64
|
export { IncrementalOutcome, incrementalStitcherIsAvailable, subscribeIncrementalState, getIncrementalNativeModule, cleanupOldKeyframes, } from './stitching/incremental';
|
|
65
65
|
export type { IncrementalState } from './stitching/incremental';
|
|
66
66
|
export { useIncrementalStitcher } from './stitching/useIncrementalStitcher';
|
|
67
|
-
export { useIncrementalJSDriver } from './stitching/useIncrementalJSDriver';
|
|
68
|
-
export type { UseIncrementalJSDriverOptions, IncrementalJSDriverHandle, } from './stitching/useIncrementalJSDriver';
|
|
69
67
|
export { useFrameProcessorDriver } from './stitching/useFrameProcessorDriver';
|
|
70
68
|
export type { UseFrameProcessorDriverOptions, FrameProcessorDriverHandle, } from './stitching/useFrameProcessorDriver';
|
|
71
69
|
export { stitchVideo } from './stitching/stitchVideo';
|
package/dist/index.js
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* adds RetaiLens-specific features on top.
|
|
23
23
|
*/
|
|
24
24
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
-
exports.stitchVideo = exports.useFrameProcessorDriver = exports.
|
|
25
|
+
exports.stitchVideo = exports.useFrameProcessorDriver = exports.useIncrementalStitcher = exports.cleanupOldKeyframes = exports.getIncrementalNativeModule = exports.subscribeIncrementalState = exports.incrementalStitcherIsAvailable = exports.IncrementalOutcome = exports.useDeviceOrientation = exports.useVideoCapture = exports.useCapture = exports.ViewportCropOverlay = exports.hybridSettingsToNativeConfig = exports.slitscanSettingsToNativeConfig = exports.panoramaSettingsToNativeConfig = exports.DEFAULT_HYBRID_SETTINGS = exports.DEFAULT_SLITSCAN_SETTINGS = exports.DEFAULT_FLOW_GATE_SETTINGS = exports.DEFAULT_PANORAMA_SETTINGS = exports.PanoramaSettingsModal = exports.PanoramaGuidance = exports.PanoramaBandOverlay = exports.IncrementalPanGuide = exports.CaptureThumbnailStrip = exports.useStitchStatsToast = exports.CaptureStitchStatsToast = exports.CaptureOrientationPill = exports.CaptureKeyframePill = exports.CaptureMemoryPill = exports.CaptureDebugOverlay = exports.CaptureStatusOverlay = exports.CapturePreview = exports.CaptureControlsBar = exports.CaptureHeader = exports.CameraView = exports.ARCameraView = exports.useIMUTranslationGate = exports.ARTrackingState = exports.useARSession = exports.CameraError = exports.Camera = void 0;
|
|
26
26
|
// ─────────────────────────────────────────────────────────────────────
|
|
27
27
|
// Layer 1 — the high-level <Camera> component
|
|
28
28
|
// ─────────────────────────────────────────────────────────────────────
|
|
@@ -139,11 +139,9 @@ Object.defineProperty(exports, "getIncrementalNativeModule", { enumerable: true,
|
|
|
139
139
|
Object.defineProperty(exports, "cleanupOldKeyframes", { enumerable: true, get: function () { return incremental_1.cleanupOldKeyframes; } });
|
|
140
140
|
var useIncrementalStitcher_1 = require("./stitching/useIncrementalStitcher");
|
|
141
141
|
Object.defineProperty(exports, "useIncrementalStitcher", { enumerable: true, get: function () { return useIncrementalStitcher_1.useIncrementalStitcher; } });
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
//
|
|
145
|
-
// driver. Preferred over `useIncrementalJSDriver` in v0.5+; the
|
|
146
|
-
// JS driver stays exported as a deprecated fallback until v0.6.
|
|
142
|
+
// vision-camera Frame Processor driver for non-AR captures. As
|
|
143
|
+
// of v0.6 the only non-AR driver exported (the legacy
|
|
144
|
+
// `useIncrementalJSDriver` was removed; was deprecated in v0.5).
|
|
147
145
|
var useFrameProcessorDriver_1 = require("./stitching/useFrameProcessorDriver");
|
|
148
146
|
Object.defineProperty(exports, "useFrameProcessorDriver", { enumerable: true, get: function () { return useFrameProcessorDriver_1.useFrameProcessorDriver; } });
|
|
149
147
|
// ── Batch stitching ───────────────────────────────────────────────────
|
|
@@ -191,23 +191,22 @@ export interface IncrementalStartOptions {
|
|
|
191
191
|
* bridge.start() requires `RNSARSession.start()` to
|
|
192
192
|
* have already been called.
|
|
193
193
|
*
|
|
194
|
-
* - 'jsDriver' — engine skips AR-session registration; JS
|
|
195
|
-
* feeds frames via `processFrameAtPath`. Use in iOS non-AR
|
|
196
|
-
* captures (vision-camera + gyro). No AR session required.
|
|
197
|
-
* LEGACY; deprecated in v0.5, removed in v0.6.
|
|
198
|
-
*
|
|
199
194
|
* - 'frameProcessor' (F8.3 iOS / F8.4 Android, v0.5+) — engine
|
|
200
195
|
* flips on `frameProcessorIngestEnabled` so the vision-camera
|
|
201
196
|
* Frame Processor plugin (`cv_flow_gate_process_frame`) can
|
|
202
197
|
* feed pixel data directly into the engine's gate path. iOS
|
|
203
198
|
* passes the `CVPixelBuffer` straight to `consumeFrame`;
|
|
204
|
-
* Android extracts the Y plane to a ByteArray and
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
* captures driven by `useFrameProcessorDriver`. Pairs
|
|
208
|
-
* `Camera`'s default driver mode.
|
|
199
|
+
* Android extracts the Y plane to a ByteArray and (since
|
|
200
|
+
* F8.6, v0.5.1) routes live-engine ingest through
|
|
201
|
+
* `addFramePixelData` without a JPEG round-trip. Use in
|
|
202
|
+
* non-AR captures driven by `useFrameProcessorDriver`. Pairs
|
|
203
|
+
* with `Camera`'s default driver mode.
|
|
204
|
+
*
|
|
205
|
+
* `'jsDriver'` was removed in v0.6 (deprecated in v0.5). Hosts
|
|
206
|
+
* that used it should switch to `useFrameProcessorDriver` (or
|
|
207
|
+
* just let `<Camera>` use its default).
|
|
209
208
|
*/
|
|
210
|
-
frameSourceMode?: 'arSession' | '
|
|
209
|
+
frameSourceMode?: 'arSession' | 'frameProcessor';
|
|
211
210
|
/** Compose-resolution width in pixels (default 720 for portrait, 960 for landscape). */
|
|
212
211
|
composeWidth?: number;
|
|
213
212
|
/** Compose-resolution height in pixels (default 960 for portrait, 720 for landscape). */
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* useFrameProcessorDriver — vision-camera Frame Processor + gyro
|
|
3
|
-
* driver for the incremental panorama engine.
|
|
4
|
-
* `useIncrementalJSDriver`
|
|
3
|
+
* driver for the incremental panorama engine. Sole non-AR driver
|
|
4
|
+
* from v0.6 onward (replaced the deprecated `useIncrementalJSDriver`
|
|
5
|
+
* hook, which was removed in v0.6).
|
|
5
6
|
*
|
|
6
|
-
* Why this exists (vs the JS-driver predecessor)
|
|
7
|
+
* Why this exists (vs the pre-v0.6 JS-driver predecessor)
|
|
7
8
|
*
|
|
8
|
-
* The JS driver
|
|
9
|
-
* path to `IncrementalStitcher.processFrameAtPath
|
|
10
|
-
*
|
|
9
|
+
* The old JS driver took a JPEG snapshot every ~250 ms and fed the
|
|
10
|
+
* path to `IncrementalStitcher.processFrameAtPath` (both removed in
|
|
11
|
+
* v0.6). That path had three costs:
|
|
11
12
|
*
|
|
12
13
|
* 1. JPEG encode (`takeSnapshot` ≈ 30–80 ms on iPhone 16 Pro)
|
|
13
14
|
* 2. Disk write of the JPEG
|
|
@@ -2,14 +2,15 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
/**
|
|
4
4
|
* useFrameProcessorDriver — vision-camera Frame Processor + gyro
|
|
5
|
-
* driver for the incremental panorama engine.
|
|
6
|
-
* `useIncrementalJSDriver`
|
|
5
|
+
* driver for the incremental panorama engine. Sole non-AR driver
|
|
6
|
+
* from v0.6 onward (replaced the deprecated `useIncrementalJSDriver`
|
|
7
|
+
* hook, which was removed in v0.6).
|
|
7
8
|
*
|
|
8
|
-
* Why this exists (vs the JS-driver predecessor)
|
|
9
|
+
* Why this exists (vs the pre-v0.6 JS-driver predecessor)
|
|
9
10
|
*
|
|
10
|
-
* The JS driver
|
|
11
|
-
* path to `IncrementalStitcher.processFrameAtPath
|
|
12
|
-
*
|
|
11
|
+
* The old JS driver took a JPEG snapshot every ~250 ms and fed the
|
|
12
|
+
* path to `IncrementalStitcher.processFrameAtPath` (both removed in
|
|
13
|
+
* v0.6). That path had three costs:
|
|
13
14
|
*
|
|
14
15
|
* 1. JPEG encode (`takeSnapshot` ≈ 30–80 ms on iPhone 16 Pro)
|
|
15
16
|
* 2. Disk write of the JPEG
|
|
@@ -220,9 +221,9 @@ function useFrameProcessorDriver(options = {}) {
|
|
|
220
221
|
// y = horizontal pan (yaw, about world-Y)
|
|
221
222
|
// x = vertical tilt (pitch, about world-X)
|
|
222
223
|
// z = wrist-twist roll (about world-Z, normal to the screen)
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
//
|
|
224
|
+
// Right-hand-rule convention throughout — same signs the pre-v0.6
|
|
225
|
+
// `useIncrementalJSDriver` produced. If field captures show
|
|
226
|
+
// inverted roll, flip the sign on `z * dt` below.
|
|
226
227
|
(0, react_native_sensors_1.setUpdateIntervalForType)(react_native_sensors_1.SensorTypes.gyroscope, gyroIntervalMs);
|
|
227
228
|
gyroSubRef.current = react_native_sensors_1.gyroscope.subscribe({
|
|
228
229
|
next: ({ x, y, z }) => {
|
|
@@ -296,8 +297,8 @@ function useFrameProcessorDriver(options = {}) {
|
|
|
296
297
|
imageWidth: w, imageHeight: h,
|
|
297
298
|
timestampMs: 0,
|
|
298
299
|
// 2 == RNSARTrackingState.tracking — we always claim "good
|
|
299
|
-
// tracking" because there's no ARKit signal to differentiate
|
|
300
|
-
// (
|
|
300
|
+
// tracking" because there's no ARKit signal to differentiate.
|
|
301
|
+
// (Same contract as the pre-v0.6 useIncrementalJSDriver.)
|
|
301
302
|
trackingStateRaw: 2,
|
|
302
303
|
});
|
|
303
304
|
// Deps array intentionally minimal: only `plugin` actually
|
package/ios/Package.swift
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// swift-tools-version:5.
|
|
1
|
+
// swift-tools-version:5.10
|
|
2
2
|
//
|
|
3
3
|
// Package.swift — SwiftPM manifest used **only for command-line testing**
|
|
4
4
|
// of the algorithm layer (QualityChecker.swift). Production builds
|
|
@@ -38,26 +38,40 @@ let package = Package(
|
|
|
38
38
|
.target(
|
|
39
39
|
name: "RNImageStitcher",
|
|
40
40
|
path: "Sources/RNImageStitcher",
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
41
|
+
// F8.3.H2-target — instead of an `exclude` list (which broke
|
|
42
|
+
// every time a new .mm landed, e.g.
|
|
43
|
+
// `KeyframeGateFrameProcessor.mm` in F8.1, because SwiftPM
|
|
44
|
+
// still scans the directory and rejects "mixed language
|
|
45
|
+
// source files" if it sees both .swift and .mm), we use an
|
|
46
|
+
// explicit `sources` allowlist of files that compile cleanly
|
|
47
|
+
// on macOS (where `swift test` runs).
|
|
48
|
+
//
|
|
49
|
+
// What's in the allowlist:
|
|
50
|
+
// * QualityChecker.swift — Accelerate / CoreImage; macOS-OK.
|
|
51
|
+
// * KeyframeGate.swift — Foundation + simd; macOS-OK.
|
|
52
|
+
//
|
|
53
|
+
// What's NOT (intentionally):
|
|
54
|
+
// * Anything with `import UIKit` / `import ARKit` — iOS only.
|
|
55
|
+
// CocoaPods compiles them for the host app via the podspec
|
|
56
|
+
// source_files glob; SwiftPM macOS doesn't need them.
|
|
57
|
+
// * .mm / .m / .h files — same. Picked up by CocoaPods.
|
|
58
|
+
// * RN-bridge Swift files (`*Bridge.swift`) — `import React`,
|
|
59
|
+
// not a SwiftPM dep.
|
|
60
|
+
//
|
|
61
|
+
// The Frame Processor plugin's Swift⇄ObjC selector pin
|
|
62
|
+
// (formerly relied on by `FrameProcessorPluginSelectorTests`)
|
|
63
|
+
// is enforced as a compile-time `#selector(...)` reference
|
|
64
|
+
// inside `IncrementalStitcher.swift` itself — see the
|
|
65
|
+
// `_consumeFrameFromPluginSelectorPin` static. Drift breaks
|
|
66
|
+
// the SDK build, which is a stronger guarantee than a test
|
|
67
|
+
// that needs iOS-Simulator infrastructure to run.
|
|
68
|
+
sources: [
|
|
69
|
+
"QualityChecker.swift",
|
|
70
|
+
// KeyframeGate.swift depends on `KeyframeGateBridge` (ObjC
|
|
71
|
+
// class in .mm) and `RNSARFramePose` (from a UIKit-using
|
|
72
|
+
// Swift file), so it doesn't compile standalone under
|
|
73
|
+
// SwiftPM on macOS — only the CocoaPods build sees the
|
|
74
|
+
// full type graph.
|
|
61
75
|
]
|
|
62
76
|
),
|
|
63
77
|
.testTarget(
|