react-native-image-stitcher 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +511 -1
- package/README.md +1 -1
- package/android/src/main/cpp/keyframe_gate_jni.cpp +138 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +412 -40
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +128 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +87 -45
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +46 -4
- package/cpp/stitcher.cpp +101 -1
- package/cpp/stitcher.hpp +8 -0
- package/dist/camera/Camera.d.ts +9 -0
- package/dist/camera/Camera.js +165 -43
- package/dist/camera/CaptureDebugOverlay.d.ts +45 -0
- package/dist/camera/CaptureDebugOverlay.js +146 -0
- package/dist/camera/CaptureKeyframePill.d.ts +28 -0
- package/dist/camera/CaptureKeyframePill.js +60 -0
- package/dist/camera/CaptureMemoryPill.d.ts +28 -0
- package/dist/camera/CaptureMemoryPill.js +109 -0
- package/dist/camera/CaptureOrientationPill.d.ts +22 -0
- package/dist/camera/CaptureOrientationPill.js +44 -0
- package/dist/camera/CaptureStitchStatsToast.d.ts +45 -0
- package/dist/camera/CaptureStitchStatsToast.js +133 -0
- package/dist/camera/PanoramaSettings.d.ts +478 -0
- package/dist/camera/PanoramaSettings.js +120 -0
- package/dist/camera/PanoramaSettingsBridge.d.ts +84 -0
- package/dist/camera/PanoramaSettingsBridge.js +208 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +50 -298
- package/dist/camera/PanoramaSettingsModal.js +189 -354
- package/dist/camera/buildPanoramaInitialSettings.d.ts +70 -0
- package/dist/camera/buildPanoramaInitialSettings.js +97 -0
- package/dist/camera/lowMemDevice.d.ts +24 -0
- package/dist/camera/lowMemDevice.js +69 -0
- package/dist/index.d.ts +16 -2
- package/dist/index.js +37 -2
- package/dist/sensors/useIMUTranslationGate.d.ts +26 -0
- package/dist/sensors/useIMUTranslationGate.js +83 -1
- package/dist/stitching/incremental.d.ts +25 -0
- package/dist/stitching/useIncrementalStitcher.d.ts +12 -1
- package/dist/stitching/useIncrementalStitcher.js +7 -1
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +321 -7
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +8 -0
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +12 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +13 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +15 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +1 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +17 -4
- package/ios/Sources/RNImageStitcher/Stitcher.swift +6 -1
- package/package.json +6 -2
- package/src/camera/Camera.tsx +220 -54
- package/src/camera/CaptureDebugOverlay.tsx +180 -0
- package/src/camera/CaptureKeyframePill.tsx +77 -0
- package/src/camera/CaptureMemoryPill.tsx +96 -0
- package/src/camera/CaptureOrientationPill.tsx +57 -0
- package/src/camera/CaptureStitchStatsToast.tsx +155 -0
- package/src/camera/PanoramaSettings.ts +605 -0
- package/src/camera/PanoramaSettingsBridge.ts +238 -0
- package/src/camera/PanoramaSettingsModal.tsx +296 -988
- package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +375 -0
- package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +119 -0
- package/src/camera/__tests__/lowMemDevice.test.ts +52 -0
- package/src/camera/buildPanoramaInitialSettings.ts +139 -0
- package/src/camera/lowMemDevice.ts +71 -0
- package/src/index.ts +61 -3
- package/src/sensors/useIMUTranslationGate.ts +112 -1
- package/src/stitching/incremental.ts +25 -0
- package/src/stitching/useIncrementalStitcher.ts +18 -0
package/src/camera/Camera.tsx
CHANGED
|
@@ -63,12 +63,20 @@ import { ARCameraView, type ARCameraViewHandle } from './ARCameraView';
|
|
|
63
63
|
import { CameraShutter } from './CameraShutter';
|
|
64
64
|
import { CameraView } from './CameraView';
|
|
65
65
|
import { CaptureStatusOverlay, type CaptureStatusPhase } from './CaptureStatusOverlay';
|
|
66
|
+
import { CaptureDebugOverlay } from './CaptureDebugOverlay';
|
|
67
|
+
import { CaptureMemoryPill } from './CaptureMemoryPill';
|
|
68
|
+
import { CaptureKeyframePill } from './CaptureKeyframePill';
|
|
69
|
+
import { CaptureOrientationPill } from './CaptureOrientationPill';
|
|
70
|
+
import { CaptureStitchStatsToast, useStitchStatsToast } from './CaptureStitchStatsToast';
|
|
66
71
|
import { PanoramaBandOverlay } from './PanoramaBandOverlay';
|
|
72
|
+
import { type PanoramaSettings } from './PanoramaSettings';
|
|
73
|
+
import { panoramaSettingsToNativeConfig } from './PanoramaSettingsBridge';
|
|
74
|
+
import { PanoramaSettingsModal } from './PanoramaSettingsModal';
|
|
67
75
|
import {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
} from './
|
|
76
|
+
buildPanoramaInitialSettings,
|
|
77
|
+
type PanoramaPropOverrides,
|
|
78
|
+
} from './buildPanoramaInitialSettings';
|
|
79
|
+
import { isLowMemDevice } from './lowMemDevice';
|
|
72
80
|
import { useCapture } from './useCapture';
|
|
73
81
|
import { useDeviceOrientation } from './useDeviceOrientation';
|
|
74
82
|
import {
|
|
@@ -129,6 +137,15 @@ export type CameraCaptureResult =
|
|
|
129
137
|
framesDropped: number;
|
|
130
138
|
finalConfidenceThresh: number;
|
|
131
139
|
durationMs: number;
|
|
140
|
+
/**
|
|
141
|
+
* 2026-05-22 (audit F2g) — which cv::Stitcher pipeline the
|
|
142
|
+
* batch finalize ran (after auto-resolution if applicable).
|
|
143
|
+
* Useful for displaying a "Stitched as: scans" pill on the
|
|
144
|
+
* output preview. Undefined when the engine wasn't
|
|
145
|
+
* batch-keyframe (hybrid / slit-scan don't go through
|
|
146
|
+
* cv::Stitcher at finalize).
|
|
147
|
+
*/
|
|
148
|
+
stitchModeResolved?: 'panorama' | 'scans';
|
|
132
149
|
};
|
|
133
150
|
|
|
134
151
|
|
|
@@ -457,40 +474,31 @@ function deriveEffectiveCaptureSource(
|
|
|
457
474
|
|
|
458
475
|
|
|
459
476
|
/**
|
|
460
|
-
*
|
|
461
|
-
*
|
|
462
|
-
*
|
|
477
|
+
* Pluck the props that influence the initial PanoramaSettings tree.
|
|
478
|
+
* Kept inline (vs. a wide structural type) so future Camera prop
|
|
479
|
+
* additions don't accidentally widen the settings-translation
|
|
480
|
+
* surface — the pure builder in `./buildPanoramaInitialSettings.ts`
|
|
481
|
+
* has the canonical interface; this just forwards the relevant
|
|
482
|
+
* fields.
|
|
463
483
|
*
|
|
464
|
-
*
|
|
465
|
-
*
|
|
466
|
-
*
|
|
467
|
-
*
|
|
484
|
+
* The `default*ResolMP` props on `CameraProps` are documented as
|
|
485
|
+
* forward-looking no-ops; the new PanoramaSettings tree has no home
|
|
486
|
+
* for them yet (the v0.3 audit found cv::Stitcher's resol knobs
|
|
487
|
+
* aren't reached by either platform's bridge). They're accepted on
|
|
488
|
+
* the prop interface for API stability and ignored here.
|
|
468
489
|
*/
|
|
469
|
-
function
|
|
490
|
+
function extractPanoramaOverrides(props: CameraProps): PanoramaPropOverrides {
|
|
470
491
|
return {
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
DEFAULT_PANORAMA_SETTINGS.flowNoveltyPercentile,
|
|
482
|
-
flowEvalEveryNFrames:
|
|
483
|
-
props.defaultFlowEvalEveryNFrames ??
|
|
484
|
-
DEFAULT_PANORAMA_SETTINGS.flowEvalEveryNFrames,
|
|
485
|
-
flowMaxTranslationCm:
|
|
486
|
-
props.defaultFlowMaxTranslationCm ??
|
|
487
|
-
DEFAULT_PANORAMA_SETTINGS.flowMaxTranslationCm,
|
|
488
|
-
keyframeMaxCount:
|
|
489
|
-
props.defaultKeyframeMaxCount ??
|
|
490
|
-
DEFAULT_PANORAMA_SETTINGS.keyframeMaxCount,
|
|
491
|
-
keyframeOverlapThreshold:
|
|
492
|
-
props.defaultKeyframeOverlapThreshold ??
|
|
493
|
-
DEFAULT_PANORAMA_SETTINGS.keyframeOverlapThreshold,
|
|
492
|
+
defaultCaptureSource: props.defaultCaptureSource,
|
|
493
|
+
defaultStitchMode: props.defaultStitchMode,
|
|
494
|
+
defaultBlender: props.defaultBlender,
|
|
495
|
+
defaultSeamFinder: props.defaultSeamFinder,
|
|
496
|
+
defaultWarper: props.defaultWarper,
|
|
497
|
+
defaultFlowNoveltyPercentile: props.defaultFlowNoveltyPercentile,
|
|
498
|
+
defaultFlowEvalEveryNFrames: props.defaultFlowEvalEveryNFrames,
|
|
499
|
+
defaultFlowMaxTranslationCm: props.defaultFlowMaxTranslationCm,
|
|
500
|
+
defaultKeyframeMaxCount: props.defaultKeyframeMaxCount,
|
|
501
|
+
defaultKeyframeOverlapThreshold: props.defaultKeyframeOverlapThreshold,
|
|
494
502
|
};
|
|
495
503
|
}
|
|
496
504
|
|
|
@@ -532,7 +540,10 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
532
540
|
);
|
|
533
541
|
const [lens, setLens] = useState<CameraLens>(defaultLens);
|
|
534
542
|
const [settings, setSettings] = useState<PanoramaSettings>(() =>
|
|
535
|
-
|
|
543
|
+
buildPanoramaInitialSettings(
|
|
544
|
+
extractPanoramaOverrides(props),
|
|
545
|
+
isLowMemDevice(),
|
|
546
|
+
),
|
|
536
547
|
);
|
|
537
548
|
const [settingsModalVisible, setSettingsModalVisible] = useState(false);
|
|
538
549
|
const [statusPhase, setStatusPhase] = useState<CaptureStatusPhase>('idle');
|
|
@@ -540,6 +551,11 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
540
551
|
null,
|
|
541
552
|
);
|
|
542
553
|
const [incrementalState, setIncrementalState] = useState<IncrementalState | null>(null);
|
|
554
|
+
// 2026-05-22 (audit F9 + F3) — debug stitch-stats toast. Hook
|
|
555
|
+
// exposes an imperative API; we fire `showResult(finalizeResult)`
|
|
556
|
+
// on every successful finalize when settings.debug is on (gated
|
|
557
|
+
// a few hundred lines below in handleHoldEnd's onCapture branch).
|
|
558
|
+
const stitchToast = useStitchStatsToast();
|
|
543
559
|
const [batchKeyframeThumbnails, setBatchKeyframeThumbnails] = useState<
|
|
544
560
|
string[]
|
|
545
561
|
>([]);
|
|
@@ -666,12 +682,27 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
666
682
|
// the C++ engine to force-accept the next frame. This is what
|
|
667
683
|
// keeps non-AR captures producing keyframes at all (the flow-
|
|
668
684
|
// novelty algorithm alone is too strict in practice).
|
|
685
|
+
//
|
|
686
|
+
// 2026-05-22 (audit F2f) — IMU translation gate. The gate's own
|
|
687
|
+
// `totalAbsMetres` accumulator (banks each segment's |displacement|
|
|
688
|
+
// at every anchor reset) is the right input for the finalize-time
|
|
689
|
+
// auto-resolver in non-AR mode (where pose-derived translation is
|
|
690
|
+
// 0). Pre-F2f this was reconstructed from `fires × budget +
|
|
691
|
+
// |residual|` — which undercounted any time a non-IMU accept
|
|
692
|
+
// (flow novelty, force-last) reset the integrator before the
|
|
693
|
+
// budget threshold was reached.
|
|
694
|
+
// The translation budget lives at `frameSelection.flow.maxTranslationCm`
|
|
695
|
+
// in the new hierarchical settings shape. When `flow` is undefined
|
|
696
|
+
// (the consumer opted out of the flow strategy entirely), the gate
|
|
697
|
+
// stays disabled — same observable behaviour as v0.3's `0` default.
|
|
698
|
+
const flowMaxTranslationCm =
|
|
699
|
+
settings.frameSelection.flow?.maxTranslationCm ?? 0;
|
|
669
700
|
const imuGate = useIMUTranslationGate({
|
|
670
701
|
enabled:
|
|
671
702
|
isNonAR
|
|
672
703
|
&& statusPhase === 'recording'
|
|
673
|
-
&&
|
|
674
|
-
budgetMeters: Math.max(0.001,
|
|
704
|
+
&& flowMaxTranslationCm > 0,
|
|
705
|
+
budgetMeters: Math.max(0.001, flowMaxTranslationCm / 100.0),
|
|
675
706
|
onBudgetExceeded: () => {
|
|
676
707
|
const mod = getIncrementalNativeModule();
|
|
677
708
|
mod?.markNextFrameAsLastKeyframe?.().catch(() => undefined);
|
|
@@ -718,12 +749,52 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
718
749
|
});
|
|
719
750
|
return () => { sub?.remove?.(); };
|
|
720
751
|
}, []);
|
|
752
|
+
// 2026-05-23 (race fix) — Previously this useEffect cleared
|
|
753
|
+
// `batchKeyframeThumbnails` + `incrementalState` when statusPhase
|
|
754
|
+
// transitioned to 'recording'. But handleHoldStart is async
|
|
755
|
+
// (`await incremental.start(...)`), and on Android the ARSession
|
|
756
|
+
// was already alive on the GL thread — it could emit an ACCEPT
|
|
757
|
+
// event during the await window, BEFORE the effect ran. Order
|
|
758
|
+
// observed in logcat:
|
|
759
|
+
// 1. setStatusPhase('recording') queued
|
|
760
|
+
// 2. await incremental.start() yields
|
|
761
|
+
// 3. ARCore frame → ingest → JS [state] emit
|
|
762
|
+
// 4. setBatchKeyframeThumbnails((prev=[]) => [keyframe-0.jpg])
|
|
763
|
+
// 5. React commits statusPhase change → THIS effect ran
|
|
764
|
+
// 6. setBatchKeyframeThumbnails([]) ← WIPED frame 0!
|
|
765
|
+
// 7. Frame 1 arrives → updater sees prev=[] → adds only frame 1
|
|
766
|
+
// ⇒ final array missing keyframe-0.jpg
|
|
767
|
+
// The reset is now done synchronously at the top of
|
|
768
|
+
// handleHoldStart, before any await, so the GL thread can't race
|
|
769
|
+
// ahead. This effect is intentionally removed.
|
|
770
|
+
|
|
771
|
+
// 2026-05-22 (audit F2f) — every accepted keyframe is a fresh
|
|
772
|
+
// anchor for the IMU translation gate, regardless of which
|
|
773
|
+
// mechanism qualified the frame (flow novelty, plane-overlap,
|
|
774
|
+
// angular fallback, IMU-budget force-accept, force-last). Reset
|
|
775
|
+
// the gate's per-segment integrator on every acceptedCount
|
|
776
|
+
// increment so the operator sees `imuΔ` reset to 0 in the debug
|
|
777
|
+
// overlay after every accept — consistent UX regardless of WHY
|
|
778
|
+
// the gate took the frame. Pre-F2f only the IMU-budget path
|
|
779
|
+
// reset the integrator; flow accepts left `posX` ticking up
|
|
780
|
+
// forever, which surprised the user.
|
|
781
|
+
//
|
|
782
|
+
// The gate's `totalAbsMetres` cumulative accumulator banks the
|
|
783
|
+
// |segment displacement| before zeroing, so finalize-time
|
|
784
|
+
// translation magnitude is preserved across non-IMU accepts.
|
|
785
|
+
const lastAcceptedCountRef = useRef(0);
|
|
721
786
|
useEffect(() => {
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
787
|
+
const accepted = incrementalState?.acceptedCount ?? 0;
|
|
788
|
+
if (accepted > lastAcceptedCountRef.current) {
|
|
789
|
+
lastAcceptedCountRef.current = accepted;
|
|
790
|
+
if (isNonAR) {
|
|
791
|
+
imuGate.resetAnchor();
|
|
792
|
+
}
|
|
793
|
+
} else if (accepted === 0) {
|
|
794
|
+
// New capture (state cleared) — reset our edge-detect ref.
|
|
795
|
+
lastAcceptedCountRef.current = 0;
|
|
725
796
|
}
|
|
726
|
-
}, [
|
|
797
|
+
}, [incrementalState?.acceptedCount, isNonAR, imuGate]);
|
|
727
798
|
|
|
728
799
|
// ── Shutter handlers ────────────────────────────────────────────
|
|
729
800
|
|
|
@@ -805,12 +876,42 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
805
876
|
return;
|
|
806
877
|
}
|
|
807
878
|
try {
|
|
879
|
+
// 2026-05-23 (race fix) — synchronously clear thumbnails +
|
|
880
|
+
// engine state at the top of handleHoldStart, BEFORE awaiting
|
|
881
|
+
// incremental.start(). In the previous effect-based design
|
|
882
|
+
// the GL thread could ingest an AR frame during the await
|
|
883
|
+
// window and add to thumbnails BEFORE React's
|
|
884
|
+
// statusPhase-effect ran and wiped them. See the removed
|
|
885
|
+
// useEffect a few hundred lines above for the full log trace.
|
|
886
|
+
// Synchronous reset here means any racing frame ingest sees
|
|
887
|
+
// an empty array and accumulates from there.
|
|
888
|
+
setBatchKeyframeThumbnails([]);
|
|
889
|
+
setIncrementalState(null);
|
|
808
890
|
setStatusPhase('recording');
|
|
809
891
|
setRecordingStartedAt(Date.now());
|
|
810
892
|
const orientationRotation: 0 | 90 | 180 | 270 =
|
|
811
893
|
deviceOrientation === 'portrait' ? 90
|
|
812
894
|
: deviceOrientation === 'portrait-upside-down' ? 270
|
|
813
895
|
: 0;
|
|
896
|
+
// v0.4 — the inline-flat config dict that v0.3 maintained here
|
|
897
|
+
// moved into `panoramaSettingsToNativeConfig` (see
|
|
898
|
+
// PanoramaSettingsBridge.ts). That adapter is the single source
|
|
899
|
+
// of truth for the JS→native wire format; both this call site
|
|
900
|
+
// AND the modal's reset-to-defaults preview agree on the same
|
|
901
|
+
// mapping. Audit fixes F1 / F4 / F6 from v0.3 are now properties
|
|
902
|
+
// of the bridge (verified by the unit tests in
|
|
903
|
+
// src/camera/__tests__/PanoramaSettingsBridge.test.ts).
|
|
904
|
+
//
|
|
905
|
+
// 2026-05-23 — override `captureSource` with the runtime-derived
|
|
906
|
+
// `effectiveCaptureSource` (from `arPreference + lens +
|
|
907
|
+
// AR-device-support`). Pre-this change the camera-screen AR
|
|
908
|
+
// toggle wrote ONLY to local `arPreference` state while the
|
|
909
|
+
// bridge read `settings.captureSource` — so native could think
|
|
910
|
+
// the capture was AR while the operator had toggled it off (or
|
|
911
|
+
// vice-versa). Single source of truth now: whatever camera the
|
|
912
|
+
// operator can see is what native is told it is. The settings
|
|
913
|
+
// modal's `captureSource` control has been removed for the same
|
|
914
|
+
// reason — see PanoramaSettingsModal.tsx for the rationale.
|
|
814
915
|
await incremental.start({
|
|
815
916
|
snapshotJpegQuality: 75,
|
|
816
917
|
snapshotEveryNAccepts: 1,
|
|
@@ -822,18 +923,10 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
822
923
|
canvasWidth: 5000,
|
|
823
924
|
canvasHeight: 5000,
|
|
824
925
|
engine: 'batch-keyframe',
|
|
825
|
-
config: {
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
seamFinderType: settings.seamFinderType,
|
|
830
|
-
flowNoveltyPercentile: settings.flowNoveltyPercentile,
|
|
831
|
-
flowEvalEveryNFrames: settings.flowEvalEveryNFrames,
|
|
832
|
-
flowMaxTranslationCm: settings.flowMaxTranslationCm,
|
|
833
|
-
keyframeMaxCount: settings.keyframeMaxCount,
|
|
834
|
-
keyframeOverlapThreshold: settings.keyframeOverlapThreshold,
|
|
835
|
-
frameSelectionMode: 'flow-based',
|
|
836
|
-
},
|
|
926
|
+
config: panoramaSettingsToNativeConfig({
|
|
927
|
+
...settings,
|
|
928
|
+
captureSource: effectiveCaptureSource,
|
|
929
|
+
}),
|
|
837
930
|
});
|
|
838
931
|
imuGate.resetAnchor();
|
|
839
932
|
// Start pumping vision-camera snapshots into the engine for
|
|
@@ -861,6 +954,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
861
954
|
isNonAR,
|
|
862
955
|
deviceOrientation,
|
|
863
956
|
settings,
|
|
957
|
+
effectiveCaptureSource,
|
|
864
958
|
imuGate,
|
|
865
959
|
jsDriver,
|
|
866
960
|
onError,
|
|
@@ -882,10 +976,19 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
882
976
|
const panoOutputPath = outputDir
|
|
883
977
|
? `${toBareFilePath(outputDir).replace(/\/$/, '')}/${defaultPanoramaFilename()}`
|
|
884
978
|
: `${await getDefaultCaptureDir()}/${defaultPanoramaFilename()}`;
|
|
979
|
+
// 2026-05-22 (audit F2f) — total IMU translation directly from
|
|
980
|
+
// the gate's cumulative accumulator (banks |segment displacement|
|
|
981
|
+
// at every anchor reset, including non-IMU-driven resets like
|
|
982
|
+
// flow-novelty accepts). No more fires × budget + residual
|
|
983
|
+
// reconstruction. Only meaningful in non-AR mode (in AR the
|
|
984
|
+
// native side uses pose-derived translation and ignores this).
|
|
985
|
+
const imuTotalTranslationM =
|
|
986
|
+
isNonAR ? imuGate.getTotalAbsMetres() : 0;
|
|
885
987
|
const result = await incremental.finalize(
|
|
886
988
|
panoOutputPath,
|
|
887
989
|
90,
|
|
888
990
|
deviceOrientation,
|
|
991
|
+
imuTotalTranslationM,
|
|
889
992
|
);
|
|
890
993
|
if (
|
|
891
994
|
typeof result.framesRequested === 'number'
|
|
@@ -910,7 +1013,15 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
910
1013
|
(result.framesRequested ?? 0) - (result.framesIncluded ?? 0),
|
|
911
1014
|
finalConfidenceThresh: result.finalConfidenceThresh ?? -1,
|
|
912
1015
|
durationMs: Date.now() - (recordingStartedAt ?? Date.now()),
|
|
1016
|
+
stitchModeResolved: result.stitchModeResolved,
|
|
913
1017
|
});
|
|
1018
|
+
// 2026-05-22 (audit F9) — fire the debug stitch-stats toast on
|
|
1019
|
+
// every successful finalize when settings.debug is on. Shows
|
|
1020
|
+
// the leaveBiggestComponent retry telemetry + resolved mode so
|
|
1021
|
+
// the operator can see what choice the auto-resolver made.
|
|
1022
|
+
if (settings.debug) {
|
|
1023
|
+
stitchToast.showResult(result);
|
|
1024
|
+
}
|
|
914
1025
|
} catch (err) {
|
|
915
1026
|
const message = err instanceof Error ? err.message : String(err);
|
|
916
1027
|
const code: CameraErrorCode =
|
|
@@ -933,6 +1044,18 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
933
1044
|
onError,
|
|
934
1045
|
recordingStartedAt,
|
|
935
1046
|
jsDriver,
|
|
1047
|
+
// F10 Phase 2 review N1 — these four were missing pre-fix. The
|
|
1048
|
+
// callback reads `settings.debug` (to gate the stitchToast),
|
|
1049
|
+
// `isNonAR` (to decide whether to read IMU totalAbs translation),
|
|
1050
|
+
// `imuGate` (the read itself), and `stitchToast` (the toast hook
|
|
1051
|
+
// object). If any of those identities change between the user
|
|
1052
|
+
// pressing-and-holding the shutter and the release, the stale-
|
|
1053
|
+
// closure read could disagree with the actual current state.
|
|
1054
|
+
// Pre-existing v0.3 bug; v0.4 was the natural time to address it.
|
|
1055
|
+
settings,
|
|
1056
|
+
isNonAR,
|
|
1057
|
+
imuGate,
|
|
1058
|
+
stitchToast,
|
|
936
1059
|
]);
|
|
937
1060
|
|
|
938
1061
|
// ── Lens / AR-toggle handlers ───────────────────────────────────
|
|
@@ -988,6 +1111,49 @@ export function Camera(props: CameraProps): React.JSX.Element {
|
|
|
988
1111
|
recordingStartedAt={recordingStartedAt ?? undefined}
|
|
989
1112
|
/>
|
|
990
1113
|
|
|
1114
|
+
{/*
|
|
1115
|
+
2026-05-22 (audit F9 + F3) — debug UI suite, all gated on
|
|
1116
|
+
settings.debug. Mounts in <Camera> automatically; Layer-2
|
|
1117
|
+
hosts can import the individual components from the public
|
|
1118
|
+
API and compose their own debug surface. Layout:
|
|
1119
|
+
- top-left: orientation pill (purple)
|
|
1120
|
+
- top-center: keyframes pill (green/amber)
|
|
1121
|
+
- top-right: memory pill (green/amber/red)
|
|
1122
|
+
- top-center: stitch-stats toast (dark capsule, transient)
|
|
1123
|
+
- left-mid: detailed metrics block (overlap, processing,
|
|
1124
|
+
imuΔ, etc.) — uses CaptureDebugOverlay
|
|
1125
|
+
*/}
|
|
1126
|
+
{settings.debug && (
|
|
1127
|
+
<>
|
|
1128
|
+
<CaptureOrientationPill
|
|
1129
|
+
orientation={deviceOrientation}
|
|
1130
|
+
topInset={insets.top}
|
|
1131
|
+
/>
|
|
1132
|
+
<CaptureKeyframePill
|
|
1133
|
+
state={incrementalState}
|
|
1134
|
+
topInset={insets.top}
|
|
1135
|
+
/>
|
|
1136
|
+
<CaptureMemoryPill topInset={insets.top} />
|
|
1137
|
+
<CaptureDebugOverlay
|
|
1138
|
+
incrementalState={incrementalState}
|
|
1139
|
+
imuTranslationMetres={
|
|
1140
|
+
isNonAR ? imuGate.getTranslationMetres() : null
|
|
1141
|
+
}
|
|
1142
|
+
captureSource={effectiveCaptureSource}
|
|
1143
|
+
frameSelectionMode={settings.frameSelection.mode}
|
|
1144
|
+
stitchMode={settings.stitcher.stitchMode}
|
|
1145
|
+
/>
|
|
1146
|
+
</>
|
|
1147
|
+
)}
|
|
1148
|
+
{/* Toast renders regardless of `settings.debug` — toast hook
|
|
1149
|
+
* is only ever fired from the debug-gated path, but mounting
|
|
1150
|
+
* unconditionally lets Layer-2 hosts wire their own showFor()
|
|
1151
|
+
* callers without needing a separate mount. */}
|
|
1152
|
+
<CaptureStitchStatsToast
|
|
1153
|
+
message={stitchToast.message}
|
|
1154
|
+
topInset={insets.top}
|
|
1155
|
+
/>
|
|
1156
|
+
|
|
991
1157
|
{/* Settings gear (top-right), gated on showSettingsButton. */}
|
|
992
1158
|
{showSettingsButton && (
|
|
993
1159
|
<SettingsButton
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
/**
|
|
3
|
+
* CaptureDebugOverlay — diagnostic overlay for capture sessions.
|
|
4
|
+
*
|
|
5
|
+
* Shows the live engine state in a floating pill at the top of the
|
|
6
|
+
* capture screen so operators can see:
|
|
7
|
+
*
|
|
8
|
+
* - which frame outcome the engine just emitted (accept/skip/reject)
|
|
9
|
+
* - keyframe count vs. cap (e.g. "3 / 6")
|
|
10
|
+
* - per-frame newContent fraction + overlap percent
|
|
11
|
+
* - latest processingMs (how long the gate eval took)
|
|
12
|
+
* - JS-side IMU translation accumulator (when non-AR)
|
|
13
|
+
* - JS heap usage estimate (rough — RN doesn't expose Native heap)
|
|
14
|
+
*
|
|
15
|
+
* The overlay is gated by `<Camera>`'s `settings.debug` flag. When
|
|
16
|
+
* `debug = false` the component renders null and consumes no CPU.
|
|
17
|
+
*
|
|
18
|
+
* Why a separate component (not inline in Camera.tsx)?
|
|
19
|
+
*
|
|
20
|
+
* Camera.tsx is already a 1200-line beast and the debug pill needs
|
|
21
|
+
* its own styling/layout that would distract from the main capture
|
|
22
|
+
* UX. Splitting it out keeps Camera.tsx focused and the debug
|
|
23
|
+
* surface easy to evolve independently (future F9 work — port the
|
|
24
|
+
* richer memory bubble + stitch toast from the RetaiLens host).
|
|
25
|
+
*
|
|
26
|
+
* This component is intentionally PRESENTATIONAL — all data is
|
|
27
|
+
* pushed in as props. The host (Camera.tsx) owns the
|
|
28
|
+
* subscriptions / refs / state and decides when to mount the
|
|
29
|
+
* overlay.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import React from 'react';
|
|
33
|
+
import { StyleSheet, Text, View } from 'react-native';
|
|
34
|
+
|
|
35
|
+
import type { IncrementalState } from '../stitching/incremental';
|
|
36
|
+
|
|
37
|
+
export interface CaptureDebugOverlayProps {
|
|
38
|
+
/** Latest engine state (null = no capture in progress). */
|
|
39
|
+
incrementalState: IncrementalState | null;
|
|
40
|
+
/** JS-side IMU translation accumulator in metres (non-AR mode). */
|
|
41
|
+
imuTranslationMetres?: number | null;
|
|
42
|
+
/** Capture-source label so the operator knows which gate path is live. */
|
|
43
|
+
captureSource: 'ar' | 'non-ar';
|
|
44
|
+
/** Effective frame selection mode that's running right now. */
|
|
45
|
+
frameSelectionMode: 'time-based' | 'pose-based' | 'flow-based';
|
|
46
|
+
/** Effective stitchMode setting (operator-set, before auto-resolution). */
|
|
47
|
+
stitchMode: 'auto' | 'panorama' | 'scans';
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Map the numeric `outcome` enum to a short human label. Mirrors
|
|
52
|
+
* the iOS/Android C++ enum. Hidden in production builds — only
|
|
53
|
+
* surfaced via this debug overlay.
|
|
54
|
+
*/
|
|
55
|
+
function outcomeLabel(outcome: number | undefined): string {
|
|
56
|
+
switch (outcome) {
|
|
57
|
+
case 1: return 'accept';
|
|
58
|
+
case 2: return 'reject';
|
|
59
|
+
case 3: return 'cap-hit';
|
|
60
|
+
default: return outcome == null ? '—' : String(outcome);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
export function CaptureDebugOverlay({
|
|
66
|
+
incrementalState,
|
|
67
|
+
imuTranslationMetres,
|
|
68
|
+
captureSource,
|
|
69
|
+
frameSelectionMode,
|
|
70
|
+
stitchMode,
|
|
71
|
+
}: CaptureDebugOverlayProps): React.JSX.Element {
|
|
72
|
+
const accepted = incrementalState?.acceptedCount ?? 0;
|
|
73
|
+
const cap = incrementalState?.keyframeMax ?? 0;
|
|
74
|
+
const overlap = incrementalState?.overlapPercent;
|
|
75
|
+
const proc = incrementalState?.processingMs;
|
|
76
|
+
const outcome = outcomeLabel(incrementalState?.outcome);
|
|
77
|
+
const isLandscape = incrementalState?.isLandscape;
|
|
78
|
+
const painted = incrementalState?.paintedExtent ?? 0;
|
|
79
|
+
const panTotal = incrementalState?.panExtent ?? 0;
|
|
80
|
+
const fillPct = panTotal > 0 ? Math.round((painted / panTotal) * 100) : 0;
|
|
81
|
+
|
|
82
|
+
// Translation pill is only meaningful in non-AR mode (in AR the
|
|
83
|
+
// engine's own pose is the source of truth; we don't surface the
|
|
84
|
+
// tx/ty/tz separately because the operator can't act on them).
|
|
85
|
+
const showImu = captureSource === 'non-ar' && imuTranslationMetres != null;
|
|
86
|
+
const imuCm = showImu ? (imuTranslationMetres! * 100).toFixed(1) : null;
|
|
87
|
+
|
|
88
|
+
return (
|
|
89
|
+
<View pointerEvents="none" style={styles.container}>
|
|
90
|
+
{/* Top row: mode summary */}
|
|
91
|
+
<View style={styles.row}>
|
|
92
|
+
<Text style={styles.label}>
|
|
93
|
+
{captureSource}/{frameSelectionMode}/{stitchMode}
|
|
94
|
+
</Text>
|
|
95
|
+
</View>
|
|
96
|
+
{/* Keyframes pill */}
|
|
97
|
+
<View style={styles.row}>
|
|
98
|
+
<Text style={styles.metricKey}>frames</Text>
|
|
99
|
+
<Text style={styles.metricVal}>
|
|
100
|
+
{accepted}{cap > 0 ? ` / ${cap}` : ''}
|
|
101
|
+
</Text>
|
|
102
|
+
</View>
|
|
103
|
+
{/* Outcome */}
|
|
104
|
+
<View style={styles.row}>
|
|
105
|
+
<Text style={styles.metricKey}>last</Text>
|
|
106
|
+
<Text style={styles.metricVal}>{outcome}</Text>
|
|
107
|
+
</View>
|
|
108
|
+
{/* Overlap + processing */}
|
|
109
|
+
{(overlap != null && overlap >= 0) && (
|
|
110
|
+
<View style={styles.row}>
|
|
111
|
+
<Text style={styles.metricKey}>overlap</Text>
|
|
112
|
+
<Text style={styles.metricVal}>{overlap.toFixed(0)}%</Text>
|
|
113
|
+
</View>
|
|
114
|
+
)}
|
|
115
|
+
{(proc != null && proc > 0) && (
|
|
116
|
+
<View style={styles.row}>
|
|
117
|
+
<Text style={styles.metricKey}>proc</Text>
|
|
118
|
+
<Text style={styles.metricVal}>{proc.toFixed(0)}ms</Text>
|
|
119
|
+
</View>
|
|
120
|
+
)}
|
|
121
|
+
{/* Pan progress (band overlay metric) */}
|
|
122
|
+
{panTotal > 0 && (
|
|
123
|
+
<View style={styles.row}>
|
|
124
|
+
<Text style={styles.metricKey}>pan</Text>
|
|
125
|
+
<Text style={styles.metricVal}>
|
|
126
|
+
{fillPct}% ({isLandscape ? 'L' : 'P'})
|
|
127
|
+
</Text>
|
|
128
|
+
</View>
|
|
129
|
+
)}
|
|
130
|
+
{/* IMU translation — only in non-AR mode */}
|
|
131
|
+
{showImu && (
|
|
132
|
+
<View style={styles.row}>
|
|
133
|
+
<Text style={styles.metricKey}>imuΔ</Text>
|
|
134
|
+
<Text style={styles.metricVal}>{imuCm}cm</Text>
|
|
135
|
+
</View>
|
|
136
|
+
)}
|
|
137
|
+
</View>
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const styles = StyleSheet.create({
|
|
142
|
+
container: {
|
|
143
|
+
position: 'absolute',
|
|
144
|
+
// 2026-05-22 — moved from top-left to left-middle so it doesn't
|
|
145
|
+
// collide with the orientation pill (top-left) or the keyframe
|
|
146
|
+
// pill (top-center) when all three are mounted together in
|
|
147
|
+
// <Camera>'s debug mode.
|
|
148
|
+
top: 160,
|
|
149
|
+
left: 12,
|
|
150
|
+
backgroundColor: 'rgba(0, 0, 0, 0.65)',
|
|
151
|
+
paddingHorizontal: 10,
|
|
152
|
+
paddingVertical: 6,
|
|
153
|
+
borderRadius: 8,
|
|
154
|
+
minWidth: 130,
|
|
155
|
+
},
|
|
156
|
+
row: {
|
|
157
|
+
flexDirection: 'row',
|
|
158
|
+
justifyContent: 'space-between',
|
|
159
|
+
alignItems: 'center',
|
|
160
|
+
marginVertical: 1,
|
|
161
|
+
},
|
|
162
|
+
label: {
|
|
163
|
+
color: '#fff',
|
|
164
|
+
fontSize: 11,
|
|
165
|
+
fontWeight: '600',
|
|
166
|
+
fontFamily: 'Menlo',
|
|
167
|
+
},
|
|
168
|
+
metricKey: {
|
|
169
|
+
color: '#9aa',
|
|
170
|
+
fontSize: 10,
|
|
171
|
+
fontFamily: 'Menlo',
|
|
172
|
+
marginRight: 8,
|
|
173
|
+
},
|
|
174
|
+
metricVal: {
|
|
175
|
+
color: '#fff',
|
|
176
|
+
fontSize: 11,
|
|
177
|
+
fontWeight: '600',
|
|
178
|
+
fontFamily: 'Menlo',
|
|
179
|
+
},
|
|
180
|
+
});
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
/**
|
|
3
|
+
* CaptureKeyframePill — top-center "Keyframes: N/M" diagnostic pill.
|
|
4
|
+
*
|
|
5
|
+
* Renders while a capture is in flight AND the engine is running
|
|
6
|
+
* the pose-driven / flow-driven keyframe gate (keyframeMax > 0).
|
|
7
|
+
* Hidden when the gate is disabled (time-based frame selection) or
|
|
8
|
+
* when no capture is active.
|
|
9
|
+
*
|
|
10
|
+
* Color-coded by closeness to the cap:
|
|
11
|
+
*
|
|
12
|
+
* - green N < M − 1 (plenty of budget remaining)
|
|
13
|
+
* - amber N ≥ M − 1 (last frame, or cap already hit — next
|
|
14
|
+
* accept will be rejected)
|
|
15
|
+
*
|
|
16
|
+
* Layer-2 hosts that compose their own capture UI can mount this
|
|
17
|
+
* pill directly; Layer-1 `<Camera>` mounts it automatically when
|
|
18
|
+
* `settings.debug = true`.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import React from 'react';
|
|
22
|
+
import { StyleSheet, Text, View } from 'react-native';
|
|
23
|
+
|
|
24
|
+
import type { IncrementalState } from '../stitching/incremental';
|
|
25
|
+
|
|
26
|
+
export interface CaptureKeyframePillProps {
|
|
27
|
+
/** Latest engine state. Null = capture not running. */
|
|
28
|
+
state: IncrementalState | null;
|
|
29
|
+
/** Top inset for safe-area placement. Pill pinned `topInset + 56`. */
|
|
30
|
+
topInset?: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function CaptureKeyframePill({
|
|
34
|
+
state,
|
|
35
|
+
topInset = 0,
|
|
36
|
+
}: CaptureKeyframePillProps): React.JSX.Element | null {
|
|
37
|
+
const accepted = state?.acceptedCount ?? 0;
|
|
38
|
+
const max = state?.keyframeMax ?? 0;
|
|
39
|
+
if (max <= 0) return null;
|
|
40
|
+
|
|
41
|
+
const isAmber = accepted >= max - 1;
|
|
42
|
+
|
|
43
|
+
return (
|
|
44
|
+
<View
|
|
45
|
+
pointerEvents="none"
|
|
46
|
+
style={[
|
|
47
|
+
styles.container,
|
|
48
|
+
{
|
|
49
|
+
top: topInset + 56,
|
|
50
|
+
backgroundColor: isAmber
|
|
51
|
+
? 'rgba(245, 158, 11, 0.95)'
|
|
52
|
+
: 'rgba(34, 197, 94, 0.95)',
|
|
53
|
+
},
|
|
54
|
+
]}
|
|
55
|
+
accessibilityRole="alert"
|
|
56
|
+
accessibilityLiveRegion="polite"
|
|
57
|
+
>
|
|
58
|
+
<Text style={styles.text}>{`Keyframes: ${accepted}/${max}`}</Text>
|
|
59
|
+
</View>
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const styles = StyleSheet.create({
|
|
64
|
+
container: {
|
|
65
|
+
position: 'absolute',
|
|
66
|
+
alignSelf: 'center',
|
|
67
|
+
paddingHorizontal: 14,
|
|
68
|
+
paddingVertical: 6,
|
|
69
|
+
borderRadius: 999,
|
|
70
|
+
zIndex: 100,
|
|
71
|
+
},
|
|
72
|
+
text: {
|
|
73
|
+
color: '#fff',
|
|
74
|
+
fontSize: 13,
|
|
75
|
+
fontWeight: '600',
|
|
76
|
+
},
|
|
77
|
+
});
|