react-native-image-stitcher 0.2.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +511 -1
  2. package/README.md +1 -1
  3. package/android/src/main/cpp/keyframe_gate_jni.cpp +138 -0
  4. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +412 -40
  5. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +128 -0
  6. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +87 -45
  7. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +46 -4
  8. package/cpp/stitcher.cpp +101 -1
  9. package/cpp/stitcher.hpp +8 -0
  10. package/dist/camera/Camera.d.ts +9 -0
  11. package/dist/camera/Camera.js +165 -43
  12. package/dist/camera/CaptureDebugOverlay.d.ts +45 -0
  13. package/dist/camera/CaptureDebugOverlay.js +146 -0
  14. package/dist/camera/CaptureKeyframePill.d.ts +28 -0
  15. package/dist/camera/CaptureKeyframePill.js +60 -0
  16. package/dist/camera/CaptureMemoryPill.d.ts +28 -0
  17. package/dist/camera/CaptureMemoryPill.js +109 -0
  18. package/dist/camera/CaptureOrientationPill.d.ts +22 -0
  19. package/dist/camera/CaptureOrientationPill.js +44 -0
  20. package/dist/camera/CaptureStitchStatsToast.d.ts +45 -0
  21. package/dist/camera/CaptureStitchStatsToast.js +133 -0
  22. package/dist/camera/PanoramaSettings.d.ts +478 -0
  23. package/dist/camera/PanoramaSettings.js +120 -0
  24. package/dist/camera/PanoramaSettingsBridge.d.ts +84 -0
  25. package/dist/camera/PanoramaSettingsBridge.js +208 -0
  26. package/dist/camera/PanoramaSettingsModal.d.ts +50 -298
  27. package/dist/camera/PanoramaSettingsModal.js +189 -354
  28. package/dist/camera/buildPanoramaInitialSettings.d.ts +70 -0
  29. package/dist/camera/buildPanoramaInitialSettings.js +97 -0
  30. package/dist/camera/lowMemDevice.d.ts +24 -0
  31. package/dist/camera/lowMemDevice.js +69 -0
  32. package/dist/index.d.ts +16 -2
  33. package/dist/index.js +37 -2
  34. package/dist/sensors/useIMUTranslationGate.d.ts +26 -0
  35. package/dist/sensors/useIMUTranslationGate.js +83 -1
  36. package/dist/stitching/incremental.d.ts +25 -0
  37. package/dist/stitching/useIncrementalStitcher.d.ts +12 -1
  38. package/dist/stitching/useIncrementalStitcher.js +7 -1
  39. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +321 -7
  40. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +8 -0
  41. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +12 -0
  42. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +13 -0
  43. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +15 -0
  44. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +1 -0
  45. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +17 -4
  46. package/ios/Sources/RNImageStitcher/Stitcher.swift +6 -1
  47. package/package.json +6 -2
  48. package/src/camera/Camera.tsx +220 -54
  49. package/src/camera/CaptureDebugOverlay.tsx +180 -0
  50. package/src/camera/CaptureKeyframePill.tsx +77 -0
  51. package/src/camera/CaptureMemoryPill.tsx +96 -0
  52. package/src/camera/CaptureOrientationPill.tsx +57 -0
  53. package/src/camera/CaptureStitchStatsToast.tsx +155 -0
  54. package/src/camera/PanoramaSettings.ts +605 -0
  55. package/src/camera/PanoramaSettingsBridge.ts +238 -0
  56. package/src/camera/PanoramaSettingsModal.tsx +296 -988
  57. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +375 -0
  58. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +119 -0
  59. package/src/camera/__tests__/lowMemDevice.test.ts +52 -0
  60. package/src/camera/buildPanoramaInitialSettings.ts +139 -0
  61. package/src/camera/lowMemDevice.ts +71 -0
  62. package/src/index.ts +61 -3
  63. package/src/sensors/useIMUTranslationGate.ts +112 -1
  64. package/src/stitching/incremental.ts +25 -0
  65. package/src/stitching/useIncrementalStitcher.ts +18 -0
@@ -63,12 +63,20 @@ import { ARCameraView, type ARCameraViewHandle } from './ARCameraView';
63
63
  import { CameraShutter } from './CameraShutter';
64
64
  import { CameraView } from './CameraView';
65
65
  import { CaptureStatusOverlay, type CaptureStatusPhase } from './CaptureStatusOverlay';
66
+ import { CaptureDebugOverlay } from './CaptureDebugOverlay';
67
+ import { CaptureMemoryPill } from './CaptureMemoryPill';
68
+ import { CaptureKeyframePill } from './CaptureKeyframePill';
69
+ import { CaptureOrientationPill } from './CaptureOrientationPill';
70
+ import { CaptureStitchStatsToast, useStitchStatsToast } from './CaptureStitchStatsToast';
66
71
  import { PanoramaBandOverlay } from './PanoramaBandOverlay';
72
+ import { type PanoramaSettings } from './PanoramaSettings';
73
+ import { panoramaSettingsToNativeConfig } from './PanoramaSettingsBridge';
74
+ import { PanoramaSettingsModal } from './PanoramaSettingsModal';
67
75
  import {
68
- DEFAULT_PANORAMA_SETTINGS,
69
- PanoramaSettingsModal,
70
- type PanoramaSettings,
71
- } from './PanoramaSettingsModal';
76
+ buildPanoramaInitialSettings,
77
+ type PanoramaPropOverrides,
78
+ } from './buildPanoramaInitialSettings';
79
+ import { isLowMemDevice } from './lowMemDevice';
72
80
  import { useCapture } from './useCapture';
73
81
  import { useDeviceOrientation } from './useDeviceOrientation';
74
82
  import {
@@ -129,6 +137,15 @@ export type CameraCaptureResult =
129
137
  framesDropped: number;
130
138
  finalConfidenceThresh: number;
131
139
  durationMs: number;
140
+ /**
141
+ * 2026-05-22 (audit F2g) — which cv::Stitcher pipeline the
142
+ * batch finalize ran (after auto-resolution if applicable).
143
+ * Useful for displaying a "Stitched as: scans" pill on the
144
+ * output preview. Undefined when the engine wasn't
145
+ * batch-keyframe (hybrid / slit-scan don't go through
146
+ * cv::Stitcher at finalize).
147
+ */
148
+ stitchModeResolved?: 'panorama' | 'scans';
132
149
  };
133
150
 
134
151
 
@@ -457,40 +474,31 @@ function deriveEffectiveCaptureSource(
457
474
 
458
475
 
459
476
  /**
460
- * Apply per-prop defaults to build the initial settings snapshot.
461
- * The settings live in component state from there; the prop values
462
- * never re-flow.
477
+ * Pluck the props that influence the initial PanoramaSettings tree.
478
+ * Kept inline (vs. a wide structural type) so future Camera prop
479
+ * additions don't accidentally widen the settings-translation
480
+ * surface — the pure builder in `./buildPanoramaInitialSettings.ts`
481
+ * has the canonical interface; this just forwards the relevant
482
+ * fields.
463
483
  *
464
- * Note: the `default*ResolMP` props don't have a home on PanoramaSettings
465
- * yet they're accepted on the prop interface for forward compatibility
466
- * but ignored here. Wiring is a follow-up once PanoramaSettings is
467
- * extended.
484
+ * The `default*ResolMP` props on `CameraProps` are documented as
485
+ * forward-looking no-ops; the new PanoramaSettings tree has no home
486
+ * for them yet (the v0.3 audit found cv::Stitcher's resol knobs
487
+ * aren't reached by either platform's bridge). They're accepted on
488
+ * the prop interface for API stability and ignored here.
468
489
  */
469
- function buildInitialSettings(props: CameraProps): PanoramaSettings {
490
+ function extractPanoramaOverrides(props: CameraProps): PanoramaPropOverrides {
470
491
  return {
471
- ...DEFAULT_PANORAMA_SETTINGS,
472
- stitchMode: props.defaultStitchMode ?? DEFAULT_PANORAMA_SETTINGS.stitchMode,
473
- blenderType:
474
- props.defaultBlender ?? DEFAULT_PANORAMA_SETTINGS.blenderType,
475
- seamFinderType:
476
- props.defaultSeamFinder ?? DEFAULT_PANORAMA_SETTINGS.seamFinderType,
477
- warperType:
478
- props.defaultWarper ?? DEFAULT_PANORAMA_SETTINGS.warperType,
479
- flowNoveltyPercentile:
480
- props.defaultFlowNoveltyPercentile ??
481
- DEFAULT_PANORAMA_SETTINGS.flowNoveltyPercentile,
482
- flowEvalEveryNFrames:
483
- props.defaultFlowEvalEveryNFrames ??
484
- DEFAULT_PANORAMA_SETTINGS.flowEvalEveryNFrames,
485
- flowMaxTranslationCm:
486
- props.defaultFlowMaxTranslationCm ??
487
- DEFAULT_PANORAMA_SETTINGS.flowMaxTranslationCm,
488
- keyframeMaxCount:
489
- props.defaultKeyframeMaxCount ??
490
- DEFAULT_PANORAMA_SETTINGS.keyframeMaxCount,
491
- keyframeOverlapThreshold:
492
- props.defaultKeyframeOverlapThreshold ??
493
- DEFAULT_PANORAMA_SETTINGS.keyframeOverlapThreshold,
492
+ defaultCaptureSource: props.defaultCaptureSource,
493
+ defaultStitchMode: props.defaultStitchMode,
494
+ defaultBlender: props.defaultBlender,
495
+ defaultSeamFinder: props.defaultSeamFinder,
496
+ defaultWarper: props.defaultWarper,
497
+ defaultFlowNoveltyPercentile: props.defaultFlowNoveltyPercentile,
498
+ defaultFlowEvalEveryNFrames: props.defaultFlowEvalEveryNFrames,
499
+ defaultFlowMaxTranslationCm: props.defaultFlowMaxTranslationCm,
500
+ defaultKeyframeMaxCount: props.defaultKeyframeMaxCount,
501
+ defaultKeyframeOverlapThreshold: props.defaultKeyframeOverlapThreshold,
494
502
  };
495
503
  }
496
504
 
@@ -532,7 +540,10 @@ export function Camera(props: CameraProps): React.JSX.Element {
532
540
  );
533
541
  const [lens, setLens] = useState<CameraLens>(defaultLens);
534
542
  const [settings, setSettings] = useState<PanoramaSettings>(() =>
535
- buildInitialSettings(props),
543
+ buildPanoramaInitialSettings(
544
+ extractPanoramaOverrides(props),
545
+ isLowMemDevice(),
546
+ ),
536
547
  );
537
548
  const [settingsModalVisible, setSettingsModalVisible] = useState(false);
538
549
  const [statusPhase, setStatusPhase] = useState<CaptureStatusPhase>('idle');
@@ -540,6 +551,11 @@ export function Camera(props: CameraProps): React.JSX.Element {
540
551
  null,
541
552
  );
542
553
  const [incrementalState, setIncrementalState] = useState<IncrementalState | null>(null);
554
+ // 2026-05-22 (audit F9 + F3) — debug stitch-stats toast. Hook
555
+ // exposes an imperative API; we fire `showResult(finalizeResult)`
556
+ // on every successful finalize when settings.debug is on (gated
557
+ // a few hundred lines below in handleHoldEnd's onCapture branch).
558
+ const stitchToast = useStitchStatsToast();
543
559
  const [batchKeyframeThumbnails, setBatchKeyframeThumbnails] = useState<
544
560
  string[]
545
561
  >([]);
@@ -666,12 +682,27 @@ export function Camera(props: CameraProps): React.JSX.Element {
666
682
  // the C++ engine to force-accept the next frame. This is what
667
683
  // keeps non-AR captures producing keyframes at all (the flow-
668
684
  // novelty algorithm alone is too strict in practice).
685
+ //
686
+ // 2026-05-22 (audit F2f) — IMU translation gate. The gate's own
687
+ // `totalAbsMetres` accumulator (banks each segment's |displacement|
688
+ // at every anchor reset) is the right input for the finalize-time
689
+ // auto-resolver in non-AR mode (where pose-derived translation is
690
+ // 0). Pre-F2f this was reconstructed from `fires × budget +
691
+ // |residual|` — which undercounted any time a non-IMU accept
692
+ // (flow novelty, force-last) reset the integrator before the
693
+ // budget threshold was reached.
694
+ // The translation budget lives at `frameSelection.flow.maxTranslationCm`
695
+ // in the new hierarchical settings shape. When `flow` is undefined
696
+ // (the consumer opted out of the flow strategy entirely), the gate
697
+ // stays disabled — same observable behaviour as v0.3's `0` default.
698
+ const flowMaxTranslationCm =
699
+ settings.frameSelection.flow?.maxTranslationCm ?? 0;
669
700
  const imuGate = useIMUTranslationGate({
670
701
  enabled:
671
702
  isNonAR
672
703
  && statusPhase === 'recording'
673
- && settings.flowMaxTranslationCm > 0,
674
- budgetMeters: Math.max(0.001, settings.flowMaxTranslationCm / 100.0),
704
+ && flowMaxTranslationCm > 0,
705
+ budgetMeters: Math.max(0.001, flowMaxTranslationCm / 100.0),
675
706
  onBudgetExceeded: () => {
676
707
  const mod = getIncrementalNativeModule();
677
708
  mod?.markNextFrameAsLastKeyframe?.().catch(() => undefined);
@@ -718,12 +749,52 @@ export function Camera(props: CameraProps): React.JSX.Element {
718
749
  });
719
750
  return () => { sub?.remove?.(); };
720
751
  }, []);
752
+ // 2026-05-23 (race fix) — Previously this useEffect cleared
753
+ // `batchKeyframeThumbnails` + `incrementalState` when statusPhase
754
+ // transitioned to 'recording'. But handleHoldStart is async
755
+ // (`await incremental.start(...)`), and on Android the ARSession
756
+ // was already alive on the GL thread — it could emit an ACCEPT
757
+ // event during the await window, BEFORE the effect ran. Order
758
+ // observed in logcat:
759
+ // 1. setStatusPhase('recording') queued
760
+ // 2. await incremental.start() yields
761
+ // 3. ARCore frame → ingest → JS [state] emit
762
+ // 4. setBatchKeyframeThumbnails((prev=[]) => [keyframe-0.jpg])
763
+ // 5. React commits statusPhase change → THIS effect ran
764
+ // 6. setBatchKeyframeThumbnails([]) ← WIPED frame 0!
765
+ // 7. Frame 1 arrives → updater sees prev=[] → adds only frame 1
766
+ // ⇒ final array missing keyframe-0.jpg
767
+ // The reset is now done synchronously at the top of
768
+ // handleHoldStart, before any await, so the GL thread can't race
769
+ // ahead. This effect is intentionally removed.
770
+
771
+ // 2026-05-22 (audit F2f) — every accepted keyframe is a fresh
772
+ // anchor for the IMU translation gate, regardless of which
773
+ // mechanism qualified the frame (flow novelty, plane-overlap,
774
+ // angular fallback, IMU-budget force-accept, force-last). Reset
775
+ // the gate's per-segment integrator on every acceptedCount
776
+ // increment so the operator sees `imuΔ` reset to 0 in the debug
777
+ // overlay after every accept — consistent UX regardless of WHY
778
+ // the gate took the frame. Pre-F2f only the IMU-budget path
779
+ // reset the integrator; flow accepts left `posX` ticking up
780
+ // forever, which surprised the user.
781
+ //
782
+ // The gate's `totalAbsMetres` cumulative accumulator banks the
783
+ // |segment displacement| before zeroing, so finalize-time
784
+ // translation magnitude is preserved across non-IMU accepts.
785
+ const lastAcceptedCountRef = useRef(0);
721
786
  useEffect(() => {
722
- if (statusPhase === 'recording') {
723
- setBatchKeyframeThumbnails([]);
724
- setIncrementalState(null);
787
+ const accepted = incrementalState?.acceptedCount ?? 0;
788
+ if (accepted > lastAcceptedCountRef.current) {
789
+ lastAcceptedCountRef.current = accepted;
790
+ if (isNonAR) {
791
+ imuGate.resetAnchor();
792
+ }
793
+ } else if (accepted === 0) {
794
+ // New capture (state cleared) — reset our edge-detect ref.
795
+ lastAcceptedCountRef.current = 0;
725
796
  }
726
- }, [statusPhase]);
797
+ }, [incrementalState?.acceptedCount, isNonAR, imuGate]);
727
798
 
728
799
  // ── Shutter handlers ────────────────────────────────────────────
729
800
 
@@ -805,12 +876,42 @@ export function Camera(props: CameraProps): React.JSX.Element {
805
876
  return;
806
877
  }
807
878
  try {
879
+ // 2026-05-23 (race fix) — synchronously clear thumbnails +
880
+ // engine state at the top of handleHoldStart, BEFORE awaiting
881
+ // incremental.start(). In the previous effect-based design
882
+ // the GL thread could ingest an AR frame during the await
883
+ // window and add to thumbnails BEFORE React's
884
+ // statusPhase-effect ran and wiped them. See the removed
885
+ // useEffect a few hundred lines above for the full log trace.
886
+ // Synchronous reset here means any racing frame ingest sees
887
+ // an empty array and accumulates from there.
888
+ setBatchKeyframeThumbnails([]);
889
+ setIncrementalState(null);
808
890
  setStatusPhase('recording');
809
891
  setRecordingStartedAt(Date.now());
810
892
  const orientationRotation: 0 | 90 | 180 | 270 =
811
893
  deviceOrientation === 'portrait' ? 90
812
894
  : deviceOrientation === 'portrait-upside-down' ? 270
813
895
  : 0;
896
+ // v0.4 — the inline-flat config dict that v0.3 maintained here
897
+ // moved into `panoramaSettingsToNativeConfig` (see
898
+ // PanoramaSettingsBridge.ts). That adapter is the single source
899
+ // of truth for the JS→native wire format; both this call site
900
+ // AND the modal's reset-to-defaults preview agree on the same
901
+ // mapping. Audit fixes F1 / F4 / F6 from v0.3 are now properties
902
+ // of the bridge (verified by the unit tests in
903
+ // src/camera/__tests__/PanoramaSettingsBridge.test.ts).
904
+ //
905
+ // 2026-05-23 — override `captureSource` with the runtime-derived
906
+ // `effectiveCaptureSource` (from `arPreference + lens +
907
+ // AR-device-support`). Pre-this change the camera-screen AR
908
+ // toggle wrote ONLY to local `arPreference` state while the
909
+ // bridge read `settings.captureSource` — so native could think
910
+ // the capture was AR while the operator had toggled it off (or
911
+ // vice-versa). Single source of truth now: whatever camera the
912
+ // operator can see is what native is told it is. The settings
913
+ // modal's `captureSource` control has been removed for the same
914
+ // reason — see PanoramaSettingsModal.tsx for the rationale.
814
915
  await incremental.start({
815
916
  snapshotJpegQuality: 75,
816
917
  snapshotEveryNAccepts: 1,
@@ -822,18 +923,10 @@ export function Camera(props: CameraProps): React.JSX.Element {
822
923
  canvasWidth: 5000,
823
924
  canvasHeight: 5000,
824
925
  engine: 'batch-keyframe',
825
- config: {
826
- stitchMode: settings.stitchMode,
827
- warperType: settings.warperType,
828
- blenderType: settings.blenderType,
829
- seamFinderType: settings.seamFinderType,
830
- flowNoveltyPercentile: settings.flowNoveltyPercentile,
831
- flowEvalEveryNFrames: settings.flowEvalEveryNFrames,
832
- flowMaxTranslationCm: settings.flowMaxTranslationCm,
833
- keyframeMaxCount: settings.keyframeMaxCount,
834
- keyframeOverlapThreshold: settings.keyframeOverlapThreshold,
835
- frameSelectionMode: 'flow-based',
836
- },
926
+ config: panoramaSettingsToNativeConfig({
927
+ ...settings,
928
+ captureSource: effectiveCaptureSource,
929
+ }),
837
930
  });
838
931
  imuGate.resetAnchor();
839
932
  // Start pumping vision-camera snapshots into the engine for
@@ -861,6 +954,7 @@ export function Camera(props: CameraProps): React.JSX.Element {
861
954
  isNonAR,
862
955
  deviceOrientation,
863
956
  settings,
957
+ effectiveCaptureSource,
864
958
  imuGate,
865
959
  jsDriver,
866
960
  onError,
@@ -882,10 +976,19 @@ export function Camera(props: CameraProps): React.JSX.Element {
882
976
  const panoOutputPath = outputDir
883
977
  ? `${toBareFilePath(outputDir).replace(/\/$/, '')}/${defaultPanoramaFilename()}`
884
978
  : `${await getDefaultCaptureDir()}/${defaultPanoramaFilename()}`;
979
+ // 2026-05-22 (audit F2f) — total IMU translation directly from
980
+ // the gate's cumulative accumulator (banks |segment displacement|
981
+ // at every anchor reset, including non-IMU-driven resets like
982
+ // flow-novelty accepts). No more fires × budget + residual
983
+ // reconstruction. Only meaningful in non-AR mode (in AR the
984
+ // native side uses pose-derived translation and ignores this).
985
+ const imuTotalTranslationM =
986
+ isNonAR ? imuGate.getTotalAbsMetres() : 0;
885
987
  const result = await incremental.finalize(
886
988
  panoOutputPath,
887
989
  90,
888
990
  deviceOrientation,
991
+ imuTotalTranslationM,
889
992
  );
890
993
  if (
891
994
  typeof result.framesRequested === 'number'
@@ -910,7 +1013,15 @@ export function Camera(props: CameraProps): React.JSX.Element {
910
1013
  (result.framesRequested ?? 0) - (result.framesIncluded ?? 0),
911
1014
  finalConfidenceThresh: result.finalConfidenceThresh ?? -1,
912
1015
  durationMs: Date.now() - (recordingStartedAt ?? Date.now()),
1016
+ stitchModeResolved: result.stitchModeResolved,
913
1017
  });
1018
+ // 2026-05-22 (audit F9) — fire the debug stitch-stats toast on
1019
+ // every successful finalize when settings.debug is on. Shows
1020
+ // the leaveBiggestComponent retry telemetry + resolved mode so
1021
+ // the operator can see what choice the auto-resolver made.
1022
+ if (settings.debug) {
1023
+ stitchToast.showResult(result);
1024
+ }
914
1025
  } catch (err) {
915
1026
  const message = err instanceof Error ? err.message : String(err);
916
1027
  const code: CameraErrorCode =
@@ -933,6 +1044,18 @@ export function Camera(props: CameraProps): React.JSX.Element {
933
1044
  onError,
934
1045
  recordingStartedAt,
935
1046
  jsDriver,
1047
+ // F10 Phase 2 review N1 — these four were missing pre-fix. The
1048
+ // callback reads `settings.debug` (to gate the stitchToast),
1049
+ // `isNonAR` (to decide whether to read IMU totalAbs translation),
1050
+ // `imuGate` (the read itself), and `stitchToast` (the toast hook
1051
+ // object). If any of those identities change between the user
1052
+ // pressing-and-holding the shutter and the release, the stale-
1053
+ // closure read could disagree with the actual current state.
1054
+ // Pre-existing v0.3 bug; v0.4 was the natural time to address it.
1055
+ settings,
1056
+ isNonAR,
1057
+ imuGate,
1058
+ stitchToast,
936
1059
  ]);
937
1060
 
938
1061
  // ── Lens / AR-toggle handlers ───────────────────────────────────
@@ -988,6 +1111,49 @@ export function Camera(props: CameraProps): React.JSX.Element {
988
1111
  recordingStartedAt={recordingStartedAt ?? undefined}
989
1112
  />
990
1113
 
1114
+ {/*
1115
+ 2026-05-22 (audit F9 + F3) — debug UI suite, all gated on
1116
+ settings.debug. Mounts in <Camera> automatically; Layer-2
1117
+ hosts can import the individual components from the public
1118
+ API and compose their own debug surface. Layout:
1119
+ - top-left: orientation pill (purple)
1120
+ - top-center: keyframes pill (green/amber)
1121
+ - top-right: memory pill (green/amber/red)
1122
+ - top-center: stitch-stats toast (dark capsule, transient)
1123
+ - left-mid: detailed metrics block (overlap, processing,
1124
+ imuΔ, etc.) — uses CaptureDebugOverlay
1125
+ */}
1126
+ {settings.debug && (
1127
+ <>
1128
+ <CaptureOrientationPill
1129
+ orientation={deviceOrientation}
1130
+ topInset={insets.top}
1131
+ />
1132
+ <CaptureKeyframePill
1133
+ state={incrementalState}
1134
+ topInset={insets.top}
1135
+ />
1136
+ <CaptureMemoryPill topInset={insets.top} />
1137
+ <CaptureDebugOverlay
1138
+ incrementalState={incrementalState}
1139
+ imuTranslationMetres={
1140
+ isNonAR ? imuGate.getTranslationMetres() : null
1141
+ }
1142
+ captureSource={effectiveCaptureSource}
1143
+ frameSelectionMode={settings.frameSelection.mode}
1144
+ stitchMode={settings.stitcher.stitchMode}
1145
+ />
1146
+ </>
1147
+ )}
1148
+ {/* Toast renders regardless of `settings.debug` — toast hook
1149
+ * is only ever fired from the debug-gated path, but mounting
1150
+ * unconditionally lets Layer-2 hosts wire their own showFor()
1151
+ * callers without needing a separate mount. */}
1152
+ <CaptureStitchStatsToast
1153
+ message={stitchToast.message}
1154
+ topInset={insets.top}
1155
+ />
1156
+
991
1157
  {/* Settings gear (top-right), gated on showSettingsButton. */}
992
1158
  {showSettingsButton && (
993
1159
  <SettingsButton
@@ -0,0 +1,180 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ /**
3
+ * CaptureDebugOverlay — diagnostic overlay for capture sessions.
4
+ *
5
+ * Shows the live engine state in a floating pill at the top of the
6
+ * capture screen so operators can see:
7
+ *
8
+ * - which frame outcome the engine just emitted (accept/skip/reject)
9
+ * - keyframe count vs. cap (e.g. "3 / 6")
10
+ * - per-frame newContent fraction + overlap percent
11
+ * - latest processingMs (how long the gate eval took)
12
+ * - JS-side IMU translation accumulator (when non-AR)
13
+ * - JS heap usage estimate (rough — RN doesn't expose Native heap)
14
+ *
15
+ * The overlay is gated by `<Camera>`'s `settings.debug` flag. When
16
+ * `debug = false` the component renders null and consumes no CPU.
17
+ *
18
+ * Why a separate component (not inline in Camera.tsx)?
19
+ *
20
+ * Camera.tsx is already a 1200-line beast and the debug pill needs
21
+ * its own styling/layout that would distract from the main capture
22
+ * UX. Splitting it out keeps Camera.tsx focused and the debug
23
+ * surface easy to evolve independently (future F9 work — port the
24
+ * richer memory bubble + stitch toast from the RetaiLens host).
25
+ *
26
+ * This component is intentionally PRESENTATIONAL — all data is
27
+ * pushed in as props. The host (Camera.tsx) owns the
28
+ * subscriptions / refs / state and decides when to mount the
29
+ * overlay.
30
+ */
31
+
32
+ import React from 'react';
33
+ import { StyleSheet, Text, View } from 'react-native';
34
+
35
+ import type { IncrementalState } from '../stitching/incremental';
36
+
37
+ export interface CaptureDebugOverlayProps {
38
+ /** Latest engine state (null = no capture in progress). */
39
+ incrementalState: IncrementalState | null;
40
+ /** JS-side IMU translation accumulator in metres (non-AR mode). */
41
+ imuTranslationMetres?: number | null;
42
+ /** Capture-source label so the operator knows which gate path is live. */
43
+ captureSource: 'ar' | 'non-ar';
44
+ /** Effective frame selection mode that's running right now. */
45
+ frameSelectionMode: 'time-based' | 'pose-based' | 'flow-based';
46
+ /** Effective stitchMode setting (operator-set, before auto-resolution). */
47
+ stitchMode: 'auto' | 'panorama' | 'scans';
48
+ }
49
+
50
+ /**
51
+ * Map the numeric `outcome` enum to a short human label. Mirrors
52
+ * the iOS/Android C++ enum. Hidden in production builds — only
53
+ * surfaced via this debug overlay.
54
+ */
55
+ function outcomeLabel(outcome: number | undefined): string {
56
+ switch (outcome) {
57
+ case 1: return 'accept';
58
+ case 2: return 'reject';
59
+ case 3: return 'cap-hit';
60
+ default: return outcome == null ? '—' : String(outcome);
61
+ }
62
+ }
63
+
64
+
65
+ export function CaptureDebugOverlay({
66
+ incrementalState,
67
+ imuTranslationMetres,
68
+ captureSource,
69
+ frameSelectionMode,
70
+ stitchMode,
71
+ }: CaptureDebugOverlayProps): React.JSX.Element {
72
+ const accepted = incrementalState?.acceptedCount ?? 0;
73
+ const cap = incrementalState?.keyframeMax ?? 0;
74
+ const overlap = incrementalState?.overlapPercent;
75
+ const proc = incrementalState?.processingMs;
76
+ const outcome = outcomeLabel(incrementalState?.outcome);
77
+ const isLandscape = incrementalState?.isLandscape;
78
+ const painted = incrementalState?.paintedExtent ?? 0;
79
+ const panTotal = incrementalState?.panExtent ?? 0;
80
+ const fillPct = panTotal > 0 ? Math.round((painted / panTotal) * 100) : 0;
81
+
82
+ // Translation pill is only meaningful in non-AR mode (in AR the
83
+ // engine's own pose is the source of truth; we don't surface the
84
+ // tx/ty/tz separately because the operator can't act on them).
85
+ const showImu = captureSource === 'non-ar' && imuTranslationMetres != null;
86
+ const imuCm = showImu ? (imuTranslationMetres! * 100).toFixed(1) : null;
87
+
88
+ return (
89
+ <View pointerEvents="none" style={styles.container}>
90
+ {/* Top row: mode summary */}
91
+ <View style={styles.row}>
92
+ <Text style={styles.label}>
93
+ {captureSource}/{frameSelectionMode}/{stitchMode}
94
+ </Text>
95
+ </View>
96
+ {/* Keyframes pill */}
97
+ <View style={styles.row}>
98
+ <Text style={styles.metricKey}>frames</Text>
99
+ <Text style={styles.metricVal}>
100
+ {accepted}{cap > 0 ? ` / ${cap}` : ''}
101
+ </Text>
102
+ </View>
103
+ {/* Outcome */}
104
+ <View style={styles.row}>
105
+ <Text style={styles.metricKey}>last</Text>
106
+ <Text style={styles.metricVal}>{outcome}</Text>
107
+ </View>
108
+ {/* Overlap + processing */}
109
+ {(overlap != null && overlap >= 0) && (
110
+ <View style={styles.row}>
111
+ <Text style={styles.metricKey}>overlap</Text>
112
+ <Text style={styles.metricVal}>{overlap.toFixed(0)}%</Text>
113
+ </View>
114
+ )}
115
+ {(proc != null && proc > 0) && (
116
+ <View style={styles.row}>
117
+ <Text style={styles.metricKey}>proc</Text>
118
+ <Text style={styles.metricVal}>{proc.toFixed(0)}ms</Text>
119
+ </View>
120
+ )}
121
+ {/* Pan progress (band overlay metric) */}
122
+ {panTotal > 0 && (
123
+ <View style={styles.row}>
124
+ <Text style={styles.metricKey}>pan</Text>
125
+ <Text style={styles.metricVal}>
126
+ {fillPct}% ({isLandscape ? 'L' : 'P'})
127
+ </Text>
128
+ </View>
129
+ )}
130
+ {/* IMU translation — only in non-AR mode */}
131
+ {showImu && (
132
+ <View style={styles.row}>
133
+ <Text style={styles.metricKey}>imuΔ</Text>
134
+ <Text style={styles.metricVal}>{imuCm}cm</Text>
135
+ </View>
136
+ )}
137
+ </View>
138
+ );
139
+ }
140
+
141
+ const styles = StyleSheet.create({
142
+ container: {
143
+ position: 'absolute',
144
+ // 2026-05-22 — moved from top-left to left-middle so it doesn't
145
+ // collide with the orientation pill (top-left) or the keyframe
146
+ // pill (top-center) when all three are mounted together in
147
+ // <Camera>'s debug mode.
148
+ top: 160,
149
+ left: 12,
150
+ backgroundColor: 'rgba(0, 0, 0, 0.65)',
151
+ paddingHorizontal: 10,
152
+ paddingVertical: 6,
153
+ borderRadius: 8,
154
+ minWidth: 130,
155
+ },
156
+ row: {
157
+ flexDirection: 'row',
158
+ justifyContent: 'space-between',
159
+ alignItems: 'center',
160
+ marginVertical: 1,
161
+ },
162
+ label: {
163
+ color: '#fff',
164
+ fontSize: 11,
165
+ fontWeight: '600',
166
+ fontFamily: 'Menlo',
167
+ },
168
+ metricKey: {
169
+ color: '#9aa',
170
+ fontSize: 10,
171
+ fontFamily: 'Menlo',
172
+ marginRight: 8,
173
+ },
174
+ metricVal: {
175
+ color: '#fff',
176
+ fontSize: 11,
177
+ fontWeight: '600',
178
+ fontFamily: 'Menlo',
179
+ },
180
+ });
@@ -0,0 +1,77 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ /**
3
+ * CaptureKeyframePill — top-center "Keyframes: N/M" diagnostic pill.
4
+ *
5
+ * Renders while a capture is in flight AND the engine is running
6
+ * the pose-driven / flow-driven keyframe gate (keyframeMax > 0).
7
+ * Hidden when the gate is disabled (time-based frame selection) or
8
+ * when no capture is active.
9
+ *
10
+ * Color-coded by closeness to the cap:
11
+ *
12
+ * - green N < M − 1 (plenty of budget remaining)
13
+ * - amber N ≥ M − 1 (last frame, or cap already hit — next
14
+ * accept will be rejected)
15
+ *
16
+ * Layer-2 hosts that compose their own capture UI can mount this
17
+ * pill directly; Layer-1 `<Camera>` mounts it automatically when
18
+ * `settings.debug = true`.
19
+ */
20
+
21
+ import React from 'react';
22
+ import { StyleSheet, Text, View } from 'react-native';
23
+
24
+ import type { IncrementalState } from '../stitching/incremental';
25
+
26
+ export interface CaptureKeyframePillProps {
27
+ /** Latest engine state. Null = capture not running. */
28
+ state: IncrementalState | null;
29
+ /** Top inset for safe-area placement. Pill pinned `topInset + 56`. */
30
+ topInset?: number;
31
+ }
32
+
33
+ export function CaptureKeyframePill({
34
+ state,
35
+ topInset = 0,
36
+ }: CaptureKeyframePillProps): React.JSX.Element | null {
37
+ const accepted = state?.acceptedCount ?? 0;
38
+ const max = state?.keyframeMax ?? 0;
39
+ if (max <= 0) return null;
40
+
41
+ const isAmber = accepted >= max - 1;
42
+
43
+ return (
44
+ <View
45
+ pointerEvents="none"
46
+ style={[
47
+ styles.container,
48
+ {
49
+ top: topInset + 56,
50
+ backgroundColor: isAmber
51
+ ? 'rgba(245, 158, 11, 0.95)'
52
+ : 'rgba(34, 197, 94, 0.95)',
53
+ },
54
+ ]}
55
+ accessibilityRole="alert"
56
+ accessibilityLiveRegion="polite"
57
+ >
58
+ <Text style={styles.text}>{`Keyframes: ${accepted}/${max}`}</Text>
59
+ </View>
60
+ );
61
+ }
62
+
63
+ const styles = StyleSheet.create({
64
+ container: {
65
+ position: 'absolute',
66
+ alignSelf: 'center',
67
+ paddingHorizontal: 14,
68
+ paddingVertical: 6,
69
+ borderRadius: 999,
70
+ zIndex: 100,
71
+ },
72
+ text: {
73
+ color: '#fff',
74
+ fontSize: 13,
75
+ fontWeight: '600',
76
+ },
77
+ });