react-native-image-stitcher 0.15.2 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/CHANGELOG.md +171 -1
  2. package/README.md +131 -5
  3. package/android/src/main/cpp/image_stitcher_jni.cpp +154 -13
  4. package/android/src/main/cpp/keyframe_gate_jni.cpp +15 -8
  5. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +223 -1
  6. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +220 -87
  7. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +1 -1
  8. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +7 -36
  9. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +14 -8
  10. package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +39 -1
  11. package/cpp/crop_quad.cpp +162 -0
  12. package/cpp/crop_quad.hpp +163 -0
  13. package/cpp/keyframe_gate.cpp +54 -15
  14. package/cpp/keyframe_gate.hpp +33 -0
  15. package/cpp/stitcher.cpp +1122 -132
  16. package/cpp/stitcher.hpp +62 -0
  17. package/cpp/warp_guard.hpp +212 -0
  18. package/dist/camera/Camera.d.ts +209 -12
  19. package/dist/camera/Camera.js +575 -36
  20. package/dist/camera/CameraView.js +35 -16
  21. package/dist/camera/CaptureCountdownOverlay.d.ts +70 -0
  22. package/dist/camera/CaptureCountdownOverlay.js +239 -0
  23. package/dist/camera/CaptureFrameCounterOverlay.d.ts +58 -0
  24. package/dist/camera/CaptureFrameCounterOverlay.js +153 -0
  25. package/dist/camera/CaptureMemoryPill.d.ts +24 -8
  26. package/dist/camera/CaptureMemoryPill.js +37 -12
  27. package/dist/camera/CapturePreview.js +2 -1
  28. package/dist/camera/CaptureStatusOverlay.d.ts +11 -4
  29. package/dist/camera/CaptureStatusOverlay.js +22 -5
  30. package/dist/camera/CaptureThumbnailStrip.js +2 -1
  31. package/dist/camera/LateralMotionModal.d.ts +85 -0
  32. package/dist/camera/LateralMotionModal.js +134 -0
  33. package/dist/camera/PanHowToOverlay.d.ts +76 -0
  34. package/dist/camera/PanHowToOverlay.js +222 -0
  35. package/dist/camera/PanoramaBandOverlay.d.ts +2 -1
  36. package/dist/camera/PanoramaBandOverlay.js +9 -3
  37. package/dist/camera/PanoramaSettings.d.ts +8 -6
  38. package/dist/camera/PanoramaSettings.js +19 -1
  39. package/dist/camera/PanoramaSettingsModal.js +4 -4
  40. package/dist/camera/RectCropPreview.d.ts +135 -0
  41. package/dist/camera/RectCropPreview.js +370 -0
  42. package/dist/camera/RotateToLandscapePrompt.d.ts +87 -0
  43. package/dist/camera/RotateToLandscapePrompt.js +138 -0
  44. package/dist/camera/buildPanoramaInitialSettings.d.ts +19 -2
  45. package/dist/camera/buildPanoramaInitialSettings.js +9 -0
  46. package/dist/camera/cameraErrorMessages.d.ts +30 -1
  47. package/dist/camera/cameraErrorMessages.js +26 -10
  48. package/dist/camera/cameraGuidanceCopy.d.ts +87 -0
  49. package/dist/camera/cameraGuidanceCopy.js +80 -0
  50. package/dist/camera/captureCountdown.d.ts +52 -0
  51. package/dist/camera/captureCountdown.js +76 -0
  52. package/dist/camera/captureWarnings.d.ts +90 -0
  53. package/dist/camera/captureWarnings.js +108 -0
  54. package/dist/camera/classifyStitchError.d.ts +30 -0
  55. package/dist/camera/classifyStitchError.js +42 -0
  56. package/dist/camera/cropGeometry.d.ts +136 -0
  57. package/dist/camera/cropGeometry.js +223 -0
  58. package/dist/camera/displayDecodeImageProps.d.ts +25 -0
  59. package/dist/camera/displayDecodeImageProps.js +29 -0
  60. package/dist/camera/guidanceGraphics.d.ts +58 -0
  61. package/dist/camera/guidanceGraphics.js +280 -0
  62. package/dist/camera/guidanceTokens.d.ts +54 -0
  63. package/dist/camera/guidanceTokens.js +58 -0
  64. package/dist/camera/panModeGate.d.ts +54 -0
  65. package/dist/camera/panModeGate.js +62 -0
  66. package/dist/camera/pickCaptureFormat.d.ts +71 -0
  67. package/dist/camera/pickCaptureFormat.js +85 -0
  68. package/dist/camera/stitchDebugInfo.d.ts +27 -0
  69. package/dist/camera/stitchDebugInfo.js +55 -0
  70. package/dist/camera/usePanMotion.d.ts +250 -0
  71. package/dist/camera/usePanMotion.js +451 -0
  72. package/dist/index.d.ts +24 -3
  73. package/dist/index.js +33 -2
  74. package/dist/stitching/computeInscribedRect.d.ts +40 -0
  75. package/dist/stitching/computeInscribedRect.js +55 -0
  76. package/dist/stitching/cropQuad.d.ts +78 -0
  77. package/dist/stitching/cropQuad.js +116 -0
  78. package/dist/stitching/incremental.d.ts +74 -0
  79. package/dist/stitching/useIncrementalStitcher.d.ts +7 -1
  80. package/dist/stitching/useIncrementalStitcher.js +7 -1
  81. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +154 -29
  82. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +4 -0
  83. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +15 -0
  84. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +2 -2
  85. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +48 -5
  86. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +27 -0
  87. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +211 -7
  88. package/ios/Sources/RNImageStitcher/RNSARSession.swift +25 -1
  89. package/ios/Sources/RNImageStitcher/Stitcher.swift +34 -1
  90. package/ios/Sources/RNImageStitcher/StitcherBridge.m +5 -0
  91. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +56 -0
  92. package/package.json +5 -1
  93. package/src/camera/Camera.tsx +945 -47
  94. package/src/camera/CameraView.tsx +48 -16
  95. package/src/camera/CaptureCountdownOverlay.tsx +272 -0
  96. package/src/camera/CaptureFrameCounterOverlay.tsx +197 -0
  97. package/src/camera/CaptureMemoryPill.tsx +50 -12
  98. package/src/camera/CapturePreview.tsx +5 -0
  99. package/src/camera/CaptureStatusOverlay.tsx +35 -7
  100. package/src/camera/CaptureThumbnailStrip.tsx +4 -0
  101. package/src/camera/LateralMotionModal.tsx +199 -0
  102. package/src/camera/PanHowToOverlay.tsx +246 -0
  103. package/src/camera/PanoramaBandOverlay.tsx +9 -1
  104. package/src/camera/PanoramaSettings.ts +27 -7
  105. package/src/camera/PanoramaSettingsModal.tsx +4 -4
  106. package/src/camera/RectCropPreview.tsx +638 -0
  107. package/src/camera/RotateToLandscapePrompt.tsx +188 -0
  108. package/src/camera/buildPanoramaInitialSettings.ts +30 -1
  109. package/src/camera/cameraErrorMessages.ts +39 -2
  110. package/src/camera/cameraGuidanceCopy.ts +145 -0
  111. package/src/camera/captureCountdown.ts +83 -0
  112. package/src/camera/captureWarnings.ts +190 -0
  113. package/src/camera/classifyStitchError.ts +68 -0
  114. package/src/camera/cropGeometry.ts +268 -0
  115. package/src/camera/displayDecodeImageProps.ts +25 -0
  116. package/src/camera/guidanceGraphics.tsx +347 -0
  117. package/src/camera/guidanceTokens.ts +57 -0
  118. package/src/camera/panModeGate.ts +81 -0
  119. package/src/camera/pickCaptureFormat.ts +130 -0
  120. package/src/camera/stitchDebugInfo.ts +71 -0
  121. package/src/camera/usePanMotion.ts +667 -0
  122. package/src/index.ts +66 -3
  123. package/src/stitching/computeInscribedRect.ts +81 -0
  124. package/src/stitching/cropQuad.ts +167 -0
  125. package/src/stitching/incremental.ts +74 -0
  126. package/src/stitching/useIncrementalStitcher.ts +13 -0
  127. package/android/src/main/java/io/imagestitcher/rn/TransferredNV21.kt +0 -100
  128. package/cpp/tests/CMakeLists.txt +0 -104
  129. package/cpp/tests/README.md +0 -86
  130. package/cpp/tests/keyframe_timebudget_test.cpp +0 -65
  131. package/cpp/tests/pose_test.cpp +0 -74
  132. package/cpp/tests/stitcher_frame_data_test.cpp +0 -132
  133. package/cpp/tests/stubs/jsi/jsi.h +0 -33
  134. package/cpp/tests/stubs/react-native-worklets-core/WKTJsiWorklet.h +0 -34
  135. package/cpp/tests/warp_guard_test.cpp +0 -48
  136. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +0 -190
  137. package/src/camera/__tests__/bandThumbRotation.test.ts +0 -120
  138. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +0 -160
  139. package/src/camera/__tests__/cameraErrorMessages.test.ts +0 -76
  140. package/src/camera/__tests__/homeIndicatorEdge.test.ts +0 -116
  141. package/src/camera/__tests__/lowMemDevice.test.ts +0 -52
  142. package/src/camera/__tests__/selectCaptureDevice.test.ts +0 -210
  143. package/src/camera/__tests__/useContentRotation.test.ts +0 -89
  144. package/src/camera/__tests__/useOrientationDrift.test.ts +0 -169
  145. package/src/stitching/__tests__/subscribeIncrementalState.refine.test.ts +0 -276
  146. package/src/stitching/__tests__/useStitcherWorklet.test.ts +0 -202
@@ -420,7 +420,7 @@ class IncrementalStitcher(
420
420
  // stalled scan still advances). iOS parity:
421
421
  // IncrementalStitcher.swift maxKeyframeIntervalMs block.
422
422
  val maxKfIntervalMs = configOverrides
423
- ?.getDoubleOrDefault("maxKeyframeIntervalMs", 2000.0) ?: 2000.0
423
+ ?.getDoubleOrDefault("maxKeyframeIntervalMs", 1500.0) ?: 1500.0
424
424
  keyframeGate.maxKeyframeIntervalMs = maxKfIntervalMs.coerceAtLeast(0.0)
425
425
  // 2026-05-22 (audit F5) — flow-strategy Shi-Tomasi
426
426
  // tunables. Pre-audit, Android had no JNI for these
@@ -633,7 +633,8 @@ class IncrementalStitcher(
633
633
  val wasBatchKeyframe = batchKeyframeMode
634
634
  val keyframePathsSnapshot = batchKeyframePaths.toList()
635
635
  val captureOrientationSnapshot = batchCaptureOrientation
636
- val warperTypeSnapshot = batchWarperType
636
+ // batchWarperType (settings) is superseded by the high-level warper tree
637
+ // (pickHighLevelWarper) below — kept as a field for back-compat, unused here.
637
638
  val blenderTypeSnapshot = batchBlenderType
638
639
  val seamFinderTypeSnapshot = batchSeamFinderType
639
640
  val useInscribedRectCropSnapshot = batchUseInscribedRectCrop
@@ -664,11 +665,31 @@ class IncrementalStitcher(
664
665
  // falls back to pose data only. Always non-negative.
665
666
  val imuTranslationMetres = (options.getDoubleOrDefault("imuTranslationMetres", 0.0) ?: 0.0)
666
667
  .coerceAtLeast(0.0)
668
+ // Resolve once so the dev readout gets the SAME tMeters / ratio / rRadians
669
+ // that drove the decision — and gets them even when the mode was forced
670
+ // (informative: shows what auto WOULD have picked).
671
+ val autoResolution = resolveStitchModeAuto(firstPose, lastPose, imuTranslationMetres)
667
672
  val stitchModeResolved: String = when (batchStitchMode) {
668
673
  "panorama" -> "panorama"
669
674
  "scans" -> "scans"
670
- else -> resolveStitchModeAuto(firstPose, lastPose, imuTranslationMetres)
675
+ else -> autoResolution.mode
671
676
  }
677
+ // Surface the gyro rotation + translation + decision ratio for EVERY
678
+ // capture (the forced modes skip the auto decision, but the dev preview
679
+ // still reads these to tune the panorama-vs-SCANS threshold).
680
+ val rRadiansResolved: Double = autoResolution.rRadians
681
+ val tMetersResolved: Double = autoResolution.tMeters
682
+ val decisionRatioResolved: Double = autoResolution.ratio
683
+ // 2026-06-16 — HIGH-LEVEL ACROSS THE BOARD. Pick the warper from the
684
+ // (motion, Mode A/B, zoom) tree and always run cv::Stitcher PANORAMA
685
+ // (useManualPipeline=false at the stitchSync call below). stitchModeResolved
686
+ // is now only the MOTION classifier feeding the tree + the dev readout;
687
+ // the actual stitch mode is always panorama. Zoom comes from the EXPLICIT
688
+ // lens label the user selected ('1x'|'0.5x') — the reliable signal (FOV
689
+ // from intrinsics was unreliable: multi-cam 0.5x doesn't change fx, and
690
+ // the non-AR path may supply fx=0 → FOV defaulted to 65° → never 0.5x).
691
+ val lensOpt = options.getString("lens") ?: "1x"
692
+ val highLevelWarper = pickHighLevelWarper(captureOrientationSnapshot, lensOpt)
672
693
  android.util.Log.i(
673
694
  "IncrementalStitcher",
674
695
  "finalize stitch-mode: configured=$batchStitchMode resolved=$stitchModeResolved " +
@@ -714,12 +735,13 @@ class IncrementalStitcher(
714
735
  keyframePathsSnapshot.toTypedArray(),
715
736
  outputPath,
716
737
  quality,
717
- warperTypeSnapshot,
738
+ highLevelWarper, // tree-chosen (was batchWarperType)
718
739
  blenderTypeSnapshot,
719
740
  seamFinderTypeSnapshot,
720
741
  captureOrientationSnapshot,
721
742
  useInscribedRectCropSnapshot,
722
- stitchMode = stitchModeResolved,
743
+ stitchMode = "panorama", // always high-level PANORAMA
744
+ useManualPipeline = false, // high level across the board
723
745
  )
724
746
  // 2026-05-15 (D) — dims layout from native JNI:
725
747
  // [0] width, [1] height, [2] framesRequested,
@@ -748,6 +770,35 @@ class IncrementalStitcher(
748
770
  // resolved cv::Stitcher mode so JS can surface it
749
771
  // on the output preview + debug toast.
750
772
  map.putString("stitchModeResolved", stitchModeResolved)
773
+ map.putDouble("rRadians", rRadiansResolved)
774
+ // Dev tuning readout — translation magnitude + the auto
775
+ // decision ratio that drove panorama-vs-SCANS.
776
+ map.putDouble("tMeters", tMetersResolved)
777
+ map.putDouble("decisionRatio", decisionRatioResolved)
778
+ // 2026-06-15 (iOS parity) — the exact keyframe JPEG
779
+ // paths used for this stitch, so JS can re-stitch
780
+ // them ON DEMAND via refinePanorama (the high-level
781
+ // preview tab) without enumerating the session dir.
782
+ // Camera.tsx gates that tab on this array being
783
+ // present, so without it the tab never appears on
784
+ // Android (the bug this fixes). Mirrors iOS'
785
+ // FinalizePayload "batchKeyframePaths": payload.paths.
786
+ val keyframePathsArray = Arguments.createArray()
787
+ keyframePathsSnapshot.forEach { keyframePathsArray.pushString(it) }
788
+ map.putArray("batchKeyframePaths", keyframePathsArray)
789
+ // The orientation THIS stitch baked into the output.
790
+ // The on-demand high-level re-stitch MUST pass the
791
+ // same value back through refinePanorama or the
792
+ // output comes out in raw sensor landscape (sideways)
793
+ // — refinePanorama otherwise defaults to "portrait"
794
+ // (no bake-rotation). Mirrors iOS' FinalizePayload
795
+ // "captureOrientation": payload.captureOrientation.
796
+ map.putString("captureOrientation", captureOrientationSnapshot)
797
+ // 2026-06-15 — DEV overlay parity with iOS: the stitcher's
798
+ // runtime recipe (pipe/warp/route/seam/blend) so the Android
799
+ // pill shows the same detail, not just mode/score/frames.
800
+ val dbg = stitcher.lastDebugSummary
801
+ if (dbg.isNotEmpty()) map.putString("debugSummary", dbg)
751
802
  } else {
752
803
  // The live engines (hybrid + firstwins/slit) and their
753
804
  // auto-refine hook were archived in the 2026-06 batch-
@@ -851,36 +902,13 @@ class IncrementalStitcher(
851
902
  grayHeight: Int,
852
903
  grayStride: Int,
853
904
  onAccept: (targetPath: String) -> Boolean,
854
- // 2026-05-21 (v0.3) — only required when batchKeyframeMode
855
- // is false (the legacy hybrid/firstwins live-engine path,
856
- // which feeds JPEG paths into addFrameAtPath for each ARCore
857
- // frame). Pass null when batchKeyframeMode is true; the
858
- // batch path uses `grayData` + `onAccept` instead. Modern
859
- // callers prefer `nv21PixelData` below`legacyJpegPath` is
860
- // kept only as a defensive fallback for older call sites
861
- // that have not yet been migrated.
862
- legacyJpegPath: String? = null,
863
- // F8.6 — pixel-data path for live engines. When supplied
864
- // (and `batchKeyframeMode == false`), takes precedence over
865
- // `legacyJpegPath`: the live engine ingests via
866
- // `addFramePixelData` (NV21 → BGR Mat in-process) instead of
867
- // `addFrameAtPath` (JPEG decode round-trip). Saves ~30-50 ms
868
- // per accepted frame on a mid-tier device. Pass null to use
869
- // the legacy JPEG path.
870
- //
871
- // OWNERSHIP: wrapped in `TransferredNV21` (audit #4A,
872
- // v0.10.0). The wrapper enforces single-use: the engine
873
- // calls `.takeOnce()` on the producer thread before
874
- // dispatching to `workScope`; subsequent attempts to extract
875
- // the bytes throw. Callers MUST construct a fresh
876
- // `TransferredNV21` per frame and MUST NOT hand the same
877
- // instance to two consumers (e.g., a sync gate-eval + an
878
- // async workScope.launch). The Frame Processor plugin and
879
- // the AR camera view both allocate fresh NV21 arrays per
880
- // frame; the wrapper is a defensive-programming guard.
881
- nv21PixelData: TransferredNV21? = null,
882
- nv21PixelWidth: Int = 0,
883
- nv21PixelHeight: Int = 0,
905
+ // 2026-06-16 (audit #8/L3) — the live-engine ingest params
906
+ // (legacyJpegPath / nv21PixelData / nv21PixelWidth/Height) were
907
+ // removed here. The live engines were archived in 2026-06, so the
908
+ // only remaining path is batch-keyframe (always on), which ingests via
909
+ // `grayData` + `onAccept`. The TransferredNV21 ownership wrapper had no
910
+ // live consumer (takeOnce() called nowhere verified by grep) and is
911
+ // deleted along with these params.
884
912
  ) {
885
913
  // ── V16 batch-keyframe: AR-driven path ─────────────────────
886
914
  //
@@ -1189,21 +1217,6 @@ class IncrementalStitcher(
1189
1217
  grayWidth = width,
1190
1218
  grayHeight = height,
1191
1219
  grayStride = yRowStride,
1192
- // F8.6 — pass the already-packed NV21 so the live
1193
- // engine branch (hybrid / firstwins) can ingest via
1194
- // `addFramePixelData` instead of JPEG-decoding a
1195
- // separately-written path. Batch-keyframe mode
1196
- // ignores these (it uses `grayData` + `onAccept`).
1197
- //
1198
- // v0.10.0 audit #4A — wrap in TransferredNV21 so the
1199
- // engine takes ownership exactly once on the producer
1200
- // thread (engine calls `.takeOnce()` before workScope).
1201
- // Misuse (handing this same instance to two consumers)
1202
- // throws at the second `.takeOnce()` site, not silently
1203
- // corrupting frames.
1204
- nv21PixelData = TransferredNV21(nv21Bytes),
1205
- nv21PixelWidth = width,
1206
- nv21PixelHeight = height,
1207
1220
  onAccept = { targetPath ->
1208
1221
  // Synchronous JPEG encode via the existing
1209
1222
  // YuvImageConverter (also used by RNSARCameraView's
@@ -1493,6 +1506,15 @@ class IncrementalStitcher(
1493
1506
  config?.getBooleanOrDefault("useInscribedRectCrop", false) ?: false
1494
1507
  val stitchMode = (config?.getString("stitchMode") ?: "auto")
1495
1508
  .let { if (it in setOf("auto", "panorama", "scans")) it else "auto" }
1509
+ // 2026-06-15 — pipeline is caller-selectable (mirrors iOS'
1510
+ // refinePanorama `refineManual`). The on-demand HIGH-LEVEL
1511
+ // preview tab (Camera.tsx requestHighLevelAlt) calls
1512
+ // refinePanorama with useManualPipeline:false to re-stitch the
1513
+ // captured keyframes via stock cv::Stitcher. Default false
1514
+ // (high-level) preserves the refine path's historical
1515
+ // cv::Stitcher behaviour.
1516
+ val useManualPipeline =
1517
+ config?.getBooleanOrDefault("useManualPipeline", false) ?: false
1496
1518
  val jpegQuality = max(1, min(100,
1497
1519
  config?.getIntOrDefault("jpegQuality", 90) ?: 90))
1498
1520
 
@@ -1538,9 +1560,18 @@ class IncrementalStitcher(
1538
1560
  warperType,
1539
1561
  blenderType,
1540
1562
  seamFinderType,
1563
+ // captureOrientation flows through so the high-level
1564
+ // re-stitch bakes the SAME rotation the capture used
1565
+ // — without it the output is sideways (raw sensor
1566
+ // landscape). The high-level tab passes back the
1567
+ // orientation the finalize emitted.
1541
1568
  captureOrientation,
1542
1569
  useInscribedRectCrop,
1543
1570
  stitchMode = effectiveMode,
1571
+ // false = stock high-level cv::Stitcher (the on-demand
1572
+ // HIGH-LEVEL preview tab); true would force the manual
1573
+ // pipeline. Sourced from the JS config above.
1574
+ useManualPipeline = useManualPipeline,
1544
1575
  )
1545
1576
  // Stitch returned — BatchStitcher writes the JPEG
1546
1577
  // synchronously, so "writing" reflects the final
@@ -1568,6 +1599,10 @@ class IncrementalStitcher(
1568
1599
  putInt("framesIncluded", framesIncluded)
1569
1600
  putInt("framesDropped", framesRequested - framesIncluded)
1570
1601
  putDouble("finalConfidenceThresh", finalConfidenceThresh)
1602
+ // DEV overlay — the high-level re-stitch's recipe so the
1603
+ // pill shows pipe/warp/route/seam/blend on the high-level tab.
1604
+ val dbg = stitcher.lastDebugSummary
1605
+ if (dbg.isNotEmpty()) putString("debugSummary", dbg)
1571
1606
  }
1572
1607
  emitRefineProgress(
1573
1608
  stage = "done",
@@ -1649,44 +1684,70 @@ class IncrementalStitcher(
1649
1684
  * via `task_info(TASK_VM_INFO)` — see
1650
1685
  * `IncrementalStitcherBridge.swift:231-259`).
1651
1686
  *
1652
- * Returns the **total PSS** (proportional set size) of this
1653
- * process in MB. PSS is the metric Android's Low-Memory-Killer
1654
- * (`lmkd`) ranks against, so it's the right one-true-number for
1655
- * the on-screen memory pill: it's "how close are we to being
1656
- * killed by the system?".
1687
+ * Returns the process **RSS** (resident set size) in MB, read from
1688
+ * `/proc/self/statm` (resident-pages × page-size). RSS is what the shared
1689
+ * C++ `[memstat]` lines report (`rss_mb()` reads the same `/proc`), so the
1690
+ * pill and the stitch logs show the SAME number handy when correlating a
1691
+ * spike with a logcat trace.
1657
1692
  *
1658
- * Total PSS = USS (private) + sum(shared / refcount). Read via
1659
- * `ActivityManager.getProcessMemoryInfo()`, which is the same API
1660
- * Android Studio's profiler uses. Granularity is 1 KB; we
1661
- * divide by 1024 to MB so the JS side displays a number directly
1662
- * comparable to the iOS phys_footprint value.
1693
+ * Why not `ActivityManager.getProcessMemoryInfo().totalPss`? It's
1694
+ * RATE-LIMITED on Android 8+ (returns a cached value when polled often), so
1695
+ * at the pill's 500 ms cadence it froze at the launch-time reading and never
1696
+ * moved. `/proc/self/statm` is a single unthrottled read.
1663
1697
  *
1664
- * Returns -1.0 on failure (very rare — `getProcessMemoryInfo()`
1665
- * is generally infallible since Android 5.0 because PSS is read
1666
- * from `/proc/self/smaps` synchronously on the calling thread).
1698
+ * Returns -1.0 on failure (very rare — `/proc/self/statm` is always present
1699
+ * and cheap to read on the calling thread).
1667
1700
  */
1668
1701
  @ReactMethod
1669
1702
  fun getMemoryFootprintMB(promise: Promise) {
1670
1703
  try {
1671
- val am = reactContext.getSystemService(ActivityManager::class.java)
1672
- if (am == null) {
1673
- promise.resolve(-1.0)
1674
- return
1675
- }
1676
- val pid = android.os.Process.myPid()
1677
- val infos = am.getProcessMemoryInfo(intArrayOf(pid))
1678
- if (infos == null || infos.isEmpty()) {
1704
+ // Read RSS from /proc/self/statm (field[1] = resident pages). This
1705
+ // is UNTHROTTLED and matches the C++ [memstat] rss_mb() in logcat, so
1706
+ // the pill tracks the same number I read from the stitch logs.
1707
+ //
1708
+ // 2026-06-15 — was ActivityManager.getProcessMemoryInfo().totalPss,
1709
+ // which is RATE-LIMITED on Android 8+: polled frequently (the pill
1710
+ // ticks every 500 ms) it returns a CACHED value, so the pill froze at
1711
+ // its launch-time reading (~310 MB) and never showed the stitch spike.
1712
+ val fields = java.io.File("/proc/self/statm")
1713
+ .readText().trim().split(' ')
1714
+ val residentPages = fields[1].toLong()
1715
+ val pageSize =
1716
+ android.system.Os.sysconf(android.system.OsConstants._SC_PAGESIZE)
1717
+ val mb = residentPages.toDouble() * pageSize.toDouble() /
1718
+ (1024.0 * 1024.0)
1719
+ promise.resolve(mb)
1720
+ } catch (t: Throwable) {
1721
+ android.util.Log.w(
1722
+ "IncrementalStitcher",
1723
+ "getMemoryFootprintMB: failed: ${t.message}",
1724
+ )
1725
+ promise.resolve(-1.0)
1726
+ }
1727
+ }
1728
+
1729
+ /**
1730
+ * Total physical RAM in MB. Lets the DEV memory pill derive RAM-aware
1731
+ * pressure bands instead of the iPhone-fixed 1500/2200 MB thresholds (which
1732
+ * never trip on a 4 GB Android phone that jetsams ~1.3 GB — false comfort).
1733
+ * Reads `_SC_PHYS_PAGES × _SC_PAGE_SIZE` (TOTAL + stable across runs, unlike
1734
+ * the rate-limited ActivityManager path). -1.0 on failure.
1735
+ */
1736
+ @ReactMethod
1737
+ fun getDeviceTotalRamMB(promise: Promise) {
1738
+ try {
1739
+ val pages = android.system.Os.sysconf(android.system.OsConstants._SC_PHYS_PAGES)
1740
+ val pageSize =
1741
+ android.system.Os.sysconf(android.system.OsConstants._SC_PAGESIZE)
1742
+ if (pages <= 0 || pageSize <= 0) {
1679
1743
  promise.resolve(-1.0)
1680
1744
  return
1681
1745
  }
1682
- // totalPss is in KB. Divide by 1024 → MB. Use Double so
1683
- // the JS overlay can render fractional MB if it wants.
1684
- val mb = infos[0].totalPss.toDouble() / 1024.0
1685
- promise.resolve(mb)
1746
+ promise.resolve(pages.toDouble() * pageSize.toDouble() / (1024.0 * 1024.0))
1686
1747
  } catch (t: Throwable) {
1687
1748
  android.util.Log.w(
1688
1749
  "IncrementalStitcher",
1689
- "getMemoryFootprintMB: failed: ${t.message}",
1750
+ "getDeviceTotalRamMB: failed: ${t.message}",
1690
1751
  )
1691
1752
  promise.resolve(-1.0)
1692
1753
  }
@@ -1883,6 +1944,24 @@ class IncrementalStitcher(
1883
1944
  *
1884
1945
  * Returns "panorama" or "scans" — never "auto".
1885
1946
  */
1947
+ /**
1948
+ * Result of [resolveStitchModeAuto]: the chosen mode PLUS the gyro rotation
1949
+ * magnitude that drove the decision. rRadians is surfaced to JS (the dev
1950
+ * 3-tab preview shows it) so the panorama-vs-SCANS rotation threshold can be
1951
+ * tuned from real captures. rRadians is 0.0 only on the no-pose fallbacks
1952
+ * (non-AR with no pose data) — there is no gyro-derived rotation to report.
1953
+ */
1954
+ private data class StitchModeResolution(
1955
+ val mode: String,
1956
+ val rRadians: Double,
1957
+ // tMeters = translation magnitude (m) that fed the ratio; ratio = the
1958
+ // tScore/(tScore+rScore) decision value (>=0.55 → SCANS). Surfaced to the
1959
+ // dev readout so the panorama-vs-SCANS threshold can be tuned from real
1960
+ // captures, alongside rRadians.
1961
+ val tMeters: Double,
1962
+ val ratio: Double,
1963
+ )
1964
+
1886
1965
  private fun resolveStitchModeAuto(
1887
1966
  firstPose: DoubleArray?,
1888
1967
  lastPose: DoubleArray?,
@@ -1890,14 +1969,16 @@ class IncrementalStitcher(
1890
1969
  // translation in METRES. Used as a fallback when pose-derived
1891
1970
  // translation is 0 (non-AR mode).
1892
1971
  imuTranslationMetres: Double = 0.0,
1893
- ): String {
1972
+ ): StitchModeResolution {
1894
1973
  if (firstPose == null || lastPose == null) {
1895
1974
  // No pose data at all — fall back on the IMU signal. IMU
1896
1975
  // > 5 cm hints SCANS; everything else hints PANORAMA.
1897
- return if (imuTranslationMetres > 0.05) "scans" else "panorama"
1976
+ return StitchModeResolution(
1977
+ if (imuTranslationMetres > 0.05) "scans" else "panorama", 0.0, 0.0, 0.0)
1898
1978
  }
1899
1979
  if (firstPose.size != 7 || lastPose.size != 7) {
1900
- return if (imuTranslationMetres > 0.05) "scans" else "panorama"
1980
+ return StitchModeResolution(
1981
+ if (imuTranslationMetres > 0.05) "scans" else "panorama", 0.0, 0.0, 0.0)
1901
1982
  }
1902
1983
 
1903
1984
  // Translation magnitude (Euclidean, in metres) — pose-derived.
@@ -1917,11 +1998,7 @@ class IncrementalStitcher(
1917
1998
  // conventions; rotated by the pose quaternion gives the world-
1918
1999
  // frame forward direction. Angle between the first and last
1919
2000
  // camera-forward vectors is the total rotation around any axis.
1920
- val fwdFirst = qrotForward(firstPose[3], firstPose[4], firstPose[5], firstPose[6])
1921
- val fwdLast = qrotForward(lastPose[3], lastPose[4], lastPose[5], lastPose[6])
1922
- val dot = (fwdFirst[0] * fwdLast[0] + fwdFirst[1] * fwdLast[1] + fwdFirst[2] * fwdLast[2])
1923
- .coerceIn(-1.0, 1.0)
1924
- val rRadians = kotlin.math.acos(dot)
2001
+ val rRadians = rotationRadians(firstPose, lastPose)
1925
2002
 
1926
2003
  // Normalisation: 10 cm of translation ≈ 1 rad of rotation as
1927
2004
  // "equivalent magnitude" for the ratio. Empirically: shelf
@@ -1932,18 +2009,74 @@ class IncrementalStitcher(
1932
2009
  val tScore = tMeters / 0.10
1933
2010
  val rScore = rRadians / 1.00
1934
2011
  val denom = tScore + rScore
1935
- if (denom <= 1e-9) return "panorama" // no motion either way
2012
+ if (denom <= 1e-9) return StitchModeResolution("panorama", rRadians, tMeters, 0.0) // no motion
1936
2013
  val ratio = tScore / denom
1937
2014
 
2015
+ // 2026-06-15 — LOW-ROTATION GUARD. The gyro rotation (rRadians) is
2016
+ // trustworthy; the IMU translation (tMeters, in non-AR) is NOT — a
2017
+ // continuous rotation leaks gravity into the double-integrated accel and
2018
+ // inflates it, which can falsely push `ratio` over 0.55 → SCANS, whose
2019
+ // affine warper can't represent the rotation. So when the gyro shows a
2020
+ // clear pan (> ~20°) with only modest translation, force PANORAMA
2021
+ // regardless of the (possibly-inflated) translation. Genuine shelf
2022
+ // scans (low rotation, large real translation) skip this and still
2023
+ // reach SCANS via the ratio. (Conservative: keeps the tMeters cap so a
2024
+ // genuine large-translation capture isn't forced to PANORAMA.)
2025
+ val lowRotationGuard = rRadians > 0.35 && tMeters < 0.25
2026
+ val mode = if (!lowRotationGuard && ratio >= 0.55) "scans" else "panorama"
1938
2027
  android.util.Log.i(
1939
2028
  "IncrementalStitcher",
1940
2029
  "stitch-mode auto: tPose=${"%.3f".format(tPose)}m " +
1941
2030
  "tImu=${"%.3f".format(imuTranslationMetres)}m " +
1942
2031
  "r=${"%.3f".format(rRadians)}rad " +
1943
2032
  "ratio=${"%.3f".format(ratio)} " +
1944
- "→ ${if (ratio >= 0.55) "scans" else "panorama"}",
2033
+ "rotGuard=$lowRotationGuard → $mode",
1945
2034
  )
1946
- return if (ratio >= 0.55) "scans" else "panorama"
2035
+ return StitchModeResolution(mode, rRadians, tMeters, ratio)
2036
+ }
2037
+
2038
+ /**
2039
+ * 2026-06-16 — high-level warper decision tree (the pipeline is now ALWAYS
2040
+ * high-level cv::Stitcher PANORAMA — useManualPipeline=false). Warper is a
2041
+ * pure function of (lens, pan direction); the rotation-vs-translation
2042
+ * (ex-SCANS) distinction was DROPPED as redundant — at 1x the same
2043
+ * direction-based warpers serve both, and 0.5x is always spherical. Inputs:
2044
+ * orientation = capture hold ("landscape*" = Mode A vertical pan;
2045
+ * "portrait*" = Mode B horizontal pan)
2046
+ * lens = the EXPLICIT lens the user selected ("0.5x" ultra-wide |
2047
+ * "1x" wide). Reliable zoom signal (FOV-from-intrinsics was
2048
+ * unreliable — multi-cam 0.5x reaches the ultra-wide by zoom
2049
+ * without changing fx, and the non-AR path may supply fx=0).
2050
+ *
2051
+ * 0.5x ultra-wide → spherical (bounded both axes; any pan)
2052
+ * 1x + Mode A (vertical) → plane
2053
+ * 1x + Mode B (horizontal) → cylindrical
2054
+ *
2055
+ * Quality-preferred warper; the C++ memory ladder force-falls to spherical
2056
+ * (and downscales compositingResol) under pressure.
2057
+ */
2058
+ private fun pickHighLevelWarper(
2059
+ orientation: String,
2060
+ lens: String,
2061
+ ): String {
2062
+ if (lens == "0.5x") return "spherical" // ultra-wide → always spherical
2063
+ val verticalPanModeA = orientation.startsWith("landscape")
2064
+ return if (verticalPanModeA) "plane" else "cylindrical" // 1x: A→plane, B→cylindrical
2065
+ }
2066
+
2067
+ /**
2068
+ * Gyro rotation magnitude (radians) between two 7-element poses
2069
+ * `[tx,ty,tz,qx,qy,qz,qw]` — the angle between the camera-forward vectors.
2070
+ * Returns 0.0 if either pose is missing/malformed (non-AR with no pose).
2071
+ * Shared by [resolveStitchModeAuto] and the finalize `rRadians` readout (DRY).
2072
+ */
2073
+ private fun rotationRadians(firstPose: DoubleArray?, lastPose: DoubleArray?): Double {
2074
+ if (firstPose == null || lastPose == null) return 0.0
2075
+ if (firstPose.size != 7 || lastPose.size != 7) return 0.0
2076
+ val f = qrotForward(firstPose[3], firstPose[4], firstPose[5], firstPose[6])
2077
+ val l = qrotForward(lastPose[3], lastPose[4], lastPose[5], lastPose[6])
2078
+ val dot = (f[0] * l[0] + f[1] * l[1] + f[2] * l[2]).coerceIn(-1.0, 1.0)
2079
+ return kotlin.math.acos(dot)
1947
2080
  }
1948
2081
 
1949
2082
  /**
@@ -151,7 +151,7 @@ internal class KeyframeGate : AutoCloseable {
151
151
  /// initializer below does NOT fire this setter, so the caller
152
152
  /// (IncrementalStitcher.kt) writes it explicitly at capture start
153
153
  /// to push the value into C++ (same contract as the iOS facade).
154
- var maxKeyframeIntervalMs: Double = 2000.0
154
+ var maxKeyframeIntervalMs: Double = 1500.0
155
155
  set(value) {
156
156
  field = value
157
157
  nativeSetMaxKeyframeIntervalMs(nativeHandle, value)
@@ -612,26 +612,13 @@ class RNSARCameraView @JvmOverloads constructor(
612
612
  val rotationForEncode = if (lastDisplayRotation >= 0)
613
613
  lastDisplayRotation else android.view.Surface.ROTATION_0
614
614
 
615
- // F8.6 (v0.6) the eager JPEG encode for live-engine mode
616
- // is gone. Pass the already-packed NV21 directly via
617
- // `nv21PixelData`; the engine's new `addFramePixelData`
618
- // path builds the BGR cv::Mat in-process via cvtColor,
619
- // skipping the JPEG decode round-trip downstream. In
620
- // batch-keyframe mode the engine ignores `nv21PixelData`
621
- // (it uses `grayData` + `onAccept` lazily); no behaviour
622
- // change there.
623
- //
624
- // (Was: eager JPEG encode for non-batch-keyframe modes,
625
- // written to `tmpJpegFile`, passed as `legacyJpegPath`.
626
- // See the v0.3 / F8.6 entries in CHANGELOG.md.)
627
- //
628
- // Synchronous engine ingest. The ARCore Image ownership
629
- // contract requires the engine to consume the TransferredNV21
630
- // before ARCore recycles the Image, so this runs inline. Only
631
- // ingest when the host has actively engaged capture
632
- // (`setIncrementalIngestionActive(true)`). (The v0.8.0 worklet-
633
- // runtime `runFirstParty` indirection + host-worklet fan-out
634
- // were archived in the 2026-06 batch-keyframe cleanup.)
615
+ // Batch-keyframe ingest. The gate reads the Y plane of the packed
616
+ // NV21 synchronously (grayData) and the lazy onAccept JPEG-encodes only
617
+ // accepted frames — no eager encode, no live-engine pixel-data path
618
+ // (the live engines + the TransferredNV21 ownership wrapper were removed
619
+ // in the 2026-06 cleanup; see audit #8). Runs inline so the gate read
620
+ // completes before ARCore recycles the Image. Only ingest when the host
621
+ // has actively engaged capture (`setIncrementalIngestionActive(true)`).
635
622
  if (ingestActive) {
636
623
  module.ingestFromARCameraView(
637
624
  tx = tArr[0].toDouble(),
@@ -654,22 +641,6 @@ class RNSARCameraView @JvmOverloads constructor(
654
641
  grayWidth = packed.width,
655
642
  grayHeight = packed.height,
656
643
  grayStride = packed.width,
657
- legacyJpegPath = null,
658
- // F8.6 — pixel-data path for live engines. Batch-
659
- // keyframe mode ignores these (bails earlier).
660
- //
661
- // v0.10.0 audit #4A — wrap `packed.nv21` in
662
- // TransferredNV21 so ownership is enforced at runtime.
663
- // The AR caller passes the SAME `packed.nv21` array as
664
- // both `grayData` (sync, gate-eval read) and
665
- // `nv21PixelData` (async, engine ingest). Today no race
666
- // because grayData is consumed inside evaluateWithFrame
667
- // before workScope.launch fires; the wrapper makes a
668
- // future refactor that reorders consumption fail loudly
669
- // instead of silently corrupting frames.
670
- nv21PixelData = TransferredNV21(packed.nv21),
671
- nv21PixelWidth = packed.width,
672
- nv21PixelHeight = packed.height,
673
644
  onAccept = { targetPath ->
674
645
  // Lazy JPEG encode. Runs ONLY if the C++ KeyframeGate
675
646
  // accepted the frame. Encodes from the pre-packed
@@ -860,21 +860,27 @@ class RNSARSession(reactContext: ReactApplicationContext)
860
860
  if (configs.isEmpty()) return
861
861
  fun aspect(s: android.util.Size): Float = s.width.toFloat() / s.height.toFloat()
862
862
 
863
- val matched = configs.filter {
864
- kotlin.math.abs(aspect(it.imageSize) - aspect(it.textureSize)) < 0.02f
865
- }
866
- val pool = if (matched.isNotEmpty()) matched else configs
867
- val chosen = pool.sortedWith(
863
+ // Option B (max FOV + bounded memory): prefer the 4:3-aspect
864
+ // IMAGE for full vertical sensor FOV, regardless of texture aspect
865
+ // (the A35 only pairs its 4:3 image with a 16:9 texture, so an
866
+ // aspect-MATCH filter would force 16:9 and lose that FOV). Among
867
+ // 4:3 images prefer the SMALLEST resolution — the keyframe is
868
+ // downscaled to AR_KEYFRAME_MAX_LONG_EDGE anyway, so smallest is
869
+ // closest to that budget + cheapest. Device-agnostic: any
870
+ // device's 4:3 image is chosen, then normalised by the downscale
871
+ // guard in YuvImageConverter. Trade-off: the 16:9 preview texture
872
+ // shows less than the 4:3 capture (accepted for max FOV).
873
+ val chosen = configs.sortedWith(
868
874
  compareBy<CameraConfig> { kotlin.math.abs(aspect(it.imageSize) - 4f / 3f) }
869
- .thenByDescending { it.imageSize.width * it.imageSize.height },
875
+ .thenBy { it.imageSize.width * it.imageSize.height },
870
876
  ).firstOrNull() ?: return
871
877
  session.setCameraConfig(chosen)
872
878
  Log.i(
873
879
  TAG,
874
- "selectMatchingCameraConfig: chose image=" +
880
+ "selectMatchingCameraConfig: chose 4:3-pref image=" +
875
881
  "${chosen.imageSize.width}x${chosen.imageSize.height} texture=" +
876
882
  "${chosen.textureSize.width}x${chosen.textureSize.height} " +
877
- "(from ${configs.size} configs, ${matched.size} aspect-matched)",
883
+ "(from ${configs.size} configs)",
878
884
  )
879
885
  } catch (t: Throwable) {
880
886
  Log.w(TAG, "selectMatchingCameraConfig failed; keeping default config: ${t.message}")
@@ -1,6 +1,8 @@
1
1
  // SPDX-License-Identifier: Apache-2.0
2
2
  package io.imagestitcher.rn.ar
3
3
 
4
+ import android.graphics.Bitmap
5
+ import android.graphics.BitmapFactory
4
6
  import android.graphics.ImageFormat
5
7
  import android.graphics.Rect
6
8
  import android.graphics.YuvImage
@@ -10,6 +12,14 @@ import androidx.exifinterface.media.ExifInterface
10
12
  import java.io.ByteArrayOutputStream
11
13
  import java.io.File
12
14
  import java.io.FileOutputStream
15
+ import kotlin.math.max
16
+ import kotlin.math.roundToInt
17
+
18
+ /** AR keyframe long-edge budget (px). Every device's acquired AR frame is
19
+ * downscaled to this before the keyframe JPEG is written, so the stitch
20
+ * held-set (and thus memory) is consistent across devices regardless of
21
+ * their ARCore 4:3 image resolution. Matches the non-AR keyframe size. */
22
+ private const val AR_KEYFRAME_MAX_LONG_EDGE = 640
13
23
 
14
24
  /**
15
25
  * Convert an ARCore `Image` (YUV_420_888) to a JPEG file on disk.
@@ -221,8 +231,36 @@ internal object YuvImageConverter {
221
231
  baos,
222
232
  )
223
233
  if (!ok) return null
234
+ // AR keyframe downscale guard — normalise the long edge to
235
+ // AR_KEYFRAME_MAX_LONG_EDGE so every device (whatever its ARCore 4:3
236
+ // image resolution) writes the same ~0.3 MP keyframe -> consistent
237
+ // stitch memory cross-device. Only the SAVED keyframe is scaled; the
238
+ // C++ keyframe gate already ran on the full-res Y plane upstream.
239
+ var jpegBytes = baos.toByteArray()
240
+ if (max(packed.width, packed.height) > AR_KEYFRAME_MAX_LONG_EDGE) {
241
+ val src = BitmapFactory.decodeByteArray(jpegBytes, 0, jpegBytes.size)
242
+ if (src != null) {
243
+ val scale =
244
+ AR_KEYFRAME_MAX_LONG_EDGE.toFloat() / max(src.width, src.height)
245
+ val dst = Bitmap.createScaledBitmap(
246
+ src,
247
+ (src.width * scale).roundToInt().coerceAtLeast(1),
248
+ (src.height * scale).roundToInt().coerceAtLeast(1),
249
+ true,
250
+ )
251
+ val baos2 = ByteArrayOutputStream()
252
+ dst.compress(
253
+ Bitmap.CompressFormat.JPEG,
254
+ jpegQuality.coerceIn(1, 100),
255
+ baos2,
256
+ )
257
+ jpegBytes = baos2.toByteArray()
258
+ if (dst !== src) dst.recycle()
259
+ src.recycle()
260
+ }
261
+ }
224
262
  try {
225
- FileOutputStream(File(outputPath)).use { it.write(baos.toByteArray()) }
263
+ FileOutputStream(File(outputPath)).use { it.write(jpegBytes) }
226
264
  } catch (e: Throwable) {
227
265
  return null
228
266
  }