react-native-image-stitcher 0.14.2 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +164 -0
  2. package/README.md +35 -0
  3. package/RNImageStitcher.podspec +8 -7
  4. package/android/build.gradle +0 -16
  5. package/android/src/main/cpp/CMakeLists.txt +2 -63
  6. package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
  7. package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
  8. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
  9. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
  10. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
  11. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
  12. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +129 -71
  13. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +49 -0
  14. package/cpp/keyframe_gate.cpp +82 -23
  15. package/cpp/keyframe_gate.hpp +31 -2
  16. package/cpp/stitcher.cpp +208 -28
  17. package/cpp/tests/CMakeLists.txt +18 -12
  18. package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
  19. package/cpp/tests/warp_guard_test.cpp +48 -0
  20. package/cpp/warp_guard.hpp +41 -0
  21. package/dist/camera/Camera.d.ts +31 -16
  22. package/dist/camera/Camera.js +11 -3
  23. package/dist/camera/CameraView.js +93 -3
  24. package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
  25. package/dist/camera/CaptureStitchStatsToast.js +27 -7
  26. package/dist/camera/PanoramaSettings.d.ts +10 -223
  27. package/dist/camera/PanoramaSettings.js +6 -28
  28. package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
  29. package/dist/camera/PanoramaSettingsBridge.js +3 -102
  30. package/dist/camera/PanoramaSettingsModal.js +7 -1
  31. package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
  32. package/dist/camera/buildPanoramaInitialSettings.js +4 -0
  33. package/dist/camera/cameraErrorMessages.d.ts +32 -0
  34. package/dist/camera/cameraErrorMessages.js +53 -0
  35. package/dist/camera/selectCaptureDevice.d.ts +5 -1
  36. package/dist/camera/selectCaptureDevice.js +22 -2
  37. package/dist/camera/useCapture.js +38 -0
  38. package/dist/index.d.ts +5 -8
  39. package/dist/index.js +11 -34
  40. package/dist/stitching/incremental.d.ts +1 -117
  41. package/dist/stitching/stitchVideo.d.ts +0 -35
  42. package/dist/types.d.ts +0 -87
  43. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
  44. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
  45. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
  46. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
  47. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
  48. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
  49. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
  50. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
  51. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
  52. package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +82 -7
  53. package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
  54. package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
  55. package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
  56. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
  57. package/package.json +3 -2
  58. package/src/camera/Camera.tsx +44 -23
  59. package/src/camera/CameraView.tsx +113 -4
  60. package/src/camera/CaptureStitchStatsToast.tsx +58 -14
  61. package/src/camera/PanoramaSettings.ts +16 -289
  62. package/src/camera/PanoramaSettingsBridge.ts +3 -114
  63. package/src/camera/PanoramaSettingsModal.tsx +14 -1
  64. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
  65. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
  66. package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
  67. package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
  68. package/src/camera/buildPanoramaInitialSettings.ts +17 -0
  69. package/src/camera/cameraErrorMessages.ts +84 -0
  70. package/src/camera/selectCaptureDevice.ts +28 -3
  71. package/src/camera/useCapture.ts +44 -1
  72. package/src/index.ts +11 -40
  73. package/src/stitching/incremental.ts +3 -140
  74. package/src/stitching/stitchVideo.ts +0 -26
  75. package/src/types.ts +0 -95
  76. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
  77. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
  78. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
  79. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
  80. package/cpp/stitcher_frame_jsi.cpp +0 -214
  81. package/cpp/stitcher_frame_jsi.hpp +0 -108
  82. package/cpp/stitcher_proxy_jsi.cpp +0 -109
  83. package/cpp/stitcher_proxy_jsi.hpp +0 -46
  84. package/cpp/stitcher_worklet_dispatch.cpp +0 -103
  85. package/cpp/stitcher_worklet_dispatch.hpp +0 -71
  86. package/cpp/stitcher_worklet_registry.cpp +0 -91
  87. package/cpp/stitcher_worklet_registry.hpp +0 -146
  88. package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
  89. package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
  90. package/dist/stitching/IncrementalStitcherView.js +0 -157
  91. package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
  92. package/dist/stitching/StitcherWorkletRegistry.js +0 -78
  93. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
  94. package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
  95. package/dist/stitching/useFrameProcessor.d.ts +0 -119
  96. package/dist/stitching/useFrameProcessor.js +0 -196
  97. package/dist/stitching/useFrameStream.d.ts +0 -34
  98. package/dist/stitching/useFrameStream.js +0 -234
  99. package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
  100. package/dist/stitching/useThrottledFrameProcessor.js +0 -132
  101. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
  102. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
  104. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
  105. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
  106. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
  107. package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
  108. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  109. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
  110. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
  111. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
  112. package/src/stitching/IncrementalStitcherView.tsx +0 -198
  113. package/src/stitching/StitcherWorkletRegistry.ts +0 -156
  114. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
  115. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
  116. package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
  117. package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
  118. package/src/stitching/useFrameProcessor.ts +0 -226
  119. package/src/stitching/useFrameStream.ts +0 -271
  120. package/src/stitching/useThrottledFrameProcessor.ts +0 -145
package/cpp/stitcher.cpp CHANGED
@@ -22,6 +22,7 @@
22
22
  #include <opencv2/stitching.hpp>
23
23
  #include <opencv2/stitching/detail/blenders.hpp>
24
24
  #include <opencv2/stitching/detail/camera.hpp>
25
+ #include <opencv2/stitching/detail/exposure_compensate.hpp>
25
26
  #include <opencv2/stitching/detail/matchers.hpp>
26
27
  #include <opencv2/stitching/detail/motion_estimators.hpp>
27
28
  #include <opencv2/stitching/detail/seam_finders.hpp>
@@ -39,6 +40,8 @@
39
40
  #include <unistd.h>
40
41
  #include <vector>
41
42
 
43
+ #include "warp_guard.hpp"
44
+
42
45
 
43
46
  namespace retailens {
44
47
 
@@ -146,21 +149,6 @@ cv::Mat bake_rotation(const cv::Mat& src, const std::string& orientation,
146
149
  return src;
147
150
  }
148
151
 
149
- // Crop the non-zero bounding rect from the stitched panorama. cv::
150
- // Stitcher's compose stage leaves a black border around the warped
151
- // region; we trim that here.
152
- cv::Mat crop_bbox(const cv::Mat& panorama) {
153
- cv::Mat gray;
154
- cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
155
- cv::Mat mask;
156
- cv::threshold(gray, mask, 0, 255, cv::THRESH_BINARY);
157
- cv::Rect bbox = cv::boundingRect(mask);
158
- if (bbox.width <= 0 || bbox.height <= 0) {
159
- return panorama;
160
- }
161
- return panorama(bbox).clone();
162
- }
163
-
164
152
  // Map cv::Stitcher::Status → StitchErrorCode. cv::Stitcher's enum
165
153
  // values aren't documented as ABI-stable so we don't rely on
166
154
  // numeric equality; switch through the named constants.
@@ -241,6 +229,61 @@ cv::Rect maxInscribedRectFromMask(const cv::Mat& mask) {
241
229
  return bestRect;
242
230
  }
243
231
 
232
+ // Pick the crop rectangle. Prefers the TRUE coverage mask from
233
+ // cv::Stitcher::resultMask() (0xFF where a frame painted, 0 where
234
+ // unfilled) so dark content is kept and only the never-covered wedges
235
+ // drop; falls back to a brightness mask if resultMask wasn't populated
236
+ // (older/edge OpenCV configs). `maskOut` receives the binary mask
237
+ // actually used, so the caller can crop a coverage sidecar to match.
238
+ cv::Rect choose_crop_rect(const cv::Mat& panorama,
239
+ const cv::Mat& coverage,
240
+ bool useInscribed,
241
+ const LogFn& logFn,
242
+ cv::Mat& maskOut) {
243
+ const bool haveCoverage =
244
+ (!coverage.empty() && coverage.size() == panorama.size());
245
+ cv::Mat mask;
246
+ if (haveCoverage) {
247
+ cv::Mat cov1 = coverage;
248
+ if (coverage.channels() != 1) {
249
+ cv::cvtColor(coverage, cov1, cv::COLOR_BGR2GRAY);
250
+ }
251
+ // Any painted pixel (>0) is "filled" — robust to feathered edges.
252
+ cv::threshold(cov1, mask, 0, 255, cv::THRESH_BINARY);
253
+ } else {
254
+ log_info(logFn, "[crop]",
255
+ "resultMask unusable (empty=%d size=%dx%d vs pano %dx%d) — "
256
+ "brightness-mask fallback",
257
+ coverage.empty() ? 1 : 0, coverage.cols, coverage.rows,
258
+ panorama.cols, panorama.rows);
259
+ cv::Mat gray;
260
+ cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
261
+ cv::threshold(gray, mask, 0, 255, cv::THRESH_BINARY);
262
+ }
263
+ maskOut = mask;
264
+
265
+ const cv::Rect bbox = cv::boundingRect(mask);
266
+ if (bbox.width <= 0 || bbox.height <= 0) {
267
+ return cv::Rect(0, 0, panorama.cols, panorama.rows);
268
+ }
269
+ if (!useInscribed) {
270
+ return bbox;
271
+ }
272
+ const cv::Rect inscribed = maxInscribedRectFromMask(mask);
273
+ if (inscribed.width <= 0 || inscribed.height <= 0) {
274
+ log_info(logFn, "[crop]",
275
+ "inscribed rect empty — bbox fallback (%dx%d)",
276
+ bbox.width, bbox.height);
277
+ return bbox;
278
+ }
279
+ log_info(logFn, "[crop]",
280
+ "inscribed %dx%d @ (%d,%d) via %s mask (bbox was %dx%d)",
281
+ inscribed.width, inscribed.height, inscribed.x, inscribed.y,
282
+ haveCoverage ? "coverage" : "brightness",
283
+ bbox.width, bbox.height);
284
+ return inscribed;
285
+ }
286
+
244
287
  } // namespace
245
288
 
246
289
 
@@ -526,13 +569,32 @@ static StitchResult stitchFramePathsImpl_(
526
569
  framesIncluded, framePaths.size(), finalThreshold, finalAttempt);
527
570
  log_info(logFn, "[memstat]", "phase=after_stitch rss=%.1f MB", rss_mb());
528
571
 
529
- // ── 4. Crop to non-zero bounding rect ──────────────────────────
530
- cv::Mat cropped = crop_bbox(panorama);
572
+ // ── 4. Crop (coverage-aware inscribed rect, or bbox) ───────────
573
+ // Pull cv::Stitcher's coverage mask (0xFF filled / 0 unfilled). It is
574
+ // computed during stitch(), so this is free and exact — dark content
575
+ // a frame painted is kept; only never-covered wedges drop.
576
+ cv::Mat coverage;
577
+ {
578
+ const cv::UMat rm = stitcher->resultMask();
579
+ if (!rm.empty()) {
580
+ rm.copyTo(coverage); // download UMat → Mat
581
+ }
582
+ }
583
+ cv::Mat cropMask;
584
+ const cv::Rect cropRect = choose_crop_rect(
585
+ panorama, coverage, config.useInscribedRectCrop, logFn, cropMask);
586
+ cv::Mat cropped = panorama(cropRect).clone();
587
+ // Crop the binary mask to the same rect → coverage sidecar (debug).
588
+ cv::Mat croppedCoverage;
589
+ if (cropMask.size() == panorama.size()) {
590
+ croppedCoverage = cropMask(cropRect).clone();
591
+ }
531
592
  log_info(logFn, "[dimstat]",
532
- "post-crop_bbox %dx%d → %dx%d data=%.2f MB (inscribedRect=%d, currently ignored)",
593
+ "post-crop %dx%d → %dx%d data=%.2f MB (inscribedRect=%d, coverage=%d)",
533
594
  panorama.cols, panorama.rows, cropped.cols, cropped.rows,
534
595
  mat_mb(cropped),
535
- config.useInscribedRectCrop ? 1 : 0);
596
+ config.useInscribedRectCrop ? 1 : 0,
597
+ coverage.empty() ? 0 : 1);
536
598
  log_info(logFn, "[memstat]", "phase=after_crop rss=%.1f MB", rss_mb());
537
599
 
538
600
  // ── 5. Bake rotation per capture orientation ───────────────────
@@ -565,6 +627,18 @@ static StitchResult stitchFramePathsImpl_(
565
627
  outputPath.c_str(), final_image.cols, final_image.rows);
566
628
  log_info(logFn, "[memstat]", "phase=after_imwrite rss=%.1f MB", rss_mb());
567
629
 
630
+ // Best-effort coverage sidecar (<output>.coverage.png), bake-rotated
631
+ // to align with the JPEG, for the debug harness. Never fails stitch.
632
+ if (!croppedCoverage.empty()) {
633
+ try {
634
+ const cv::Mat coverageRotated =
635
+ bake_rotation(croppedCoverage, config.captureOrientation, logFn);
636
+ cv::imwrite(outputPath + ".coverage.png", coverageRotated);
637
+ } catch (...) {
638
+ // sidecar is debug-only — ignore failures
639
+ }
640
+ }
641
+
568
642
  // ── 7. Fill the result ─────────────────────────────────────────
569
643
  const auto t1 = std::chrono::steady_clock::now();
570
644
  result.success = true;
@@ -823,6 +897,10 @@ StitchResult stitchFramePathsManual(
823
897
  // 5. CylindricalWarper warps each frame using cameras
824
898
  // 6. GraphCutSeamFinder + MultiBandBlender produce final panorama
825
899
  cv::Mat panorama;
900
+ // v0.15 — the blender's dst_mask (TRUE frame coverage) hoisted to
901
+ // outer scope so the crop + sidecar below use it instead of a
902
+ // brightness threshold (which drops dark content like a mirror).
903
+ cv::Mat coverageMask;
826
904
  // Breadcrumbs in the device console. If the next stitch
827
905
  // crashes, the last logged step pinpoints the failure point —
828
906
  // makes debugging without Xcode much faster. Prefix is
@@ -1542,6 +1620,44 @@ StitchResult stitchFramePathsManual(
1542
1620
  composeFrames.empty() ? 0 : composeFrames[0].rows,
1543
1621
  compose_scale);
1544
1622
 
1623
+ // Step 7.6: cylindrical-fallback pre-pass. The configured warper
1624
+ // (plane by default) projects as ~tan(theta), so a wide ultra-wide
1625
+ // (0.5x) sweep can blow a single frame's warp canvas past the
1626
+ // 100 MP guard and hard-fail with "degenerate camera params".
1627
+ // warpRoi() is a cheap corner projection (no pixel work), so probe
1628
+ // every frame here; if any would diverge AND we're not already on
1629
+ // the bounded cylindrical projection, fall back to cylindrical for
1630
+ // the one real warp pass below. Everything downstream (seam /
1631
+ // blender / compose / crop) consumes the warper's OUTPUTS, so the
1632
+ // swap is transparent. If even cylindrical diverges, the in-loop
1633
+ // guard (step8b) still throws — the genuine-failure safety net.
1634
+ if (config.warperType != "cylindrical" && !composeFrames.empty()) {
1635
+ bool wouldDiverge = false;
1636
+ size_t divergeFrame = 0;
1637
+ for (size_t i = 0; i < composeFrames.size(); i++) {
1638
+ if (composeFrames[i].empty()) continue;
1639
+ cv::Mat preK;
1640
+ cameras[i].K().convertTo(preK, CV_32F);
1641
+ const cv::Rect r = warper->warpRoi(
1642
+ composeFrames[i].size(), preK, cameras[i].R);
1643
+ if (warpRoiExceedsGuard(r.width, r.height)) {
1644
+ wouldDiverge = true;
1645
+ divergeFrame = i;
1646
+ break;
1647
+ }
1648
+ }
1649
+ if (wouldDiverge) {
1650
+ log_info(logFn, "[stitch-bc]",
1651
+ "step7.6: '%s' warp diverges at frame %zu (>%lld MP "
1652
+ "guard) -- falling back to cylindrical projection",
1653
+ config.warperType.c_str(), divergeFrame,
1654
+ (long long)(kMaxWarpPixels / 1000000));
1655
+ if (auto cyl = make_warper("cylindrical")) {
1656
+ warper = cyl->create(warpedScale);
1657
+ }
1658
+ }
1659
+ }
1660
+
1545
1661
  // Step 8: warp + (optional) seam finder + blender feed.
1546
1662
  //
1547
1663
  // Two paths based on caller's seamFinderType:
@@ -1645,12 +1761,14 @@ StitchResult stitchFramePathsManual(
1645
1761
  // any panorama frame requiring more than 100 MP of
1646
1762
  // intermediate storage is from a broken estimator,
1647
1763
  // not a real capture worth completing.
1648
- constexpr int64_t kMaxWarpPixels = 100ll * 1000ll * 1000ll;
1649
1764
  const int64_t roiPixels =
1650
1765
  static_cast<int64_t>(roi.width)
1651
1766
  * static_cast<int64_t>(roi.height);
1652
- if (roi.width <= 0 || roi.height <= 0
1653
- || roiPixels > kMaxWarpPixels) {
1767
+ // Final safety net. If we reach here the warper in use
1768
+ // is already cylindrical (either the host chose it, or
1769
+ // the step7.6 pre-pass fell back to it) and STILL
1770
+ // diverges — a genuinely broken estimate, so fail.
1771
+ if (warpRoiExceedsGuard(roi.width, roi.height)) {
1654
1772
  log_error(logFn, "[stitch-bc]",
1655
1773
  "step8b: warpRoi degenerate for frame "
1656
1774
  "%zu (%dx%d = %lld px > %lld limit) — "
@@ -1831,6 +1949,30 @@ StitchResult stitchFramePathsManual(
1831
1949
  }
1832
1950
  masksWarpedU_seam.clear();
1833
1951
 
1952
+ // Exposure compensation — parity with cv::Stitcher::PANORAMA,
1953
+ // which runs a GainCompensator before blending. Without it,
1954
+ // per-frame auto-exposure differences surface as brightness
1955
+ // steps at the seams. The manual path previously skipped this
1956
+ // entirely (the high-level path Android uses gets it for free),
1957
+ // which is one reason iOS output looked worse. GAIN_BLOCKS
1958
+ // matches cv::Stitcher's default compensator.
1959
+ //
1960
+ // NOTE: BATCH path only — it has every warped frame in memory,
1961
+ // which the compensator needs before it can solve gains. The
1962
+ // STREAM path (low-RAM, one frame at a time) can't feed the
1963
+ // compensator globally and keeps its current no-compensation
1964
+ // behaviour; see docs/stitch-pipeline-architecture.md.
1965
+ auto compensator = cv::detail::ExposureCompensator::createDefault(
1966
+ cv::detail::ExposureCompensator::GAIN_BLOCKS);
1967
+ {
1968
+ std::vector<cv::UMat> compImgs(N), compMasks(N);
1969
+ for (size_t i = 0; i < N; i++) {
1970
+ imagesWarped[i].copyTo(compImgs[i]);
1971
+ masksWarped[i].copyTo(compMasks[i]);
1972
+ }
1973
+ compensator->feed(corners, compImgs, compMasks);
1974
+ }
1975
+
1834
1976
  // Feed the blender, releasing each frame as we go.
1835
1977
  log_info(logFn, "[stitch-bc]", "step10a: blender->prepare");
1836
1978
  blender->prepare(corners, sizes);
@@ -1838,6 +1980,10 @@ StitchResult stitchFramePathsManual(
1838
1980
  "step10b: feeding blender (N=%zu)", N);
1839
1981
  for (size_t i = 0; i < N; i++) {
1840
1982
  log_info(logFn, "[stitch-bc]", "step10c: feed frame %zu", i);
1983
+ // Apply the per-frame exposure gain solved above, in place,
1984
+ // before converting + feeding the blender.
1985
+ compensator->apply(static_cast<int>(i), corners[i],
1986
+ imagesWarped[i], masksWarped[i]);
1841
1987
  cv::Mat imgS;
1842
1988
  imagesWarped[i].convertTo(imgS, CV_16S);
1843
1989
  blender->feed(imgS, masksWarped[i], corners[i]);
@@ -1899,6 +2045,9 @@ StitchResult stitchFramePathsManual(
1899
2045
  "step11b: blend complete (panoramaS=%dx%d)",
1900
2046
  panoramaS.cols, panoramaS.rows);
1901
2047
  panoramaS.convertTo(panorama, CV_8U);
2048
+ // Keep the blend coverage mask alive past this try scope (ref-
2049
+ // counted, so this is cheap) for the crop + sidecar below.
2050
+ coverageMask = panoramaMask;
1902
2051
  {
1903
2052
  auto _t = std::chrono::steady_clock::now();
1904
2053
  double _ms = std::chrono::duration_cast<std::chrono::milliseconds>(
@@ -2030,25 +2179,37 @@ StitchResult stitchFramePathsManual(
2030
2179
  // < 50% of bbox area, use bbox — the mask shape is pathological
2031
2180
  // and shipping bbox-with-corners is better than a sliver.
2032
2181
  cv::Mat finalImage = panorama;
2182
+ // v0.15 — coverage cropped to the same region(s) as finalImage, for
2183
+ // the debug-harness sidecar written after rotation below.
2184
+ cv::Mat coverageCropped;
2185
+ const bool haveCoverage =
2186
+ (!coverageMask.empty() && coverageMask.size() == panorama.size());
2033
2187
  try {
2034
- cv::Mat gray;
2035
- cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
2188
+ // Prefer the TRUE coverage mask (blender dst_mask): dark content a
2189
+ // frame painted is kept; only never-covered pixels drop. Fall back
2190
+ // to a brightness mask only if coverage is somehow unavailable.
2036
2191
  cv::Mat mask;
2037
- cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
2192
+ if (haveCoverage) {
2193
+ cv::threshold(coverageMask, mask, 0, 255, cv::THRESH_BINARY);
2194
+ } else {
2195
+ cv::Mat gray;
2196
+ cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
2197
+ cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
2198
+ }
2038
2199
 
2039
2200
  // V16 Phase 1b.fix5c — operator-toggleable crop strategy.
2040
2201
  //
2041
- // useInscribedRectCrop = NO (default in settings modal):
2202
+ // useInscribedRectCrop = NO (v0.15 default):
2042
2203
  // Final crop is just cv::boundingRect(mask) — preserves all
2043
2204
  // stitched content at the cost of possible black corners
2044
2205
  // where cv::Stitcher's projection didn't fill.
2045
2206
  //
2046
- // useInscribedRectCrop = YES (operator opt-in):
2207
+ // useInscribedRectCrop = YES (opt in via prop / settings modal):
2047
2208
  // Run the full inscribed-rect pipeline (morph-close + 50%
2048
2209
  // safety floor + column-projection second pass) for a clean
2049
2210
  // -cornered rectangle. Can over-aggressively shrink the
2050
2211
  // output on lopsided masks (1146×1102 bbox → 602×1102 strip
2051
- // in one field log).
2212
+ // in one field log) — which is why it's opt-in, not the default.
2052
2213
  cv::Rect bbox;
2053
2214
  if (config.useInscribedRectCrop) {
2054
2215
  cv::Mat closedMask;
@@ -2091,6 +2252,7 @@ StitchResult stitchFramePathsManual(
2091
2252
  if (bbox.width > 0 && bbox.height > 0
2092
2253
  && bbox.width <= panorama.cols && bbox.height <= panorama.rows) {
2093
2254
  finalImage = panorama(bbox).clone();
2255
+ if (haveCoverage) coverageCropped = coverageMask(bbox).clone();
2094
2256
  }
2095
2257
 
2096
2258
  // V16 Phase 1b.fix5c — column-projection second pass ALSO gated
@@ -2140,6 +2302,8 @@ StitchResult stitchFramePathsManual(
2140
2302
  cv::Rect rectCrop(cropLeft, 0,
2141
2303
  cropRight - cropLeft + 1, rows);
2142
2304
  finalImage = finalImage(rectCrop).clone();
2305
+ if (!coverageCropped.empty())
2306
+ coverageCropped = coverageCropped(rectCrop).clone();
2143
2307
  log_info(logFn, "[BatchStitcher]",
2144
2308
  "rectCrop applied: %dx%d → %dx%d",
2145
2309
  cols, rows, finalImage.cols, finalImage.rows);
@@ -2163,6 +2327,8 @@ StitchResult stitchFramePathsManual(
2163
2327
  cv::Rect rectCrop(cropLeft, 0,
2164
2328
  cropRight - cropLeft + 1, rows);
2165
2329
  finalImage = finalImage(rectCrop).clone();
2330
+ if (!coverageCropped.empty())
2331
+ coverageCropped = coverageCropped(rectCrop).clone();
2166
2332
  log_info(logFn, "[BatchStitcher]",
2167
2333
  "rectCrop relaxed applied: %dx%d → %dx%d",
2168
2334
  cols, rows, finalImage.cols, finalImage.rows);
@@ -2264,6 +2430,20 @@ StitchResult stitchFramePathsManual(
2264
2430
  config.captureOrientation,
2265
2431
  logFn);
2266
2432
 
2433
+ // v0.15 — best-effort coverage sidecar (<output>.coverage.png),
2434
+ // cropped + rotated to match the written JPEG, for the debug harness
2435
+ // (computeInscribedRect / debugMaskOverlay prefer it over brightness).
2436
+ if (!coverageCropped.empty()) {
2437
+ try {
2438
+ const cv::Mat covRot = bake_rotation(coverageCropped,
2439
+ config.captureOrientation,
2440
+ logFn);
2441
+ cv::imwrite(outputPath + ".coverage.png", covRot);
2442
+ } catch (...) {
2443
+ // sidecar is debug-only — ignore failures
2444
+ }
2445
+ }
2446
+
2267
2447
  // Encode + write the JPEG. Clamp quality into [0, 100] to defend
2268
2448
  // against caller bugs.
2269
2449
  //
@@ -11,16 +11,23 @@
11
11
  #
12
12
  # Or, from the repo root: `scripts/run-cpp-tests.sh`.
13
13
  #
14
- # What this DOES test (v0.10.0 scope):
14
+ # What this DOES test:
15
15
  # - Pure-C++ types in cpp/: `Pose`, `PlaneTransform`,
16
16
  # `StitcherFrameData`, `PixelBufferReader` interface contract.
17
- # - `StitcherWorkletRegistry` lifecycle (count, snapshot, uninstall,
18
- # thread-safety) compiled against JSI/worklets-core stubs under
19
- # `cpp/tests/stubs/`. See `stubs/jsi/jsi.h` for the strategy.
17
+ # - `timeBudgetCrossed` the keyframe gate's OpenCV-free time-budget
18
+ # force-accept predicate (keyframe_timebudget_test.cpp).
19
+ # - `warpRoiExceedsGuard` the OpenCV-free warp-canvas size guard that
20
+ # triggers the cylindrical-fallback pre-pass (warp_guard_test.cpp).
21
+ # (The StitcherWorkletRegistry test was removed when that source was
22
+ # archived in the Phase-3 cleanup — its CMake entries had gone stale
23
+ # and broke the whole suite's configure step.)
20
24
  #
21
25
  # What this DOES NOT test yet (deferred to v0.11.0+):
22
- # - `KeyframeGate` — depends on OpenCV. Will need an OpenCV-aware
23
- # CMake config (link the same opencv_world the prod build uses).
26
+ # - `KeyframeGate` (the full gate) — depends on OpenCV. Will need an
27
+ # OpenCV-aware CMake config (link the same opencv_world the prod
28
+ # build uses). NOTE: the OpenCV-free `timeBudgetCrossed` predicate
29
+ # (the time-budget force-accept boundary logic) IS covered, via
30
+ # keyframe_timebudget_test.cpp — it's an inline header-only function.
24
31
  # - JSI host-object dispatch — needs a real Hermes runtime.
25
32
  # - Anything in `stitcher.cpp` (uses OpenCV stitching pipeline).
26
33
 
@@ -66,14 +73,13 @@ include_directories(
66
73
  # Test executable
67
74
  # ─────────────────────────────────────────────────────────────────────
68
75
  add_executable(stitcher_cpp_tests
69
- # Production sources under test (only those whose deps we can satisfy
70
- # without OpenCV / a real JSI runtime).
71
- ${CMAKE_CURRENT_SOURCE_DIR}/../stitcher_worklet_registry.cpp
72
-
73
- # Test sources.
76
+ # Test sources. All subjects are header-only (Pose / PlaneTransform,
77
+ # StitcherFrameData, and the gate's inline `timeBudgetCrossed`
78
+ # predicate), so no production .cpp needs linking here.
74
79
  ${CMAKE_CURRENT_SOURCE_DIR}/pose_test.cpp
75
80
  ${CMAKE_CURRENT_SOURCE_DIR}/stitcher_frame_data_test.cpp
76
- ${CMAKE_CURRENT_SOURCE_DIR}/stitcher_worklet_registry_test.cpp
81
+ ${CMAKE_CURRENT_SOURCE_DIR}/keyframe_timebudget_test.cpp
82
+ ${CMAKE_CURRENT_SOURCE_DIR}/warp_guard_test.cpp
77
83
  )
78
84
 
79
85
  target_link_libraries(stitcher_cpp_tests
@@ -0,0 +1,65 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ //
3
+ // keyframe_timebudget_test.cpp — host unit tests for the pure
4
+ // `retailens::timeBudgetCrossed` predicate (the keyframe gate's
5
+ // time-budget force-accept decision).
6
+ //
7
+ // The full KeyframeGate depends on OpenCV and cannot run in this
8
+ // harness (see CMakeLists.txt). The time-budget boundary logic was
9
+ // therefore deliberately extracted into an inline, OpenCV-free
10
+ // predicate in keyframe_gate.hpp precisely so it CAN be unit-tested
11
+ // here without linking the gate's OpenCV-dependent .cpp.
12
+
13
+ #include <gtest/gtest.h>
14
+
15
+ #include "keyframe_gate.hpp"
16
+
17
+ using retailens::timeBudgetCrossed;
18
+
19
+ // intervalMs <= 0 disables the budget entirely (opt-out path).
20
+ TEST(TimeBudgetCrossed, DisabledWhenIntervalNonPositive) {
21
+ EXPECT_FALSE(timeBudgetCrossed(0.0, 1000, 999999));
22
+ EXPECT_FALSE(timeBudgetCrossed(-5.0, 1000, 999999));
23
+ }
24
+
25
+ // Never fires before the first accept (lastAcceptMs sentinel -1),
26
+ // regardless of how large `nowMs` is.
27
+ TEST(TimeBudgetCrossed, NeverFiresBeforeFirstAccept) {
28
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, -1, 1000000));
29
+ }
30
+
31
+ // Fires exactly at the boundary (elapsed == interval) and beyond.
32
+ TEST(TimeBudgetCrossed, FiresAtAndAfterBoundary) {
33
+ EXPECT_TRUE(timeBudgetCrossed(2000.0, 1000, 3000)); // elapsed == 2000
34
+ EXPECT_TRUE(timeBudgetCrossed(2000.0, 1000, 5000)); // elapsed > 2000
35
+ }
36
+
37
+ // Does NOT fire just under the boundary.
38
+ TEST(TimeBudgetCrossed, DoesNotFireJustUnderBoundary) {
39
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, 1000, 2999)); // elapsed == 1999
40
+ }
41
+
42
+ // A backwards or equal clock must never fire. A monotonic source
43
+ // should prevent now < lastAccept, but the predicate must be robust.
44
+ TEST(TimeBudgetCrossed, BackwardsOrEqualClockDoesNotFire) {
45
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, 5000, 4000)); // now < lastAccept
46
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, 5000, 5000)); // elapsed 0 < 2000
47
+ }
48
+
49
+ // Sub-millisecond budget must NOT collapse to "accept every frame":
50
+ // the predicate compares elapsed in double, so a 0.5 ms budget needs
51
+ // ~1 ms elapsed (not 0). Guards the truncation regression.
52
+ TEST(TimeBudgetCrossed, SubMillisecondBudgetDoesNotAcceptEveryFrame) {
53
+ EXPECT_FALSE(timeBudgetCrossed(0.5, 1000, 1000)); // elapsed 0.0 < 0.5
54
+ EXPECT_TRUE(timeBudgetCrossed(0.5, 1000, 1001)); // elapsed 1.0 >= 0.5
55
+ }
56
+
57
+ // Realistic 2 s budget across a slow pan: a keyframe accepted at t,
58
+ // the next force-accept lands at t + 2000 ms, not before.
59
+ TEST(TimeBudgetCrossed, TwoSecondBudgetTypicalUse) {
60
+ const int64_t lastAccept = 10000;
61
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, lastAccept, lastAccept + 1500));
62
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, lastAccept, lastAccept + 1999));
63
+ EXPECT_TRUE(timeBudgetCrossed(2000.0, lastAccept, lastAccept + 2000));
64
+ EXPECT_TRUE(timeBudgetCrossed(2000.0, lastAccept, lastAccept + 2001));
65
+ }
@@ -0,0 +1,48 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ /**
3
+ * Unit tests for the warp-canvas size guard (cpp/warp_guard.hpp).
4
+ *
5
+ * The guard decides when a warp ROI is "degenerate" — the trigger for
6
+ * both the cylindrical-fallback pre-pass and the in-loop final safety net.
7
+ * The cases that matter: normal ROIs pass, non-positive dims fail, the
8
+ * 100 MP boundary is inclusive, the real observed divergence (8171×12336)
9
+ * is caught, and a ROI whose int32 area would overflow is still caught.
10
+ */
11
+ #include "warp_guard.hpp"
12
+
13
+ #include <gtest/gtest.h>
14
+
15
+ using retailens::warpRoiExceedsGuard;
16
+
17
+ TEST(WarpGuard, AcceptsNormalRoi) {
18
+ EXPECT_FALSE(warpRoiExceedsGuard(4000, 2000)); // 8 MP
19
+ EXPECT_FALSE(warpRoiExceedsGuard(1, 1));
20
+ }
21
+
22
+ TEST(WarpGuard, RejectsNonPositiveDims) {
23
+ EXPECT_TRUE(warpRoiExceedsGuard(0, 1000));
24
+ EXPECT_TRUE(warpRoiExceedsGuard(1000, 0));
25
+ EXPECT_TRUE(warpRoiExceedsGuard(-5, 1000));
26
+ EXPECT_TRUE(warpRoiExceedsGuard(1000, -5));
27
+ }
28
+
29
+ TEST(WarpGuard, BoundaryIsInclusive) {
30
+ EXPECT_FALSE(warpRoiExceedsGuard(100000, 1000)); // exactly 100 MP — allowed
31
+ EXPECT_TRUE(warpRoiExceedsGuard(100000, 1001)); // 100.1 MP — over
32
+ }
33
+
34
+ TEST(WarpGuard, RejectsTheObservedDivergence) {
35
+ // 8171×12336 = 100.8 MP — the exact STITCH_CAMERA_PARAMS_FAIL canvas.
36
+ EXPECT_TRUE(warpRoiExceedsGuard(8171, 12336));
37
+ }
38
+
39
+ TEST(WarpGuard, RejectsInt32OverflowingRoi) {
40
+ // 65536×65536 = 2^32; an int32 area would wrap to 0 and slip past the
41
+ // guard. The int64 area math catches it.
42
+ EXPECT_TRUE(warpRoiExceedsGuard(65536, 65536));
43
+ }
44
+
45
+ TEST(WarpGuard, HonoursCustomThreshold) {
46
+ EXPECT_FALSE(warpRoiExceedsGuard(1000, 1000, 2'000'000)); // 1 MP < 2 MP
47
+ EXPECT_TRUE(warpRoiExceedsGuard(2000, 1000, 1'000'000)); // 2 MP > 1 MP
48
+ }
@@ -0,0 +1,41 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ #pragma once
3
+ #include <cstdint>
4
+
5
+ // ─────────────────────────────────────────────────────────────────────
6
+ // Warp-canvas size guard — shared by the warp pre-pass (which decides
7
+ // whether to fall back from a diverging plane projection to the bounded
8
+ // cylindrical one) and the in-loop final safety net in stitcher.cpp.
9
+ //
10
+ // Header-only + OpenCV-free on purpose: the predicate below is the one
11
+ // piece of warp-guard logic worth unit-testing in isolation (the int64
12
+ // overflow handling in particular), and the cpp test suite is NOT
13
+ // OpenCV-aware. Keep this file free of any cv:: dependency.
14
+ // ─────────────────────────────────────────────────────────────────────
15
+
16
+ namespace retailens {
17
+
18
+ // A single warped frame requiring more than this many pixels of
19
+ // intermediate storage is from a broken estimator (degenerate camera
20
+ // params), not a real capture: at 3-4 bytes/px that's 300-400 MB for ONE
21
+ // frame, and blending several would jetsam-OOM the app. 100 megapixels.
22
+ constexpr int64_t kMaxWarpPixels = 100LL * 1000LL * 1000LL;
23
+
24
+ // True if a warp ROI of `width`×`height` px is degenerate: non-positive
25
+ // in either dimension, or strictly larger than `maxPixels` (so a canvas
26
+ // exactly at the limit is still allowed).
27
+ //
28
+ // Computes the area in int64 so a wildly degenerate ROI (e.g.
29
+ // 65536×65536 = 2^32, whose int32 area wraps to 0) is still caught
30
+ // instead of silently slipping past the guard.
31
+ inline bool warpRoiExceedsGuard(int width, int height,
32
+ int64_t maxPixels = kMaxWarpPixels) {
33
+ if (width <= 0 || height <= 0) {
34
+ return true;
35
+ }
36
+ const int64_t pixels =
37
+ static_cast<int64_t>(width) * static_cast<int64_t>(height);
38
+ return pixels > maxPixels;
39
+ }
40
+
41
+ } // namespace retailens
@@ -144,6 +144,11 @@ export interface CameraProps {
144
144
  defaultFlowMaxTranslationCm?: number;
145
145
  defaultKeyframeMaxCount?: number;
146
146
  defaultKeyframeOverlapThreshold?: number;
147
+ /** Time-budget force-accept (ms) for the keyframe gate — accept a
148
+ * keyframe at least this often during a pan even if novelty is low,
149
+ * so slow / static pans don't leave temporal gaps. `0` disables it.
150
+ * Default 2000 (2 s). Applies to both AR and non-AR captures. */
151
+ defaultMaxKeyframeIntervalMs?: number;
147
152
  /** Forward-looking — wires through to cv::Stitcher's compositingResol
148
153
  * once PanoramaSettings exposes the field (currently a no-op). */
149
154
  defaultCompositingResolMP?: number;
@@ -151,6 +156,26 @@ export interface CameraProps {
151
156
  defaultRegistrationResolMP?: number;
152
157
  /** Forward-looking — see above. */
153
158
  defaultSeamEstimationResolMP?: number;
159
+ /**
160
+ * Crop strategy for the stitched panorama. `false` (default) keeps the
161
+ * bounding-rect of non-black pixels, which preserves all stitched
162
+ * content but may leave black corners. `true` crops to the maximum
163
+ * axis-aligned rectangle inscribed in the coverage mask — clean edges,
164
+ * no black corners (slightly more CPU at finalize) — but it can shrink
165
+ * the output substantially on lopsided / ultra-wide masks, which is why
166
+ * it's opt-in.
167
+ *
168
+ * Implemented as a start-time stitcher config (like the other
169
+ * stitcher settings), so this value is read once at mount to seed the
170
+ * initial setting; the in-app settings modal can override it at
171
+ * runtime. It changes image geometry (the crop), not encoding.
172
+ *
173
+ * Since the default is `false`, only pass this prop to opt in:
174
+ * @example
175
+ * // Crop to a clean inscribed rectangle (no black corners):
176
+ * <Camera maxInscribedRectCrop={true} />
177
+ */
178
+ maxInscribedRectCrop?: boolean;
154
179
  enablePhotoMode?: boolean;
155
180
  enablePanoramaMode?: boolean;
156
181
  showSettingsButton?: boolean;
@@ -169,23 +194,13 @@ export interface CameraProps {
169
194
  captureSources?: CaptureSourcesMode;
170
195
  style?: StyleProp<ViewStyle>;
171
196
  /**
172
- * Which incremental stitcher engine to drive. Default
173
- * `'batch-keyframe'` collects accepted JPEGs and runs
174
- * `cv::Stitcher` once at finalize time. This is the v0.4+
175
- * production default and what the v0.5 Frame Processor migration
176
- * exercises.
177
- *
178
- * Switch to a live engine (`'firstwins-rectilinear'` or
179
- * `'hybrid'`) for low-latency in-flight stitching. Live engines
180
- * exercise the F8.6 pixel-buffer ingest path (skipping the JPEG
181
- * encode/decode round-trip; ~30–50 ms saved per accept) when the
182
- * Frame Processor driver is active.
183
- *
184
- * See `docs/f8-frame-processor-plan.md` and the v0.5.0
185
- * CHANGELOG for the trade-offs between batch-keyframe and live
186
- * engines.
197
+ * Which stitcher engine to drive. Only `'batch-keyframe'` is
198
+ * supported (and the default): it collects accepted keyframe JPEGs
199
+ * during the hold-pan-release capture and runs the stitch once at
200
+ * finalize. The live engines (hybrid / slit-scan / firstwins) were
201
+ * archived in the batch-keyframe cleanup — see `archive/`.
187
202
  */
188
- engine?: 'batch-keyframe' | 'hybrid' | 'slitscan-rotate' | 'slitscan-both' | 'firstwins' | 'firstwins-zoomed' | 'firstwins-rectilinear' | 'slitscan';
203
+ engine?: 'batch-keyframe';
189
204
  /**
190
205
  * Optional destination directory for captures. When set, the lib
191
206
  * lands tap-photos at `${outputDir}/photo-${ts}.jpg` and panoramas