react-native-image-stitcher 0.14.1 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +160 -0
  2. package/README.md +35 -0
  3. package/RNImageStitcher.podspec +8 -1
  4. package/android/build.gradle +0 -16
  5. package/android/src/main/cpp/CMakeLists.txt +2 -63
  6. package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
  7. package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
  8. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
  9. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
  10. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
  11. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
  12. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
  13. package/cpp/keyframe_gate.cpp +82 -23
  14. package/cpp/keyframe_gate.hpp +31 -2
  15. package/cpp/stitcher.cpp +208 -28
  16. package/cpp/tests/CMakeLists.txt +18 -12
  17. package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
  18. package/cpp/tests/warp_guard_test.cpp +48 -0
  19. package/cpp/warp_guard.hpp +41 -0
  20. package/dist/ar/useARSession.d.ts +9 -0
  21. package/dist/ar/useARSession.js +24 -2
  22. package/dist/camera/Camera.d.ts +31 -16
  23. package/dist/camera/Camera.js +27 -4
  24. package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
  25. package/dist/camera/CaptureStitchStatsToast.js +27 -7
  26. package/dist/camera/PanoramaSettings.d.ts +10 -223
  27. package/dist/camera/PanoramaSettings.js +6 -28
  28. package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
  29. package/dist/camera/PanoramaSettingsBridge.js +3 -102
  30. package/dist/camera/PanoramaSettingsModal.js +7 -1
  31. package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
  32. package/dist/camera/buildPanoramaInitialSettings.js +4 -0
  33. package/dist/camera/cameraErrorMessages.d.ts +32 -0
  34. package/dist/camera/cameraErrorMessages.js +53 -0
  35. package/dist/camera/selectCaptureDevice.d.ts +5 -1
  36. package/dist/camera/selectCaptureDevice.js +22 -2
  37. package/dist/camera/useCapture.js +38 -0
  38. package/dist/index.d.ts +5 -8
  39. package/dist/index.js +11 -34
  40. package/dist/stitching/incremental.d.ts +1 -117
  41. package/dist/stitching/stitchVideo.d.ts +0 -35
  42. package/dist/types.d.ts +0 -87
  43. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
  44. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
  45. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
  46. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
  47. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
  48. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
  49. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
  50. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
  51. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
  52. package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
  53. package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
  54. package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
  55. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
  56. package/package.json +3 -2
  57. package/src/ar/useARSession.ts +35 -5
  58. package/src/camera/Camera.tsx +63 -24
  59. package/src/camera/CaptureStitchStatsToast.tsx +58 -14
  60. package/src/camera/PanoramaSettings.ts +16 -289
  61. package/src/camera/PanoramaSettingsBridge.ts +3 -114
  62. package/src/camera/PanoramaSettingsModal.tsx +14 -1
  63. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
  64. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
  65. package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
  66. package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
  67. package/src/camera/buildPanoramaInitialSettings.ts +17 -0
  68. package/src/camera/cameraErrorMessages.ts +84 -0
  69. package/src/camera/selectCaptureDevice.ts +28 -3
  70. package/src/camera/useCapture.ts +44 -1
  71. package/src/index.ts +11 -40
  72. package/src/stitching/incremental.ts +3 -140
  73. package/src/stitching/stitchVideo.ts +0 -26
  74. package/src/types.ts +0 -95
  75. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
  76. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
  77. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
  78. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
  79. package/cpp/stitcher_frame_jsi.cpp +0 -214
  80. package/cpp/stitcher_frame_jsi.hpp +0 -108
  81. package/cpp/stitcher_proxy_jsi.cpp +0 -109
  82. package/cpp/stitcher_proxy_jsi.hpp +0 -46
  83. package/cpp/stitcher_worklet_dispatch.cpp +0 -103
  84. package/cpp/stitcher_worklet_dispatch.hpp +0 -71
  85. package/cpp/stitcher_worklet_registry.cpp +0 -91
  86. package/cpp/stitcher_worklet_registry.hpp +0 -146
  87. package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
  88. package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
  89. package/dist/stitching/IncrementalStitcherView.js +0 -157
  90. package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
  91. package/dist/stitching/StitcherWorkletRegistry.js +0 -78
  92. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
  93. package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
  94. package/dist/stitching/useFrameProcessor.d.ts +0 -119
  95. package/dist/stitching/useFrameProcessor.js +0 -196
  96. package/dist/stitching/useFrameStream.d.ts +0 -34
  97. package/dist/stitching/useFrameStream.js +0 -234
  98. package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
  99. package/dist/stitching/useThrottledFrameProcessor.js +0 -132
  100. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
  101. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
  102. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
  104. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
  105. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
  106. package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
  107. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  108. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
  109. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
  110. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
  111. package/src/stitching/IncrementalStitcherView.tsx +0 -198
  112. package/src/stitching/StitcherWorkletRegistry.ts +0 -156
  113. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
  114. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
  115. package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
  116. package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
  117. package/src/stitching/useFrameProcessor.ts +0 -226
  118. package/src/stitching/useFrameStream.ts +0 -271
  119. package/src/stitching/useThrottledFrameProcessor.ts +0 -145
package/cpp/stitcher.cpp CHANGED
@@ -22,6 +22,7 @@
22
22
  #include <opencv2/stitching.hpp>
23
23
  #include <opencv2/stitching/detail/blenders.hpp>
24
24
  #include <opencv2/stitching/detail/camera.hpp>
25
+ #include <opencv2/stitching/detail/exposure_compensate.hpp>
25
26
  #include <opencv2/stitching/detail/matchers.hpp>
26
27
  #include <opencv2/stitching/detail/motion_estimators.hpp>
27
28
  #include <opencv2/stitching/detail/seam_finders.hpp>
@@ -39,6 +40,8 @@
39
40
  #include <unistd.h>
40
41
  #include <vector>
41
42
 
43
+ #include "warp_guard.hpp"
44
+
42
45
 
43
46
  namespace retailens {
44
47
 
@@ -146,21 +149,6 @@ cv::Mat bake_rotation(const cv::Mat& src, const std::string& orientation,
146
149
  return src;
147
150
  }
148
151
 
149
- // Crop the non-zero bounding rect from the stitched panorama. cv::
150
- // Stitcher's compose stage leaves a black border around the warped
151
- // region; we trim that here.
152
- cv::Mat crop_bbox(const cv::Mat& panorama) {
153
- cv::Mat gray;
154
- cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
155
- cv::Mat mask;
156
- cv::threshold(gray, mask, 0, 255, cv::THRESH_BINARY);
157
- cv::Rect bbox = cv::boundingRect(mask);
158
- if (bbox.width <= 0 || bbox.height <= 0) {
159
- return panorama;
160
- }
161
- return panorama(bbox).clone();
162
- }
163
-
164
152
  // Map cv::Stitcher::Status → StitchErrorCode. cv::Stitcher's enum
165
153
  // values aren't documented as ABI-stable so we don't rely on
166
154
  // numeric equality; switch through the named constants.
@@ -241,6 +229,61 @@ cv::Rect maxInscribedRectFromMask(const cv::Mat& mask) {
241
229
  return bestRect;
242
230
  }
243
231
 
232
+ // Pick the crop rectangle. Prefers the TRUE coverage mask from
233
+ // cv::Stitcher::resultMask() (0xFF where a frame painted, 0 where
234
+ // unfilled) so dark content is kept and only the never-covered wedges
235
+ // drop; falls back to a brightness mask if resultMask wasn't populated
236
+ // (older/edge OpenCV configs). `maskOut` receives the binary mask
237
+ // actually used, so the caller can crop a coverage sidecar to match.
238
+ cv::Rect choose_crop_rect(const cv::Mat& panorama,
239
+ const cv::Mat& coverage,
240
+ bool useInscribed,
241
+ const LogFn& logFn,
242
+ cv::Mat& maskOut) {
243
+ const bool haveCoverage =
244
+ (!coverage.empty() && coverage.size() == panorama.size());
245
+ cv::Mat mask;
246
+ if (haveCoverage) {
247
+ cv::Mat cov1 = coverage;
248
+ if (coverage.channels() != 1) {
249
+ cv::cvtColor(coverage, cov1, cv::COLOR_BGR2GRAY);
250
+ }
251
+ // Any painted pixel (>0) is "filled" — robust to feathered edges.
252
+ cv::threshold(cov1, mask, 0, 255, cv::THRESH_BINARY);
253
+ } else {
254
+ log_info(logFn, "[crop]",
255
+ "resultMask unusable (empty=%d size=%dx%d vs pano %dx%d) — "
256
+ "brightness-mask fallback",
257
+ coverage.empty() ? 1 : 0, coverage.cols, coverage.rows,
258
+ panorama.cols, panorama.rows);
259
+ cv::Mat gray;
260
+ cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
261
+ cv::threshold(gray, mask, 0, 255, cv::THRESH_BINARY);
262
+ }
263
+ maskOut = mask;
264
+
265
+ const cv::Rect bbox = cv::boundingRect(mask);
266
+ if (bbox.width <= 0 || bbox.height <= 0) {
267
+ return cv::Rect(0, 0, panorama.cols, panorama.rows);
268
+ }
269
+ if (!useInscribed) {
270
+ return bbox;
271
+ }
272
+ const cv::Rect inscribed = maxInscribedRectFromMask(mask);
273
+ if (inscribed.width <= 0 || inscribed.height <= 0) {
274
+ log_info(logFn, "[crop]",
275
+ "inscribed rect empty — bbox fallback (%dx%d)",
276
+ bbox.width, bbox.height);
277
+ return bbox;
278
+ }
279
+ log_info(logFn, "[crop]",
280
+ "inscribed %dx%d @ (%d,%d) via %s mask (bbox was %dx%d)",
281
+ inscribed.width, inscribed.height, inscribed.x, inscribed.y,
282
+ haveCoverage ? "coverage" : "brightness",
283
+ bbox.width, bbox.height);
284
+ return inscribed;
285
+ }
286
+
244
287
  } // namespace
245
288
 
246
289
 
@@ -526,13 +569,32 @@ static StitchResult stitchFramePathsImpl_(
526
569
  framesIncluded, framePaths.size(), finalThreshold, finalAttempt);
527
570
  log_info(logFn, "[memstat]", "phase=after_stitch rss=%.1f MB", rss_mb());
528
571
 
529
- // ── 4. Crop to non-zero bounding rect ──────────────────────────
530
- cv::Mat cropped = crop_bbox(panorama);
572
+ // ── 4. Crop (coverage-aware inscribed rect, or bbox) ───────────
573
+ // Pull cv::Stitcher's coverage mask (0xFF filled / 0 unfilled). It is
574
+ // computed during stitch(), so this is free and exact — dark content
575
+ // a frame painted is kept; only never-covered wedges drop.
576
+ cv::Mat coverage;
577
+ {
578
+ const cv::UMat rm = stitcher->resultMask();
579
+ if (!rm.empty()) {
580
+ rm.copyTo(coverage); // download UMat → Mat
581
+ }
582
+ }
583
+ cv::Mat cropMask;
584
+ const cv::Rect cropRect = choose_crop_rect(
585
+ panorama, coverage, config.useInscribedRectCrop, logFn, cropMask);
586
+ cv::Mat cropped = panorama(cropRect).clone();
587
+ // Crop the binary mask to the same rect → coverage sidecar (debug).
588
+ cv::Mat croppedCoverage;
589
+ if (cropMask.size() == panorama.size()) {
590
+ croppedCoverage = cropMask(cropRect).clone();
591
+ }
531
592
  log_info(logFn, "[dimstat]",
532
- "post-crop_bbox %dx%d → %dx%d data=%.2f MB (inscribedRect=%d, currently ignored)",
593
+ "post-crop %dx%d → %dx%d data=%.2f MB (inscribedRect=%d, coverage=%d)",
533
594
  panorama.cols, panorama.rows, cropped.cols, cropped.rows,
534
595
  mat_mb(cropped),
535
- config.useInscribedRectCrop ? 1 : 0);
596
+ config.useInscribedRectCrop ? 1 : 0,
597
+ coverage.empty() ? 0 : 1);
536
598
  log_info(logFn, "[memstat]", "phase=after_crop rss=%.1f MB", rss_mb());
537
599
 
538
600
  // ── 5. Bake rotation per capture orientation ───────────────────
@@ -565,6 +627,18 @@ static StitchResult stitchFramePathsImpl_(
565
627
  outputPath.c_str(), final_image.cols, final_image.rows);
566
628
  log_info(logFn, "[memstat]", "phase=after_imwrite rss=%.1f MB", rss_mb());
567
629
 
630
+ // Best-effort coverage sidecar (<output>.coverage.png), bake-rotated
631
+ // to align with the JPEG, for the debug harness. Never fails stitch.
632
+ if (!croppedCoverage.empty()) {
633
+ try {
634
+ const cv::Mat coverageRotated =
635
+ bake_rotation(croppedCoverage, config.captureOrientation, logFn);
636
+ cv::imwrite(outputPath + ".coverage.png", coverageRotated);
637
+ } catch (...) {
638
+ // sidecar is debug-only — ignore failures
639
+ }
640
+ }
641
+
568
642
  // ── 7. Fill the result ─────────────────────────────────────────
569
643
  const auto t1 = std::chrono::steady_clock::now();
570
644
  result.success = true;
@@ -823,6 +897,10 @@ StitchResult stitchFramePathsManual(
823
897
  // 5. CylindricalWarper warps each frame using cameras
824
898
  // 6. GraphCutSeamFinder + MultiBandBlender produce final panorama
825
899
  cv::Mat panorama;
900
+ // v0.15 — the blender's dst_mask (TRUE frame coverage) hoisted to
901
+ // outer scope so the crop + sidecar below use it instead of a
902
+ // brightness threshold (which drops dark content like a mirror).
903
+ cv::Mat coverageMask;
826
904
  // Breadcrumbs in the device console. If the next stitch
827
905
  // crashes, the last logged step pinpoints the failure point —
828
906
  // makes debugging without Xcode much faster. Prefix is
@@ -1542,6 +1620,44 @@ StitchResult stitchFramePathsManual(
1542
1620
  composeFrames.empty() ? 0 : composeFrames[0].rows,
1543
1621
  compose_scale);
1544
1622
 
1623
+ // Step 7.6: cylindrical-fallback pre-pass. The configured warper
1624
+ // (plane by default) projects as ~tan(theta), so a wide ultra-wide
1625
+ // (0.5x) sweep can blow a single frame's warp canvas past the
1626
+ // 100 MP guard and hard-fail with "degenerate camera params".
1627
+ // warpRoi() is a cheap corner projection (no pixel work), so probe
1628
+ // every frame here; if any would diverge AND we're not already on
1629
+ // the bounded cylindrical projection, fall back to cylindrical for
1630
+ // the one real warp pass below. Everything downstream (seam /
1631
+ // blender / compose / crop) consumes the warper's OUTPUTS, so the
1632
+ // swap is transparent. If even cylindrical diverges, the in-loop
1633
+ // guard (step8b) still throws — the genuine-failure safety net.
1634
+ if (config.warperType != "cylindrical" && !composeFrames.empty()) {
1635
+ bool wouldDiverge = false;
1636
+ size_t divergeFrame = 0;
1637
+ for (size_t i = 0; i < composeFrames.size(); i++) {
1638
+ if (composeFrames[i].empty()) continue;
1639
+ cv::Mat preK;
1640
+ cameras[i].K().convertTo(preK, CV_32F);
1641
+ const cv::Rect r = warper->warpRoi(
1642
+ composeFrames[i].size(), preK, cameras[i].R);
1643
+ if (warpRoiExceedsGuard(r.width, r.height)) {
1644
+ wouldDiverge = true;
1645
+ divergeFrame = i;
1646
+ break;
1647
+ }
1648
+ }
1649
+ if (wouldDiverge) {
1650
+ log_info(logFn, "[stitch-bc]",
1651
+ "step7.6: '%s' warp diverges at frame %zu (>%lld MP "
1652
+ "guard) -- falling back to cylindrical projection",
1653
+ config.warperType.c_str(), divergeFrame,
1654
+ (long long)(kMaxWarpPixels / 1000000));
1655
+ if (auto cyl = make_warper("cylindrical")) {
1656
+ warper = cyl->create(warpedScale);
1657
+ }
1658
+ }
1659
+ }
1660
+
1545
1661
  // Step 8: warp + (optional) seam finder + blender feed.
1546
1662
  //
1547
1663
  // Two paths based on caller's seamFinderType:
@@ -1645,12 +1761,14 @@ StitchResult stitchFramePathsManual(
1645
1761
  // any panorama frame requiring more than 100 MP of
1646
1762
  // intermediate storage is from a broken estimator,
1647
1763
  // not a real capture worth completing.
1648
- constexpr int64_t kMaxWarpPixels = 100ll * 1000ll * 1000ll;
1649
1764
  const int64_t roiPixels =
1650
1765
  static_cast<int64_t>(roi.width)
1651
1766
  * static_cast<int64_t>(roi.height);
1652
- if (roi.width <= 0 || roi.height <= 0
1653
- || roiPixels > kMaxWarpPixels) {
1767
+ // Final safety net. If we reach here the warper in use
1768
+ // is already cylindrical (either the host chose it, or
1769
+ // the step7.6 pre-pass fell back to it) and STILL
1770
+ // diverges — a genuinely broken estimate, so fail.
1771
+ if (warpRoiExceedsGuard(roi.width, roi.height)) {
1654
1772
  log_error(logFn, "[stitch-bc]",
1655
1773
  "step8b: warpRoi degenerate for frame "
1656
1774
  "%zu (%dx%d = %lld px > %lld limit) — "
@@ -1831,6 +1949,30 @@ StitchResult stitchFramePathsManual(
1831
1949
  }
1832
1950
  masksWarpedU_seam.clear();
1833
1951
 
1952
+ // Exposure compensation — parity with cv::Stitcher::PANORAMA,
1953
+ // which runs a GainCompensator before blending. Without it,
1954
+ // per-frame auto-exposure differences surface as brightness
1955
+ // steps at the seams. The manual path previously skipped this
1956
+ // entirely (the high-level path Android uses gets it for free),
1957
+ // which is one reason iOS output looked worse. GAIN_BLOCKS
1958
+ // matches cv::Stitcher's default compensator.
1959
+ //
1960
+ // NOTE: BATCH path only — it has every warped frame in memory,
1961
+ // which the compensator needs before it can solve gains. The
1962
+ // STREAM path (low-RAM, one frame at a time) can't feed the
1963
+ // compensator globally and keeps its current no-compensation
1964
+ // behaviour; see docs/stitch-pipeline-architecture.md.
1965
+ auto compensator = cv::detail::ExposureCompensator::createDefault(
1966
+ cv::detail::ExposureCompensator::GAIN_BLOCKS);
1967
+ {
1968
+ std::vector<cv::UMat> compImgs(N), compMasks(N);
1969
+ for (size_t i = 0; i < N; i++) {
1970
+ imagesWarped[i].copyTo(compImgs[i]);
1971
+ masksWarped[i].copyTo(compMasks[i]);
1972
+ }
1973
+ compensator->feed(corners, compImgs, compMasks);
1974
+ }
1975
+
1834
1976
  // Feed the blender, releasing each frame as we go.
1835
1977
  log_info(logFn, "[stitch-bc]", "step10a: blender->prepare");
1836
1978
  blender->prepare(corners, sizes);
@@ -1838,6 +1980,10 @@ StitchResult stitchFramePathsManual(
1838
1980
  "step10b: feeding blender (N=%zu)", N);
1839
1981
  for (size_t i = 0; i < N; i++) {
1840
1982
  log_info(logFn, "[stitch-bc]", "step10c: feed frame %zu", i);
1983
+ // Apply the per-frame exposure gain solved above, in place,
1984
+ // before converting + feeding the blender.
1985
+ compensator->apply(static_cast<int>(i), corners[i],
1986
+ imagesWarped[i], masksWarped[i]);
1841
1987
  cv::Mat imgS;
1842
1988
  imagesWarped[i].convertTo(imgS, CV_16S);
1843
1989
  blender->feed(imgS, masksWarped[i], corners[i]);
@@ -1899,6 +2045,9 @@ StitchResult stitchFramePathsManual(
1899
2045
  "step11b: blend complete (panoramaS=%dx%d)",
1900
2046
  panoramaS.cols, panoramaS.rows);
1901
2047
  panoramaS.convertTo(panorama, CV_8U);
2048
+ // Keep the blend coverage mask alive past this try scope (ref-
2049
+ // counted, so this is cheap) for the crop + sidecar below.
2050
+ coverageMask = panoramaMask;
1902
2051
  {
1903
2052
  auto _t = std::chrono::steady_clock::now();
1904
2053
  double _ms = std::chrono::duration_cast<std::chrono::milliseconds>(
@@ -2030,25 +2179,37 @@ StitchResult stitchFramePathsManual(
2030
2179
  // < 50% of bbox area, use bbox — the mask shape is pathological
2031
2180
  // and shipping bbox-with-corners is better than a sliver.
2032
2181
  cv::Mat finalImage = panorama;
2182
+ // v0.15 — coverage cropped to the same region(s) as finalImage, for
2183
+ // the debug-harness sidecar written after rotation below.
2184
+ cv::Mat coverageCropped;
2185
+ const bool haveCoverage =
2186
+ (!coverageMask.empty() && coverageMask.size() == panorama.size());
2033
2187
  try {
2034
- cv::Mat gray;
2035
- cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
2188
+ // Prefer the TRUE coverage mask (blender dst_mask): dark content a
2189
+ // frame painted is kept; only never-covered pixels drop. Fall back
2190
+ // to a brightness mask only if coverage is somehow unavailable.
2036
2191
  cv::Mat mask;
2037
- cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
2192
+ if (haveCoverage) {
2193
+ cv::threshold(coverageMask, mask, 0, 255, cv::THRESH_BINARY);
2194
+ } else {
2195
+ cv::Mat gray;
2196
+ cv::cvtColor(panorama, gray, cv::COLOR_BGR2GRAY);
2197
+ cv::threshold(gray, mask, 1, 255, cv::THRESH_BINARY);
2198
+ }
2038
2199
 
2039
2200
  // V16 Phase 1b.fix5c — operator-toggleable crop strategy.
2040
2201
  //
2041
- // useInscribedRectCrop = NO (default in settings modal):
2202
+ // useInscribedRectCrop = NO (v0.15 default):
2042
2203
  // Final crop is just cv::boundingRect(mask) — preserves all
2043
2204
  // stitched content at the cost of possible black corners
2044
2205
  // where cv::Stitcher's projection didn't fill.
2045
2206
  //
2046
- // useInscribedRectCrop = YES (operator opt-in):
2207
+ // useInscribedRectCrop = YES (opt in via prop / settings modal):
2047
2208
  // Run the full inscribed-rect pipeline (morph-close + 50%
2048
2209
  // safety floor + column-projection second pass) for a clean
2049
2210
  // -cornered rectangle. Can over-aggressively shrink the
2050
2211
  // output on lopsided masks (1146×1102 bbox → 602×1102 strip
2051
- // in one field log).
2212
+ // in one field log) — which is why it's opt-in, not the default.
2052
2213
  cv::Rect bbox;
2053
2214
  if (config.useInscribedRectCrop) {
2054
2215
  cv::Mat closedMask;
@@ -2091,6 +2252,7 @@ StitchResult stitchFramePathsManual(
2091
2252
  if (bbox.width > 0 && bbox.height > 0
2092
2253
  && bbox.width <= panorama.cols && bbox.height <= panorama.rows) {
2093
2254
  finalImage = panorama(bbox).clone();
2255
+ if (haveCoverage) coverageCropped = coverageMask(bbox).clone();
2094
2256
  }
2095
2257
 
2096
2258
  // V16 Phase 1b.fix5c — column-projection second pass ALSO gated
@@ -2140,6 +2302,8 @@ StitchResult stitchFramePathsManual(
2140
2302
  cv::Rect rectCrop(cropLeft, 0,
2141
2303
  cropRight - cropLeft + 1, rows);
2142
2304
  finalImage = finalImage(rectCrop).clone();
2305
+ if (!coverageCropped.empty())
2306
+ coverageCropped = coverageCropped(rectCrop).clone();
2143
2307
  log_info(logFn, "[BatchStitcher]",
2144
2308
  "rectCrop applied: %dx%d → %dx%d",
2145
2309
  cols, rows, finalImage.cols, finalImage.rows);
@@ -2163,6 +2327,8 @@ StitchResult stitchFramePathsManual(
2163
2327
  cv::Rect rectCrop(cropLeft, 0,
2164
2328
  cropRight - cropLeft + 1, rows);
2165
2329
  finalImage = finalImage(rectCrop).clone();
2330
+ if (!coverageCropped.empty())
2331
+ coverageCropped = coverageCropped(rectCrop).clone();
2166
2332
  log_info(logFn, "[BatchStitcher]",
2167
2333
  "rectCrop relaxed applied: %dx%d → %dx%d",
2168
2334
  cols, rows, finalImage.cols, finalImage.rows);
@@ -2264,6 +2430,20 @@ StitchResult stitchFramePathsManual(
2264
2430
  config.captureOrientation,
2265
2431
  logFn);
2266
2432
 
2433
+ // v0.15 — best-effort coverage sidecar (<output>.coverage.png),
2434
+ // cropped + rotated to match the written JPEG, for the debug harness
2435
+ // (computeInscribedRect / debugMaskOverlay prefer it over brightness).
2436
+ if (!coverageCropped.empty()) {
2437
+ try {
2438
+ const cv::Mat covRot = bake_rotation(coverageCropped,
2439
+ config.captureOrientation,
2440
+ logFn);
2441
+ cv::imwrite(outputPath + ".coverage.png", covRot);
2442
+ } catch (...) {
2443
+ // sidecar is debug-only — ignore failures
2444
+ }
2445
+ }
2446
+
2267
2447
  // Encode + write the JPEG. Clamp quality into [0, 100] to defend
2268
2448
  // against caller bugs.
2269
2449
  //
@@ -11,16 +11,23 @@
11
11
  #
12
12
  # Or, from the repo root: `scripts/run-cpp-tests.sh`.
13
13
  #
14
- # What this DOES test (v0.10.0 scope):
14
+ # What this DOES test:
15
15
  # - Pure-C++ types in cpp/: `Pose`, `PlaneTransform`,
16
16
  # `StitcherFrameData`, `PixelBufferReader` interface contract.
17
- # - `StitcherWorkletRegistry` lifecycle (count, snapshot, uninstall,
18
- # thread-safety) compiled against JSI/worklets-core stubs under
19
- # `cpp/tests/stubs/`. See `stubs/jsi/jsi.h` for the strategy.
17
+ # - `timeBudgetCrossed` the keyframe gate's OpenCV-free time-budget
18
+ # force-accept predicate (keyframe_timebudget_test.cpp).
19
+ # - `warpRoiExceedsGuard` the OpenCV-free warp-canvas size guard that
20
+ # triggers the cylindrical-fallback pre-pass (warp_guard_test.cpp).
21
+ # (The StitcherWorkletRegistry test was removed when that source was
22
+ # archived in the Phase-3 cleanup — its CMake entries had gone stale
23
+ # and broke the whole suite's configure step.)
20
24
  #
21
25
  # What this DOES NOT test yet (deferred to v0.11.0+):
22
- # - `KeyframeGate` — depends on OpenCV. Will need an OpenCV-aware
23
- # CMake config (link the same opencv_world the prod build uses).
26
+ # - `KeyframeGate` (the full gate) — depends on OpenCV. Will need an
27
+ # OpenCV-aware CMake config (link the same opencv_world the prod
28
+ # build uses). NOTE: the OpenCV-free `timeBudgetCrossed` predicate
29
+ # (the time-budget force-accept boundary logic) IS covered, via
30
+ # keyframe_timebudget_test.cpp — it's an inline header-only function.
24
31
  # - JSI host-object dispatch — needs a real Hermes runtime.
25
32
  # - Anything in `stitcher.cpp` (uses OpenCV stitching pipeline).
26
33
 
@@ -66,14 +73,13 @@ include_directories(
66
73
  # Test executable
67
74
  # ─────────────────────────────────────────────────────────────────────
68
75
  add_executable(stitcher_cpp_tests
69
- # Production sources under test (only those whose deps we can satisfy
70
- # without OpenCV / a real JSI runtime).
71
- ${CMAKE_CURRENT_SOURCE_DIR}/../stitcher_worklet_registry.cpp
72
-
73
- # Test sources.
76
+ # Test sources. All subjects are header-only (Pose / PlaneTransform,
77
+ # StitcherFrameData, and the gate's inline `timeBudgetCrossed`
78
+ # predicate), so no production .cpp needs linking here.
74
79
  ${CMAKE_CURRENT_SOURCE_DIR}/pose_test.cpp
75
80
  ${CMAKE_CURRENT_SOURCE_DIR}/stitcher_frame_data_test.cpp
76
- ${CMAKE_CURRENT_SOURCE_DIR}/stitcher_worklet_registry_test.cpp
81
+ ${CMAKE_CURRENT_SOURCE_DIR}/keyframe_timebudget_test.cpp
82
+ ${CMAKE_CURRENT_SOURCE_DIR}/warp_guard_test.cpp
77
83
  )
78
84
 
79
85
  target_link_libraries(stitcher_cpp_tests
@@ -0,0 +1,65 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ //
3
+ // keyframe_timebudget_test.cpp — host unit tests for the pure
4
+ // `retailens::timeBudgetCrossed` predicate (the keyframe gate's
5
+ // time-budget force-accept decision).
6
+ //
7
+ // The full KeyframeGate depends on OpenCV and cannot run in this
8
+ // harness (see CMakeLists.txt). The time-budget boundary logic was
9
+ // therefore deliberately extracted into an inline, OpenCV-free
10
+ // predicate in keyframe_gate.hpp precisely so it CAN be unit-tested
11
+ // here without linking the gate's OpenCV-dependent .cpp.
12
+
13
+ #include <gtest/gtest.h>
14
+
15
+ #include "keyframe_gate.hpp"
16
+
17
+ using retailens::timeBudgetCrossed;
18
+
19
+ // intervalMs <= 0 disables the budget entirely (opt-out path).
20
+ TEST(TimeBudgetCrossed, DisabledWhenIntervalNonPositive) {
21
+ EXPECT_FALSE(timeBudgetCrossed(0.0, 1000, 999999));
22
+ EXPECT_FALSE(timeBudgetCrossed(-5.0, 1000, 999999));
23
+ }
24
+
25
+ // Never fires before the first accept (lastAcceptMs sentinel -1),
26
+ // regardless of how large `nowMs` is.
27
+ TEST(TimeBudgetCrossed, NeverFiresBeforeFirstAccept) {
28
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, -1, 1000000));
29
+ }
30
+
31
+ // Fires exactly at the boundary (elapsed == interval) and beyond.
32
+ TEST(TimeBudgetCrossed, FiresAtAndAfterBoundary) {
33
+ EXPECT_TRUE(timeBudgetCrossed(2000.0, 1000, 3000)); // elapsed == 2000
34
+ EXPECT_TRUE(timeBudgetCrossed(2000.0, 1000, 5000)); // elapsed > 2000
35
+ }
36
+
37
+ // Does NOT fire just under the boundary.
38
+ TEST(TimeBudgetCrossed, DoesNotFireJustUnderBoundary) {
39
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, 1000, 2999)); // elapsed == 1999
40
+ }
41
+
42
+ // A backwards or equal clock must never fire. A monotonic source
43
+ // should prevent now < lastAccept, but the predicate must be robust.
44
+ TEST(TimeBudgetCrossed, BackwardsOrEqualClockDoesNotFire) {
45
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, 5000, 4000)); // now < lastAccept
46
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, 5000, 5000)); // elapsed 0 < 2000
47
+ }
48
+
49
+ // Sub-millisecond budget must NOT collapse to "accept every frame":
50
+ // the predicate compares elapsed in double, so a 0.5 ms budget needs
51
+ // ~1 ms elapsed (not 0). Guards the truncation regression.
52
+ TEST(TimeBudgetCrossed, SubMillisecondBudgetDoesNotAcceptEveryFrame) {
53
+ EXPECT_FALSE(timeBudgetCrossed(0.5, 1000, 1000)); // elapsed 0.0 < 0.5
54
+ EXPECT_TRUE(timeBudgetCrossed(0.5, 1000, 1001)); // elapsed 1.0 >= 0.5
55
+ }
56
+
57
+ // Realistic 2 s budget across a slow pan: a keyframe accepted at t,
58
+ // the next force-accept lands at t + 2000 ms, not before.
59
+ TEST(TimeBudgetCrossed, TwoSecondBudgetTypicalUse) {
60
+ const int64_t lastAccept = 10000;
61
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, lastAccept, lastAccept + 1500));
62
+ EXPECT_FALSE(timeBudgetCrossed(2000.0, lastAccept, lastAccept + 1999));
63
+ EXPECT_TRUE(timeBudgetCrossed(2000.0, lastAccept, lastAccept + 2000));
64
+ EXPECT_TRUE(timeBudgetCrossed(2000.0, lastAccept, lastAccept + 2001));
65
+ }
@@ -0,0 +1,48 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ /**
3
+ * Unit tests for the warp-canvas size guard (cpp/warp_guard.hpp).
4
+ *
5
+ * The guard decides when a warp ROI is "degenerate" — the trigger for
6
+ * both the cylindrical-fallback pre-pass and the in-loop final safety net.
7
+ * The cases that matter: normal ROIs pass, non-positive dims fail, the
8
+ * 100 MP boundary is inclusive, the real observed divergence (8171×12336)
9
+ * is caught, and a ROI whose int32 area would overflow is still caught.
10
+ */
11
+ #include "warp_guard.hpp"
12
+
13
+ #include <gtest/gtest.h>
14
+
15
+ using retailens::warpRoiExceedsGuard;
16
+
17
+ TEST(WarpGuard, AcceptsNormalRoi) {
18
+ EXPECT_FALSE(warpRoiExceedsGuard(4000, 2000)); // 8 MP
19
+ EXPECT_FALSE(warpRoiExceedsGuard(1, 1));
20
+ }
21
+
22
+ TEST(WarpGuard, RejectsNonPositiveDims) {
23
+ EXPECT_TRUE(warpRoiExceedsGuard(0, 1000));
24
+ EXPECT_TRUE(warpRoiExceedsGuard(1000, 0));
25
+ EXPECT_TRUE(warpRoiExceedsGuard(-5, 1000));
26
+ EXPECT_TRUE(warpRoiExceedsGuard(1000, -5));
27
+ }
28
+
29
+ TEST(WarpGuard, BoundaryIsInclusive) {
30
+ EXPECT_FALSE(warpRoiExceedsGuard(100000, 1000)); // exactly 100 MP — allowed
31
+ EXPECT_TRUE(warpRoiExceedsGuard(100000, 1001)); // 100.1 MP — over
32
+ }
33
+
34
+ TEST(WarpGuard, RejectsTheObservedDivergence) {
35
+ // 8171×12336 = 100.8 MP — the exact STITCH_CAMERA_PARAMS_FAIL canvas.
36
+ EXPECT_TRUE(warpRoiExceedsGuard(8171, 12336));
37
+ }
38
+
39
+ TEST(WarpGuard, RejectsInt32OverflowingRoi) {
40
+ // 65536×65536 = 2^32; an int32 area would wrap to 0 and slip past the
41
+ // guard. The int64 area math catches it.
42
+ EXPECT_TRUE(warpRoiExceedsGuard(65536, 65536));
43
+ }
44
+
45
+ TEST(WarpGuard, HonoursCustomThreshold) {
46
+ EXPECT_FALSE(warpRoiExceedsGuard(1000, 1000, 2'000'000)); // 1 MP < 2 MP
47
+ EXPECT_TRUE(warpRoiExceedsGuard(2000, 1000, 1'000'000)); // 2 MP > 1 MP
48
+ }
@@ -0,0 +1,41 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ #pragma once
3
+ #include <cstdint>
4
+
5
+ // ─────────────────────────────────────────────────────────────────────
6
+ // Warp-canvas size guard — shared by the warp pre-pass (which decides
7
+ // whether to fall back from a diverging plane projection to the bounded
8
+ // cylindrical one) and the in-loop final safety net in stitcher.cpp.
9
+ //
10
+ // Header-only + OpenCV-free on purpose: the predicate below is the one
11
+ // piece of warp-guard logic worth unit-testing in isolation (the int64
12
+ // overflow handling in particular), and the cpp test suite is NOT
13
+ // OpenCV-aware. Keep this file free of any cv:: dependency.
14
+ // ─────────────────────────────────────────────────────────────────────
15
+
16
+ namespace retailens {
17
+
18
+ // A single warped frame requiring more than this many pixels of
19
+ // intermediate storage is from a broken estimator (degenerate camera
20
+ // params), not a real capture: at 3-4 bytes/px that's 300-400 MB for ONE
21
+ // frame, and blending several would jetsam-OOM the app. 100 megapixels.
22
+ constexpr int64_t kMaxWarpPixels = 100LL * 1000LL * 1000LL;
23
+
24
+ // True if a warp ROI of `width`×`height` px is degenerate: non-positive
25
+ // in either dimension, or strictly larger than `maxPixels` (so a canvas
26
+ // exactly at the limit is still allowed).
27
+ //
28
+ // Computes the area in int64 so a wildly degenerate ROI (e.g.
29
+ // 65536×65536 = 2^32, whose int32 area wraps to 0) is still caught
30
+ // instead of silently slipping past the guard.
31
+ inline bool warpRoiExceedsGuard(int width, int height,
32
+ int64_t maxPixels = kMaxWarpPixels) {
33
+ if (width <= 0 || height <= 0) {
34
+ return true;
35
+ }
36
+ const int64_t pixels =
37
+ static_cast<int64_t>(width) * static_cast<int64_t>(height);
38
+ return pixels > maxPixels;
39
+ }
40
+
41
+ } // namespace retailens
@@ -67,6 +67,15 @@ export interface UseARSessionReturn {
67
67
  * older iPhones, simulators, and unsupported Android devices.
68
68
  */
69
69
  isAvailable: boolean;
70
+ /**
71
+ * Whether the one-shot `isSupported()` probe has resolved (success OR
72
+ * failure). `false` only during the brief async window right after
73
+ * mount; `true` thereafter. Lets consumers distinguish "AR not
74
+ * supported" (probed && !isAvailable) from "support not yet known"
75
+ * (!probed), so they don't prematurely mount the non-AR camera and
76
+ * lose a camera-handoff race when AR is the intended source.
77
+ */
78
+ supportProbed: boolean;
70
79
  /**
71
80
  * Whether the session is currently running. True between
72
81
  * `start()` and `stop()`.
@@ -56,6 +56,7 @@ function getNativeModule() {
56
56
  const STATE_POLL_INTERVAL_MS = 500;
57
57
  function useARSession() {
58
58
  const [isAvailable, setIsAvailable] = (0, react_1.useState)(false);
59
+ const [supportProbed, setSupportProbed] = (0, react_1.useState)(false);
59
60
  const [isRunning, setIsRunning] = (0, react_1.useState)(false);
60
61
  const [trackingState, setTrackingState] = (0, react_1.useState)(ARTrackingState.NotAvailable);
61
62
  const pollRef = (0, react_1.useRef)(null);
@@ -64,12 +65,32 @@ function useARSession() {
64
65
  // AR support shouldn't crash anything — `isAvailable` stays
65
66
  // false and the rest of the SDK falls back to vision-camera.
66
67
  (0, react_1.useEffect)(() => {
67
- if (!native)
68
+ if (!native) {
69
+ // No native module at all — treat the probe as resolved
70
+ // (unsupported) so consumers don't wait forever for AR.
71
+ setSupportProbed(true);
68
72
  return;
69
- native.isSupported().then(setIsAvailable).catch((err) => {
73
+ }
74
+ let cancelled = false;
75
+ native
76
+ .isSupported()
77
+ .then((ok) => {
78
+ if (!cancelled)
79
+ setIsAvailable(ok);
80
+ })
81
+ .catch((err) => {
70
82
  // eslint-disable-next-line no-console
71
83
  console.warn('[useARSession] isSupported failed', err);
84
+ })
85
+ .finally(() => {
86
+ // Mark the probe resolved either way so the non-AR fallback
87
+ // (or AR mount) can proceed exactly once support is known.
88
+ if (!cancelled)
89
+ setSupportProbed(true);
72
90
  });
91
+ return () => {
92
+ cancelled = true;
93
+ };
73
94
  }, [native]);
74
95
  const stopPolling = (0, react_1.useCallback)(() => {
75
96
  if (pollRef.current !== null) {
@@ -122,6 +143,7 @@ function useARSession() {
122
143
  }, [native]);
123
144
  return {
124
145
  isAvailable,
146
+ supportProbed,
125
147
  isRunning,
126
148
  trackingState,
127
149
  start,